MapReduce清洗日志数据统计PV量

 package mapreduce.webpv;

 import java.io.IOException;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;

 public class WebPvMapReduce extends Configured implements Tool {

     // step 1: Mapper
     public static class WebPvMapper extends
             Mapper<LongWritable, Text, IntWritable, IntWritable> {
         private IntWritable mapOutputKey = new IntWritable();
         private IntWritable mapOutputValue = new IntWritable(1);

         @Override
         public void map(LongWritable key, Text value, Context context)
                 throws IOException, InterruptedException {

             // line value
             String lineValue = value.toString();

             // spilt
             String[] values = lineValue.split("\t");

             // url
             String urlValue = values[1];

             if (StringUtils.isBlank(urlValue)) {
                 // conuter
                 context.getCounter("WEBPVMAPPER_CUUNTERS", "URL_BLANK")
                         .increment(1L);
                 return;
             }

             if (30 > values.length) {

                 // conuter
                 context.getCounter("WEBPVMAPPER_CUUNTERS", "LENGTH_LT_30")
                         .increment(1L);

                 return;
             }

             // province id
             String provinceIdValue = values[23];

             if (StringUtils.isBlank(provinceIdValue)) {
                 // conuter
                 context.getCounter("WEBPVMAPPER_CUUNTERS", "PROVINCEID_BLANK")
                         .increment(1L);
                 return;
             }

             Integer provinceId = Integer.MAX_VALUE;
             try {
                 provinceId = Integer.valueOf(provinceIdValue);
             } catch (Exception e) {
                 // conuter
                 context.getCounter("WEBPVMAPPER_CUUNTERS",
                         "PROVINCEID_NOT_NUMBER").increment(1L);
                 return;
             }

             // map outpu key
             mapOutputKey.set(provinceId);

             context.write(mapOutputKey, mapOutputValue);
         }
     }

     // step 2: Reducer
     public static class WebPvReducer extends
             Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
         private IntWritable outputValue = new IntWritable();

         @Override
         protected void reduce(IntWritable key, Iterable<IntWritable> values,
                 Context context) throws IOException, InterruptedException {
             // temp sum
             int sum = 0;

             // iterator
             for (IntWritable value : values) {
                 sum += value.get();
             }

             // set output
             outputValue.set(sum);

             context.write(key, outputValue);
         }
     }

     // step 3: Driver
     public int run(String[] args) throws Exception {

         Configuration configuration = this.getConf();

         Job job = Job.getInstance(configuration, this.getClass()
                 .getSimpleName());
         job.setJarByClass(WebPvMapReduce.class);

         // set job
         // input
         Path inpath = new Path(args[0]);
         FileInputFormat.addInputPath(job, inpath);

         // output
         Path outPath = new Path(args[1]);
         FileOutputFormat.setOutputPath(job, outPath);

         // Mapper
         job.setMapperClass(WebPvMapper.class);
         job.setMapOutputKeyClass(IntWritable.class);
         job.setMapOutputValueClass(IntWritable.class);

         // Reducer
         job.setReducerClass(WebPvReducer.class);
         job.setOutputKeyClass(IntWritable.class);
         job.setOutputValueClass(IntWritable.class);

         // submit job -> YARN
         boolean isSuccess = job.waitForCompletion(true);
         return isSuccess ? 0 : 1;
     }

     public static void main(String[] args) throws Exception {

         Configuration configuration = new Configuration();

         args = new String[] {
                 "hdfs://beifeng01:8020//user/beifeng01/mapreduce/input/testdata/2015082818",
                 "hdfs://beifeng01:8020//user/beifeng01/mapreduce/output1" };

         int status = ToolRunner.run(configuration, new WebPvMapReduce(), args);

         // exit program
         System.exit(status);
     }
 }

查看结果

 $ bin/hdfs dfs -text /user/beifeng01/mapreduce/output1/pa*
 1       3527
 2       1672
 3       511
 4       325
 5       776
 6       661
 7       95
 8       80
 9       183
 10      93
 11      135
 12      289
 13      264
 14      374
 15      163
 16      419
 17      306
 18      272
 19      226
 20      2861
 21      124
 22      38
 23      96
 24      100
 25      20
 26      157
 27      49
 28      21
 29      85
 30      42
 32      173