大数据作业之利用MapRedeuce实现简单的数据操作

Map/Reduce编程作业

现有student.txt和student_score.txt。将两个文件上传到hdfs上。使用Map/Reduce框架完成下面的题目

student.txt

2016001,王毅
2016002,张小明
2016003,李学彭
2016004,王东
2016005,王笑笑

student_score.txt

2016001,操作系统,60
2016001,数据库,88
2016001,大数据概论,85
2016002,操作系统,91
2016002,大数据概论,91
2016003,大数据概论,56
2016003,操作系统,88
2016004,数据库,90
2016004,大数据概论,82
2016004,操作系统,78
2016005,操作系统,69
2016005,大数据概论,70
2016005,数据库,89

1）将stduent.txt和student_score.txt连接，输出学号、姓名、课程、分数字段。

2）统计每个同学的平均成绩，显示学号、姓名和平均成绩，并按照成绩高低降序排序。

3）统计每门课的最高分、最低分和平均分。

问题一：

StudentScore1.java

import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class StudentScore1 {

	public static void main(String[] args) throws IllegalArgumentException, IOException, ClassNotFoundException, InterruptedException {
		Configuration conf=new Configuration();
		Job job=Job.getInstance(conf,"StudentScore1");
		job.setJarByClass(StudentScore1.class);

		job.setMapperClass(ScoreMapper.class);
		//Map的输出，避免程序不确定Map输出的值的类型不确定
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(SC.class);

		job.setReducerClass(ScoreReduce.class);
		//输出类型
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(NullWritable.class);

		//数据来源
		FileInputFormat.addInputPath(job,new Path("/StudentInput"));
		//输出位置
		FileOutputFormat.setOutputPath(job, new Path("/Output1"));

		System.exit(job.waitForCompletion(true)?0:1);
	}
     public static class ScoreMapper extends Mapper<Object, Text, Text, SC>{

		@Override
		protected void map(Object key, Text value, Mapper<Object, Text, Text, SC>.Context context)
				throws IOException, InterruptedException {
			//以“,”分割字符串
			//Student         2016001,王毅          [2016001,王毅]
			//Student_score   2016001,操作系统,60    [2016001,操作系统,60]
			String[] words=value.toString().split(",");
			//记录学号
			String Sid=words[0];
			SC sc=new SC();
			//区分字符串属于那个类型
			if(words.length==2) {//长度为2的记录信息是  学生
				sc.setSid(Sid);
				sc.setName(words[1]);
				sc.setTable("Student");
			    context.write(new Text(Sid), sc);
			}else {//长度为3的记录信息是   学科成绩
			sc.setSid(Sid);
			sc.setCourse(words[1]);
			sc.setScore(Integer.parseInt(words[2]));
			sc.setTable("Student_score");
			context.write(new Text(Sid), sc);
		     }

         }
      }
     public static class ScoreReduce extends Reducer<Text, SC, Text, NullWritable>{

		@Override
		protected void reduce(Text key, Iterable<SC> values,
				Reducer<Text, SC, Text,NullWritable>.Context context) throws IOException, InterruptedException {

			List<SC> list=new ArrayList<SC>();
			String Name="";
			//遍历结果集的value
			for(SC value:values) {

				if(value.getTable().equals("Student")) {//只有姓名信息的记录下来
					Name=value.getName();
				}else {//否则，将其添加到待输出list中
					SC sc=new SC();
					try {
						BeanUtils.copyProperties(sc, value);
						list.add(sc);
					} catch (IllegalAccessException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					} catch (InvocationTargetException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}

				}
			}
			//遍历list
			for(SC sc:list) {
				sc.setName(Name);
				context.write(new Text(sc.toString()), NullWritable.get());
			}
		}

     }
}

SC.java

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Writable;

public class SC implements Writable{

	private String Name="";
	private String Sid="";
	private String Course="";
	private String Table="";
	private int Score=0;
	public String getName() {
		return Name;
	}
	public void setName(String name) {
		Name = name;
	}
	public String getSid() {
		return Sid;
	}
	public void setSid(String sid) {
		Sid = sid;
	}
	public String getCourse() {
		return Course;
	}
	public void setCourse(String course) {
		Course = course;
	}
	public String getTable() {
		return Table;
	}
	public void setTable(String table) {
		Table = table;
	}
	public int getScore() {
		return Score;
	}
	public void setScore(int score) {
		Score = score;
	}

	@Override
	public String toString() {
		return  Sid + "," + Name + "," + Course + "," + Score;
	}
	@Override
	public void readFields(DataInput in) throws IOException {
		this.Sid=in.readUTF();
		this.Name=in.readUTF();
		this.Course=in.readUTF();
		this.Table=in.readUTF();
		this.Score=in.readInt();

	}
	@Override
	public void write(DataOutput out) throws IOException {
		out.writeUTF(Sid);
		out.writeUTF(Name);
		out.writeUTF(Course);
		out.writeUTF(Table);
		out.writeInt(Score);
	}
}

结果：

2016001,王毅,操作系统,60
2016001,王毅,数据库,88
2016001,王毅,大数据概论,85
2016002,张小明,操作系统,91
2016002,张小明,大数据概论,91
2016003,李学彭,操作系统,88
2016003,李学彭,大数据概论,56
2016004,王东,大数据概论,82
2016004,王东,操作系统,78
2016004,王东,数据库,90
2016005,王笑笑,数据库,89
2016005,王笑笑,操作系统,69
2016005,王笑笑,大数据概论,70

问题二：

Average2.java

import java.io.IOException;
import java.util.Comparator;
import java.util.TreeMap;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class Average2 {

	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		Configuration conf=new Configuration();
		Job job=Job.getInstance(conf,"Average2");

		job.setJarByClass(Average2.class);
		job.setMapperClass(Average2Mapper.class);
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(DoubleWritable.class);

		job.setReducerClass(Average2Reduce.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(DoubleWritable.class);

		FileInputFormat.addInputPath(job, new Path("/Output1"));
		FileOutputFormat.setOutputPath(job, new Path("/Output2"));
		System.exit(job.waitForCompletion(true)?0:1);

	}

     public static class Average2Mapper extends Mapper<Object,Text,Text,DoubleWritable>{
		@Override
		protected void map(Object key, Text value, Mapper<Object, Text, Text, DoubleWritable>.Context context)
				throws IOException, InterruptedException {
			//分割
			String[] words=value.toString().split(",");
			//keybuf=[2016001,王毅,]
			StringBuffer keybuf=new StringBuffer();
			keybuf.append(words[0]).append(",").append(words[1]).append(",");
			//score用来记录成绩
			Double score=Double.parseDouble(words[3]);
			context.write(new Text(keybuf.toString()), new DoubleWritable(score));
		}
     }

    public static class Average2Reduce extends Reducer<Text,DoubleWritable,Text,DoubleWritable>{
        //new Comparetor<Double> 的方法  倒叙（从高到低）排序
    	private TreeMap<Double, String> treeMap=new TreeMap<Double, String>(new Comparator<Double>() {
			@Override
			public int compare(Double x, Double y) {
				return y.compareTo(x);
			}
		});

		@Override
		protected void reduce(Text key, Iterable<DoubleWritable> values,
				Reducer<Text, DoubleWritable, Text, DoubleWritable>.Context context)
				throws IOException, InterruptedException {
			//reduce的操作对象是[key,<value1,value2...>]
			Double sumscore=0.0;
			int num=0;
			for(DoubleWritable value:values) {
				num++;
				sumscore=sumscore+value.get();
			}
			Double avg= sumscore/num;
			//得到的结果先不输出，到treepMap里面先排个序
			treeMap.put(avg, key.toString());
		}
                //输出
		protected void cleanup(Context context) throws IOException, InterruptedException {
			for(Double key:treeMap.keySet()) {
				context.write(new Text(treeMap.get(key)), new DoubleWritable(key));
			}
		}

     }
}

结果：

2016002,张小明,	91.0
2016004,王东,	83.33333333333333
2016001,王毅,	77.66666666666667
2016005,王笑笑,	76.0
2016003,李学彭,	72.0

问题三：

Course3.java

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class Course3 {

	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		Configuration conf=new Configuration();
		Job job=Job.getInstance(conf,"Course3");

		job.setJarByClass(Course3.class);
		job.setMapperClass(Course3Mapper.class);
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(IntWritable.class);

		job.setReducerClass(Course3Reduce.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);

		FileInputFormat.addInputPath(job, new Path("/Output1"));
		FileOutputFormat.setOutputPath(job, new Path("/Output3"));
		System.exit(job.waitForCompletion(true)?0:1);

	}

     public static class Course3Mapper extends Mapper<Object,Text,Text,IntWritable>{

		@Override
		protected void map(Object key, Text value, Mapper<Object, Text,Text, IntWritable>.Context context)
				throws IOException, InterruptedException {
			//分割
			String[] words=value.toString().split(",");
			int Score=Integer.parseInt(words[3]);
			//key=课程  value=某人某科成绩
			context.write(new Text(words[2]), new IntWritable(Score));

		}
     }

     public static class Course3Reduce extends Reducer<Text,IntWritable,Text,Text>{

		@Override
		protected void reduce(Text key, Iterable<IntWritable> values,
				Reducer<Text, IntWritable, Text, Text>.Context context) throws IOException, InterruptedException {

			int mmax=0;//最大值
			int mmin=101;//最小值
			double avg=0;//平均成绩
			int num=0;//每科人数
			for(IntWritable value:values) {
				num++;
				if(value.get()>mmax) mmax=value.get();
				if(value.get()<mmin) mmin=value.get();
				avg=avg+value.get();
			}
			avg=avg/num;
			String score=String.valueOf(mmax)+","+String.valueOf(mmin)+","+String.valueOf(avg);
			context.write(key,new Text(score));
		}
     }
}

结果：

大数据概论	91,56,76.8
操作系统	91,60,77.2
数据库	90,88,89.0