基于MapReduce的手机流量统计分析
1,代码
package mr; import java.io.IOException; import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; /**
* 使用ArrayWritable
*/
public class TrafficApp4 { public static void main(String[] args) throws Exception{
Configuration conf = new Configuration();
Job job = Job.getInstance(conf , TrafficApp4.class.getSimpleName());
job.setJarByClass(TrafficApp4.class); FileInputFormat.setInputPaths(job, args[]);
job.setMapperClass(TrafficMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongArrayWritable.class); job.setReducerClass(TrafficReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongArrayWritable.class);
FileOutputFormat.setOutputPath(job, new Path(args[])); job.waitForCompletion(true);
} public static class TrafficMapper extends Mapper<LongWritable, Text, Text, LongArrayWritable>{
@Override
protected void map(LongWritable key, Text value,
Mapper<LongWritable, Text, Text, LongArrayWritable>.Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] splited = line.split("\t");
String phonenumber = splited[]; String upPackNum = splited[];
String downPackNum = splited[];
String upPayLoad = splited[];
String downPayLoad = splited[]; Text k2 = new Text(phonenumber);
LongArrayWritable v2 = new LongArrayWritable(upPackNum, downPackNum, upPayLoad, downPayLoad);
context.write(k2, v2);
}
} public static class TrafficReducer extends Reducer<Text, LongArrayWritable, Text, LongArrayWritable>{
@Override
protected void reduce(Text k2, Iterable<LongArrayWritable> v2s,
Reducer<Text, LongArrayWritable, Text, LongArrayWritable>.Context context)
throws IOException, InterruptedException { long upPackNum = 0L;
long downPackNum = 0L;
long upPayLoad = 0L;
long downPayLoad = 0L;
for (LongArrayWritable v2 : v2s) {
Writable[] values = v2.get();
upPackNum += ((LongWritable)values[]).get();
downPackNum += ((LongWritable)values[]).get();
upPayLoad += ((LongWritable)values[]).get();
downPayLoad += ((LongWritable)values[]).get();
} LongArrayWritable v3 = new LongArrayWritable(upPackNum, downPackNum, upPayLoad, downPayLoad);
context.write(k2, v3);
}
} public static class LongArrayWritable extends ArrayWritable{
/**
* 在调用的时候,首先调用该方法,然后调用set(Writable[])
*/
public LongArrayWritable() {
super(LongWritable.class);
}
/**
* 直接调用该方法即可
* @param values
*/
public LongArrayWritable(LongWritable[] values) {
super(LongWritable.class, values);
}
/**
* 直接调用该方法即可
* @param upPackNum
* @param downPackNum
* @param upPayLoad
* @param downPayLoad
*/
public LongArrayWritable(Long upPackNum, Long downPackNum, Long upPayLoad, Long downPayLoad) {
super(LongWritable.class);
LongWritable[] values = new LongWritable[];
values[] = new LongWritable(upPackNum);
values[] = new LongWritable(downPackNum);
values[] = new LongWritable(upPayLoad);
values[] = new LongWritable(downPayLoad);
super.set(values);
}
/**
* 直接调用该方法即可
* @param upPackNum
* @param downPackNum
* @param upPayLoad
* @param downPayLoad
*/
public LongArrayWritable(String upPackNum, String downPackNum, String upPayLoad, String downPayLoad) {
super(LongWritable.class);
LongWritable[] values = new LongWritable[];
values[] = new LongWritable(Long.parseLong(upPackNum));
values[] = new LongWritable(Long.parseLong(downPackNum));
values[] = new LongWritable(Long.parseLong(upPayLoad));
values[] = new LongWritable(Long.parseLong(downPayLoad));
super.set(values);
} @Override
public String toString() {
String[] array = super.toStrings();
return StringUtils.join(array, "\t");
}
} }
2,ArrayWritable的API
org.apache.hadoop.io
Class ArrayWritable
java.lang.Object
org.apache.hadoop.io.ArrayWritable
- 已实现的接口:
- Writable
public class ArrayWritableextends Objectimplements Writable
A Writable for arrays containing instances of a class. The elements of this writable must all be instances of the same class. If this writable will be the input for a Reducer, you will need to create a subclass that sets the value to be of the proper type. For example: public class IntArrayWritable extends ArrayWritable { public IntArrayWritable() { super(IntWritable.class); } }
构造方法摘要 | |
---|---|
ArrayWritable(Class<? extends Writable> valueClass) |
|
ArrayWritable(Class<? extends Writable> valueClass, Writable[] values) |
|
ArrayWritable(String[] strings) |
方法摘要 | |
---|---|
Writable[] |
get() |
Class |
getValueClass() |
void |
readFields(DataInput in) Deserialize the fields of this object from in . |
void |
set(Writable[] values) |
Object |
toArray() |
String[] |
toStrings() |
void |
write(DataOutput out) Serialize the fields of this object to out . |
Methods inherited from class java.lang.Object |
---|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
构造方法详细信息 |
---|
ArrayWritable
public ArrayWritable(Class<? extends Writable> valueClass)
ArrayWritable
public ArrayWritable(Class<? extends Writable> valueClass,
Writable[] values)
ArrayWritable
public ArrayWritable(String[] strings)
方法详细信息 |
---|
getValueClass
public Class getValueClass()
toStrings
public String[] toStrings()
toArray
public Object toArray()
set
public void set(Writable[] values)
get
public Writable[] get()
readFields
public void readFields(DataInput in)
throws IOException
- Description copied from interface:
Writable
- Deserialize the fields of this object from
in
.For efficiency, implementations should attempt to re-use storage in the existing object where possible.
-
- Specified by:
readFields
in interfaceWritable
-
- Parameters:
in
-DataInput
to deseriablize this object from.- Throws:
IOException
write
public void write(DataOutput out)
throws IOException
- Description copied from interface:
Writable
- Serialize the fields of this object to
out
. -
- Specified by:
write
in interfaceWritable
-
- Parameters:
out
-DataOuput
to serialize this object into.- Throws:
IOException
基于MapReduce的手机流量统计分析的更多相关文章
- MapReduce的手机流量统计的案例
程序:(另外一个关于单词计数的总结:http://www.cnblogs.com/DreamDrive/p/5492572.html) import java.io.IOException; impo ...
- 基于winpcap的以太网流量分析器(java)
开发工具 IDE:eclipse -neon JDK:1.8 OS:Win10-64bit 主要功能 1.要求完成一个基于Winpcap的网络流量统计分析系统,具有易用.美观的界面. 2.完成局域网( ...
- 023_数量类型练习——Hadoop MapReduce手机流量统计
1) 分析业务需求:用户使用手机上网,存在流量的消耗.流量包括两部分:其一是上行流量(发送消息流量),其二是下行流量(接收消息的流量).每种流量在网络传输过程中,有两种形式说明:包的大小,流量的大小. ...
- 第2节 mapreduce深入学习:8、手机流量汇总求和
第2节 mapreduce深入学习:8.手机流量汇总求和 例子:MapReduce综合练习之上网流量统计. 数据格式参见资料夹 需求一:统计求和 统计每个手机号的上行流量总和,下行流量总和,上行总流量 ...
- MapReduce 经典案例手机流量排序的分析
在进行流量排序之前,先要明白排序是发生在map阶段,排序之后(排序结束后map阶段才会显示100%完成)才会到reduce阶段(事实上reduce也会排序),.此外排序之前要已经完成了手机流量的统计工 ...
- 基于mapreduce的大规模连通图寻找算法
基于mapreduce的大规模连通图寻找算法 当我们想要知道哪些账号是一个人的时候往往可以通过业务得到两个账号之间有联系,但是这种联系如何传播呢? 问题 已知每个账号之间的联系 如: A B B C ...
- 字节数转换为b,kb,mb,gb的方法 通用的手机流量计算方法
//通用的手机流量计算方法 private String byteToMB(long size){ long kb = 1024; long mb = kb*1024; long gb = mb*10 ...
- MapReduce教程(一)基于MapReduce框架开发<转>
1 MapReduce编程 1.1 MapReduce简介 MapReduce是一种编程模型,用于大规模数据集(大于1TB)的并行运算,用于解决海量数据的计算问题. MapReduce分成了两个部分: ...
- 基于MapReduce的贝叶斯网络算法研究参考文献
原文链接(系列):http://blog.csdn.net/XuanZuoNuo/article/details/10472219 论文: 加速贝叶斯网络:Accelerating Bayesian ...
随机推荐
- P1107 最大整数
P1107 最大整数 题目描述 设有n个正整数 (n<=20), 将它们连接成一排, 组成一个最大的多位整数. 例如: n=3时, 3个整数13, 312, 343连接成的最大整数为: 3433 ...
- javascript代码规范 [转]
原文:http://www.css88.com/archives/5366 全局命名空间污染与 IIFE 总是将代码包裹成一个 IIFE(Immediately-Invoked Function Ex ...
- SQL On Streaming
此文已由作者岳猛授权网易云社区发布. 欢迎访问网易云社区,了解更多网易技术产品运营经验. 实时计算的一个方向 实时计算未来会成为一个趋势,基本上所有的离线计算任务都能通过实时计算来完成,对于实时计算来 ...
- jmeter3.0生成html格式的dashboard性能测试结果
jmeter3.0以上支持生成dashboard的html报告,官网介绍:https://jmeter.apache.org/usermanual/generating-dashboard.html ...
- C#调用C++编写的dll
界面还是C#写的方便点,主要是有一个可视化的编辑器,不想画太多的时间在界面上.但是自己又对C++了解的多一些,所以在需要一个良好的界面的情况下,使用C++来写代码逻辑,将其编译成一个dll,然后用C# ...
- Ubuntu-C++环境设置
在学习C++,顺便想熟悉一下Linux下开发 所以就开始搭建Linux环境 第一步就是下载虚拟机和Ubuntu 虚拟机 vm 12.1 Ubuntu 16 VM安装省略 Ubuntu安装省略 下面是我 ...
- Jmeter从文件中读取参数值
1. 通过函数助手,从本地文件中取值选项->函数助手对话框->选择__CSVRead函数->调用参数其中,函数助手对话框中,第一栏填写本地文件所在地址,第二栏写需要入参的值,有点类似 ...
- 基于规则的中文分词 - NLP中文篇
之前在其他博客文章有提到如何对英文进行分词,也说后续会增加解释我们中文是如何分词的,我们都知道英文或者其他国家或者地区一些语言文字是词与词之间有空格(分隔符),这样子分词处理起来其实是要相对容易很多, ...
- HDU 4467 Graph(图论+暴力)(2012 Asia Chengdu Regional Contest)
Description P. T. Tigris is a student currently studying graph theory. One day, when he was studying ...
- ExtJS新手学习中常见问题
1.常常出现运行之后不出现应该出现的效果. 这种情况一般是引用ExtJS路径不正确,要确保路径正确. 示例: <!DOCTYPE html> <html lang="en& ...