spark streaming 实现接收网络传输数据进行WordCount功能
package iie.udps.example.operator.spark; import scala.Tuple2; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.streaming.Duration;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.streaming.Time; import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.List; import com.google.common.io.Files; import org.apache.spark.api.java.JavaPairRDD; import com.google.common.base.Optional; /**
* To run this on your local machine, you need to first run a Netcat server
*
* `$ nc -lk 9999`
*
* and run the example as
*
* spark-submit --class iie.udps.example.operator.spark.JavaNetworkWordCount
* --master local /home/xdf/test2.jar localhost 9999 /user/test/checkpoint/
* /home/xdf/outputFile /home/xdf/totalOutputFile
*
* 此示例接收Netcat server产生的数据,进行WordCount操作,分别输出当前结果和历史结果到本地文件中
*/
public final class JavaNetworkWordCount { @SuppressWarnings("serial")
public static void main(String[] args) { if (args.length != 5) {
System.err.println("You arguments were " + Arrays.asList(args));
System.err
.println("Usage: JavaNetworkWordCount <hostname> <port> <checkpoint-directory>\n"
+ " <output-file> <total-output-file>. <hostname> and <port> describe the TCP server that Spark\n"
+ " Streaming would connect to receive data. <checkpoint-directory> directory to\n"
+ " HDFS-compatible file system which checkpoint data <output-file> file to which\n"
+ " the word counts will be appended\n"
+ " <total-output-file> file to which the total word counts will be appended\n"
+ "\n"
+ "In local mode, <master> should be 'local[n]' with n > 1\n"
+ "Both <checkpoint-directory> and <output-file> and <total-output-file> must be absolute paths");
System.exit(1);
} final String checkpointDirectory = args[2]; // 检查点目录
final String curOutputPath = args[3];// 输出当前WordCount结果的路径
final String totalOutputPath = args[4];// 输出全部累计WordCount结果的路径
System.out.println("Creating new context");
final File curOutputFile = new File(curOutputPath);
if (curOutputFile.exists()) {
curOutputFile.delete();
}
final File totalOutputFile = new File(totalOutputPath);
if (totalOutputFile.exists()) {
totalOutputFile.delete();
}
// Create a StreamingContext
SparkConf conf = new SparkConf().setAppName("NetworkWordCount");
final JavaStreamingContext jssc = new JavaStreamingContext(conf,
new Duration(1000)); jssc.checkpoint(checkpointDirectory); // Create a DStream that will connect to hostname:port, like
// localhost:9999
JavaReceiverInputDStream<String> lines = jssc.socketTextStream(args[0],
Integer.parseInt(args[1])); // Split each line into words
JavaDStream<String> words = lines
.flatMap(new FlatMapFunction<String, String>() {
@Override
public Iterable<String> call(String x) {
return Arrays.asList(x.split(" "));
}
}); // Count each word in each batch
JavaPairDStream<String, Integer> pairs = words
.mapToPair(new PairFunction<String, String, Integer>() {
@Override
public Tuple2<String, Integer> call(String s)
throws Exception {
return new Tuple2<String, Integer>(s, 1);
}
});
JavaPairDStream<String, Integer> runningCounts = pairs
.reduceByKey(new Function2<Integer, Integer, Integer>() {
@Override
public Integer call(Integer i1, Integer i2)
throws Exception {
return i1 + i2;
}
}); runningCounts
.foreachRDD(new Function2<JavaPairRDD<String, Integer>, Time, Void>() {
@Override
public Void call(JavaPairRDD<String, Integer> rdd, Time time)
throws IOException {
String counts = "Counts at time " + time + " "
+ rdd.collect();
System.out.println(counts);
System.out.println("Appending to "
+ curOutputFile.getAbsolutePath());
Files.append(counts + "\n", curOutputFile,
Charset.defaultCharset());
return null;
}
}); Function2<List<Integer>, Optional<Integer>, Optional<Integer>> updateFunction = new Function2<List<Integer>, Optional<Integer>, Optional<Integer>>() {
@Override
public Optional<Integer> call(List<Integer> values,
Optional<Integer> state) {
Integer newSum = state.or(0);
for (Integer i : values) {
newSum += i;
}
return Optional.of(newSum);
}
}; JavaPairDStream<String, Integer> TotalCounts = words.mapToPair(
new PairFunction<String, String, Integer>() {
@Override
public Tuple2<String, Integer> call(String s) {
return new Tuple2<String, Integer>(s, 1);
}
}).updateStateByKey(updateFunction); TotalCounts
.foreachRDD(new Function2<JavaPairRDD<String, Integer>, Time, Void>() {
@Override
public Void call(JavaPairRDD<String, Integer> rdd, Time time)
throws IOException {
String counts = "Counts at time " + time + " "
+ rdd.collect();
System.out.println(counts);
System.out.println("Appending to "
+ totalOutputFile.getAbsolutePath());
Files.append(counts + "\n", totalOutputFile,
Charset.defaultCharset());
return null;
}
}); jssc.start(); // Start the computation
jssc.awaitTermination(); // Wait for the computation to terminate
System.exit(0);
} }
spark streaming 实现接收网络传输数据进行WordCount功能的更多相关文章
- Spark Streaming 数据接收过程
SparkStreaming 源码分析 一节中从源码角度,描述了Streaming执行时代码的调用过程.下边就接收转化阶段过程再简单分析一下,为分析backpressure作准备. SparkStre ...
- Spark Streaming与kafka整合实践之WordCount
本次实践使用kafka console作为消息的生产者,Spark Streaming作为消息的消费者,具体实践代码如下 首先启动kafka server .\bin\windows\kafka-se ...
- Spark Streaming的接收KAFKA的数据
https://github.com/lw-lin/CoolplaySpark/blob/master/Spark%20Streaming%20%E6%BA%90%E7%A0%81%E8%A7%A3% ...
- Spark Streaming源码解读之流数据不断接收全生命周期彻底研究和思考
本期内容 : 数据接收架构设计模式 数据接收源码彻底研究 一.Spark Streaming数据接收设计模式 Spark Streaming接收数据也相似MVC架构: 1. Mode相当于Rece ...
- Spark Streaming源码解读之流数据不断接收和全生命周期彻底研究和思考
本节的主要内容: 一.数据接受架构和设计模式 二.接受数据的源码解读 Spark Streaming不断持续的接收数据,具有Receiver的Spark 应用程序的考虑. Receiver和Drive ...
- Spark入门实战系列--7.Spark Streaming(上)--实时流计算Spark Streaming原理介绍
[注]该系列文章以及使用到安装包/测试数据 可以在<倾情大奉送--Spark入门实战系列>获取 .Spark Streaming简介 1.1 概述 Spark Streaming 是Spa ...
- Spark Streaming简介及原理
简介: SparkStreaming是一套框架. SparkStreaming是Spark核心API的一个扩展,可以实现高吞吐量的,具备容错机制的实时流数据处理. 支持多种数据源获取数据: Spark ...
- .Spark Streaming(上)--实时流计算Spark Streaming原理介
Spark入门实战系列--7.Spark Streaming(上)--实时流计算Spark Streaming原理介绍 http://www.cnblogs.com/shishanyuan/p/474 ...
- spark streaming的理解和应用
1.Spark Streaming简介 官方网站解释:http://spark.apache.org/docs/latest/streaming-programming-guide.html 该博客转 ...
随机推荐
- 表单form的属性,单行文本框、密码框、单选多选按钮
基础表单结构: <body> <h1> <hr /> <form action="" name="myFrom" en ...
- 本地电脑localhost指向127.0.0.1的配置
windows系统电脑,我们如果想访问本机部署的项目,通常使用的是localhost来指向本机,但是有时候发现不行,我们不妨打开资源管理器,C:\Windows\System32\drivers\et ...
- LinearLayout练习
<?xml version="1.0" encoding="utf-8"?> <LinearLayout xmlns:android=&quo ...
- ubuntu下 GCC编译程序出现 undefined reference to `std::ios_base::Init::Init()'问题
网上的解释是:“ you need to add -lstdc++, or use 'g++' rather than 'gcc' as your driver program.”,也就是说如果想要使 ...
- NOIP2010解题报告
今天状态不错..1个小时AC了前3题,第四题第一次也拿到了80%的分数,后来换了算法才拿到全部分数.. 第一题: 小晨的电脑上安装了一个机器翻译软件,他经常用这个软件来翻译英语文章. 这个翻译软件的原 ...
- C# 子窗体点击按钮产生的新子窗体放在父窗体里
情景展示: 父窗体Form1,左边是按钮,右边是panel(放置子窗体) 父窗体点击按钮,在panel显示第一个子窗体AA, AA有个按钮,点击按钮,是第二个子窗体ZZ, 怎样将AA的子窗体ZZ也显示 ...
- WebBrowers & HtmlViewers collection
WebBrowers & HtmlViewers collection 浏览: 加入我的收藏 楼主: THtmlViewerhttps://github.com/BerndGabriel/Ht ...
- 解决 “fatal error C1083: ”无法打开包括文件
添加该项目的附加路径 . 1)右键查看该项目的属性 2)点击配置属性——〉 C/C++ ——〉 常规 ——〉 附加包含目录——〉将缺失文件所在目录添加进去
- 一张图告诉你为什么 Gmail 是最好的邮箱,以及大量私货
今天早上,我的同事详细介绍了 Gmail 相比其他邮箱的优势,比如强大的垃圾邮件过滤.简单的使用界面.强大的功能设置等等.但是对我来说,这些并不是我使用 Gmail 的最重要原因. 我第一个正式的邮箱 ...
- C# IList<T>转为DataTable
public class WebUtil { /// <summary> /// 转换IList<T>为DataTable/// </summary> /// &l ...