【Storm】storm安装、配置、使用以及Storm单词计数程序的实例分析
前言:阅读笔记
storm jar all-my-code.jar backtype.storm.MyTopology arg1 arg2
storm jar负责连接nimbus而且上传jar。
spouts和bolts是执行应用逻辑要实现的接口。
watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvc2ltb25jaGk=/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/Center" alt="">
watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvc2ltb25jaGk=/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/Center" alt="">
下载安装
http://storm.apache.org/downloads.html
1、依赖安装
2、zk集群
http://blog.csdn.net/simonchi/article/details/43019401
3、zeromq&jzmq
4、python
配置执行
storm.yaml
watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQvc2ltb25jaGk=/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/Center" alt="">
单词计数程序
Spout
package com.cmcc.chiwei.storm; import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.util.Map;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values; public class WordReader extends BaseRichSpout { private static final long serialVersionUID = 1L;
private SpoutOutputCollector collector;
private FileReader fileReader;
private boolean completed = false;
public boolean isDistributed() {
return false;
}
public void ack(Object msgId) {
System.out.println("OK:"+msgId);
}
public void close() {}
public void fail(Object msgId) {
System.out.println("FAIL:"+msgId);
} /**
* The only thing that the methods will do It is emit each
* file line
*/
public void nextTuple() {
if(completed){
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
//Do nothing
}
return;
}
String str;
//Open the reader
BufferedReader reader = new BufferedReader(fileReader);
try{
//Read all lines
while((str = reader.readLine()) != null){
/**
* By each line emmit a new value with the line as a their
*/
this.collector.emit(new Values(str),str);
}
}catch(Exception e){
throw new RuntimeException("Error reading tuple",e);
}finally{
completed = true;
}
} /**
* We will create the file and get the collector object
*/
public void open(Map conf, TopologyContext context,
SpoutOutputCollector collector) {
try {
this.fileReader = new FileReader(conf.get("wordsFile").toString());
} catch (FileNotFoundException e) {
throw new RuntimeException("Error reading file ["+conf.get("wordFile")+"]");
}
this.collector = collector;
} /**
* Declare the output field "word"
*/
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("line"));
}
}
Bolt1
package com.cmcc.chiwei.storm; import backtype.storm.topology.BasicOutputCollector;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseBasicBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values; public class WordNormalizer extends BaseBasicBolt { private static final long serialVersionUID = 1L; public void cleanup() {} /**
* The bolt will receive the line from the
* words file and process it to Normalize this line
*
* The normalize will be put the words in lower case
* and split the line to get all words in this
*/
public void execute(Tuple input, BasicOutputCollector collector) {
String sentence = input.getString(0);
String[] words = sentence.split(" ");
for(String word : words){
word = word.trim();
if(!word.isEmpty()){
word = word.toLowerCase();
collector.emit(new Values(word));
}
}
} /**
* The bolt will only emit the field "word"
*/
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("word"));
}
}
Bolt2
package com.cmcc.chiwei.storm; import java.util.HashMap;
import java.util.Map; import backtype.storm.task.TopologyContext;
import backtype.storm.topology.BasicOutputCollector;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseBasicBolt;
import backtype.storm.tuple.Tuple; public class WordCounter extends BaseBasicBolt { private static final long serialVersionUID = 1L;
Integer id;
String name;
Map<String, Integer> counters; /**
* At the end of the spout (when the cluster is shutdown
* We will show the word counters
*/
@Override
public void cleanup() {
System.out.println("-- Word Counter ["+name+"-"+id+"] --");
for(Map.Entry<String, Integer> entry : counters.entrySet()){
System.out.println(entry.getKey()+": "+entry.getValue());
}
} /**
* On create
*/
@Override
public void prepare(Map stormConf, TopologyContext context) {
this.counters = new HashMap<String, Integer>();
this.name = context.getThisComponentId();
this.id = context.getThisTaskId();
} public void declareOutputFields(OutputFieldsDeclarer declarer) {} public void execute(Tuple input, BasicOutputCollector collector) {
String str = input.getString(0);
if(!counters.containsKey(str)){
counters.put(str, 1);
}else{
Integer c = counters.get(str) + 1;
counters.put(str, c);
}
}
}
Topology
package com.cmcc.chiwei.storm; import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.tuple.Fields; public class TopologyMain {
public static void main(String[] args) throws InterruptedException { //Topology创建拓扑,安排storm各个节点以及它们交换数据的方式
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("word-reader",new WordReader());
builder.setBolt("word-normalizer", new WordNormalizer())
.shuffleGrouping("word-reader");
builder.setBolt("word-counter", new WordCounter(),1)
.fieldsGrouping("word-normalizer", new Fields("word")); //Configuration
Config conf = new Config();
conf.put("wordsFile", args[0]);
conf.setDebug(false);
//Topology run
conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("Getting-Started-Topology", conf, builder.createTopology());
Thread.sleep(2000);
cluster.shutdown();
}
}
words.txt
hello
world
storm flume hadoop hdfs
what's wrong flume ?
what's up hdfs ?
Hi,storm,what are you doing ?
执行结果
OK:hello
OK:world
OK:storm flume hadoop hdfs
OK:what's wrong flume ? OK:what's up hdfs ?
OK:Hi,storm,what are you doing ?
-- Word Counter [word-counter-2] --
what's: 2
flume: 2
hdfs: 2
you: 1
storm: 1
up: 1
hello: 1
hadoop: 1
hi,storm,what: 1
are: 1
doing: 1
wrong: 1
? : 3
world: 1
分析内容:
Spout
open --> nextTuple
Bolt1
declareOutputFields --> execute
Bolt2
prepare --> execute --> cleanup
更具体的内容,将在兴许慢慢解说,我也在研究中。
。。
。。
望各位不吝不吝赐教!。
【Storm】storm安装、配置、使用以及Storm单词计数程序的实例分析的更多相关文章
- Hadoop分布环境搭建步骤,及自带MapReduce单词计数程序实现
Hadoop分布环境搭建步骤: 1.软硬件环境 CentOS 7.2 64 位 JDK- 1.8 Hadoo p- 2.7.4 2.安装SSH sudo yum install openssh-cli ...
- 第一章 flex单词计数程序
学习Flex&Bison目标, 读懂SQLite中SQL解析部分代码 Flex&Bison简介Flex做词法分析Bison做语法分析 第一个Flex程序, wc.fl, 单词计数程序 ...
- ubuntu14.04LTS 下storm单机版安装配置
1.下载storm 的安装文件 http://www.apache.org/dyn/closer.cgi/incubator/storm/apache-storm-0.9.2-incubating/a ...
- storm的安装配置
一.安装Zookeeper 1.设置.profile文件: export ZOOKEEPER_HOME=/home/hadoop/streamdata/zookeeper-3.4.5-cdh4.5.0 ...
- 三:Storm设计一个Topology用来统计单词的TopN的实例
Storm的单词统计设计 一:Storm的wordCount和Hadoop的wordCount实例对比
- storm单机版安装配置
1,install zeromq 期间可能出现:configure: error: cannot link with -luuid, install uuid-dev. 因此可以先安装 sudo ap ...
- asp.Net Core免费开源分布式异常日志收集框架Exceptionless安装配置以及简单使用图文教程
最近在学习张善友老师的NanoFabric 框架的时了解到Exceptionless : https://exceptionless.com/ !因此学习了一下这个开源框架!下面对Exceptionl ...
- C#实现多级子目录Zip压缩解压实例 NET4.6下的UTC时间转换 [译]ASP.NET Core Web API 中使用Oracle数据库和Dapper看这篇就够了 asp.Net Core免费开源分布式异常日志收集框架Exceptionless安装配置以及简单使用图文教程 asp.net core异步进行新增操作并且需要判断某些字段是否重复的三种解决方案 .NET Core开发日志
C#实现多级子目录Zip压缩解压实例 参考 https://blog.csdn.net/lki_suidongdong/article/details/20942977 重点: 实现多级子目录的压缩, ...
- 【转】asp.Net Core免费开源分布式异常日志收集框架Exceptionless安装配置以及简单使用图文教程
最近在学习张善友老师的NanoFabric 框架的时了解到Exceptionless : https://exceptionless.com/ !因此学习了一下这个开源框架!下面对Exceptionl ...
随机推荐
- Beauty Contest(凸包求最远点)
http://poj.org/problem?id=2187 题意:求凸包上最远点距离的平方 思路:开始用旋转卡壳求的最远点,WA,改了好久..后来又改成枚举凸包上的点..AC了.. #include ...
- [Apple开发者帐户帮助]五、管理标识符(2)启用应用服务
您可以在证书,标识符和配置文件中查看和启用App ID的服务.包含已修改的App ID的供应配置文件将变为无效.您需要重新生成使用该App ID的配置文件. 注意:要为应用程序完全配置服务,请在Xco ...
- [Apple开发者帐户帮助]二、管理你的团队(3)删除团队成员
如果您已加入Apple开发者计划,您将在App Store Connect中管理团队成员.有关详细信息,请转到App Store Connect帮助中的添加和编辑用户. 如果您已加入Apple Dev ...
- 【专题系列】单调队列优化DP
Tip:还有很多更有深度的题目,这里不再给出,只给了几道基本的题目(本来想继续更的,但是现在做的题目不是这一块内容,以后有空可能会继续补上) 单调队列——看起来就是很高级的玩意儿,显然是个队列,而且其 ...
- [跨域]js设置document.domain实现跨域
document.domain用来得到当前网页的域名.比如在地址栏里输入: 代码如下: javascript:alert(document.domain); //www.jb51.net 我们也可以给 ...
- 推荐几个好用的windows软件
好久没上了,= = 之前用了一段MAC,感觉好用的地方就是 spotlight和触摸板,哈哈 然后在用回windows之后就找了一下,我之前有用Launchy这个软件,但是他是用来启动程序的,当然你也 ...
- PythonOpenCV:MLP用于最近邻搜索
一:简单C++版本的链接: http://blog.csdn.net/kaka20080622/article/details/9039749 OpenCV的ml模块实现了人工神经网络(Artific ...
- java中为什么不允许类多重继承,却允许接口多重继承
首先看下面这一段代码:(底下有热心网友更正,jdk1.8之后情况确实有点变化,等改天有空继续更) interface a{ void b();}interface a1 extends a{ void ...
- BZOJ 1426: 收集邮票 数学期望 + DP
Description 有n种不同的邮票,皮皮想收集所有种类的邮票.唯一的收集方法是到同学凡凡那里购买,每次只能买一张,并且 买到的邮票究竟是n种邮票中的哪一种是等概率的,概率均为1/n.但是由于凡凡 ...
- 【剑指Offer】5、用两个栈实现队列
题目描述: 用两个栈来实现一个队列,完成队列的Push和Pop操作. 队列中的元素为int类型. 解题思路: 本题的基本意图是:用两个后入先出的栈来实现先入先出的队列.对于这个问题,我 ...