Storm工程创建
1.创建maven项目:
pom.xml:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion></modelVersion> <groupId>storm.book</groupId> <artifactId>Getting-Started</artifactId> <version>-SNAPSHOT</version> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <version></version> <configuration> <source>1.6</source> <target>1.6</target> <compilerVersion>1.6</compilerVersion> </configuration> </plugin> </plugins> </build> <repositories> <!-- Repository where we can found the storm dependencies --> <repository> <id>clojars.org</id> <url>http://clojars.org/repo</url> </repository> </repositories> <dependencies> <!-- Storm Dependency --> <dependency> <groupId>storm</groupId> <artifactId>storm</artifactId> <version></version> </dependency> </dependencies> </project>
相关类:
WordReader:
package spouts; import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.FileReader; import java.util.Map; import backtype.storm.spout.SpoutOutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseRichSpout; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Values; public class WordReader extends BaseRichSpout { private SpoutOutputCollector collector; private FileReader fileReader; private boolean completed = false; public void ack(Object msgId) { System.out.println("OK:"+msgId); } public void close() {} public void fail(Object msgId) { System.out.println("FAIL:"+msgId); } /** * The only thing that the methods will do It is emit each * file line */ public void nextTuple() { /** * The nextuple it is called forever, so if we have been readed the file * we will wait and then return */ if(completed){ try { Thread.sleep(); } catch (InterruptedException e) { //Do nothing } return; } String str; //Open the reader BufferedReader reader = new BufferedReader(fileReader); try{ //Read all lines while((str = reader.readLine()) != null){ /** * By each line emmit a new value with the line as a their */ this.collector.emit(new Values(str),str); } }catch(Exception e){ throw new RuntimeException("Error reading tuple",e); }finally{ completed = true; } } /** * We will create the file and get the collector object */ public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { try { this.fileReader = new FileReader(conf.get("wordsFile").toString()); } catch (FileNotFoundException e) { throw new RuntimeException("Error reading file ["+conf.get("wordFile")+"]"); } this.collector = collector; } /** * Declare the output field "word" */ public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("line")); } }
WordCounter:
package bolts; import java.util.HashMap; import java.util.Map; import backtype.storm.task.TopologyContext; import backtype.storm.topology.BasicOutputCollector; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.tuple.Tuple; public class WordCounter extends BaseBasicBolt { Integer id; String name; Map<String, Integer> counters; /** * At the end of the spout (when the cluster is shutdown * We will show the word counters */ @Override public void cleanup() { System.out.println("-- Word Counter ["+name+"-"+id+"] --"); for(Map.Entry<String, Integer> entry : counters.entrySet()){ System.out.println(entry.getKey()+": "+entry.getValue()); } } /** * On create */ @Override public void prepare(Map stormConf, TopologyContext context) { this.counters = new HashMap<String, Integer>(); this.name = context.getThisComponentId(); this.id = context.getThisTaskId(); } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) {} @Override public void execute(Tuple input, BasicOutputCollector collector) { String str = input.getString(); /** * If the word dosn't exist in the map we will create * this, if not We will add 1 */ if(!counters.containsKey(str)){ counters.put(str, ); }else{ Integer c = counters.; counters.put(str, c); } } }
WordNormalizer:
package bolts; import backtype.storm.topology.BasicOutputCollector; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; public class WordNormalizer extends BaseBasicBolt { public void cleanup() {} /** * The bolt will receive the line from the * words file and process it to Normalize this line * * The normalize will be put the words in lower case * and split the line to get all words in this */ public void execute(Tuple input, BasicOutputCollector collector) { String sentence = input.getString(); String[] words = sentence.split(" "); for(String word : words){ word = word.trim(); if(!word.isEmpty()){ word = word.toLowerCase(); collector.emit(new Values(word)); } } } /** * The bolt will only emit the field "word" */ public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("word")); } }
TopologyMain:
import spouts.WordReader; import backtype.storm.Config; import backtype.storm.LocalCluster; import backtype.storm.topology.TopologyBuilder; import backtype.storm.tuple.Fields; import bolts.WordCounter; import bolts.WordNormalizer; public class TopologyMain { public static void main(String[] args) throws InterruptedException { //Topology definition TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("word-reader",new WordReader()); builder.setBolt("word-normalizer", new WordNormalizer()) .shuffleGrouping("word-reader"); builder.setBolt() .fieldsGrouping("word-normalizer", new Fields("word")); //Configuration Config conf = new Config(); conf.put(]); conf.setDebug(false); //Topology run conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, ); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("Getting-Started-Toplogie", conf, builder.createTopology()); Thread.sleep(); cluster.shutdown(); } }
工程架构
运行:
结果
强大的流处理!
github:https://github.com/super-d2/strom_demo
Storm工程创建的更多相关文章
- Android开发之基于AndroidStudio环境搭建和工程创建
断断续续的学习安卓也有一段时间了.因为之前是搞iOS开发的, 之前有关iOS的博客请看<我的iOS开发系列博文>.<我的Objective-C系列文章>和<窥探Swift ...
- 161130、Dubbo+SpringMVC工程创建详解
Dubbo出现的目的是为了应对现在高并发,高数据量请求的问题.目前的垂直应用架构已经无法满足现在大数据的冲击,SOA就应运而生,而Dubbo在国内使用的还是比较多,稳定性也比较不错. 架构 节点角色说 ...
- VC++2008 用空工程创建 DLL
VC++2008 用空工程创建 DLL 一.创建 DLL 工程项目: 1)点击菜单[File] -> [New] -> [Project...] 弹出 “New Project” 对话框: ...
- Django初探——工程创建以及models数据库相关配置
Python的WEB框架有Django.Tornado.Flask 等多种,Django相较与其他WEB框架其优势为:大而全,框架本身集成了ORM.模型绑定.模板引擎.缓存.Session等诸多功能. ...
- N76E003 工程创建教程
一.准备工作: 1.下载编译工具keil c51 2.下载N76E003提供的板级支持包(BSP),可到nuvoton上下载 二.开发环境搭建 1.安装keil c51,然后和谐...不能随便发链 ...
- 第1天:Django框架简介与工程创建
Django简介 Django特点 环境搭建 创建工程 启动服务 使用pycharm打开工程 创建子应用 Django简介 Django,是用Python语言写的开源web开发框架,并遵循MVC设计. ...
- java web 工程创建及servlet简单使用
1.java web工程创建 (1)File--->new--->project (2)选择java enterprise,按照下图操作 (3)点击next后,会进入如下界面,修改工程名后 ...
- Django工程创建
方法一: 1.win+r进入cmd命令窗口: 2.找到Django的安装地址: 3.cmd窗口中利用cd 进入相应的文件夹,再输入命令如下: django-admin.exe startproject ...
- Django—工程创建以及models数据库易错点
Python的WEB框架有Django.Tornado.Flask 等多种,Django相较与其他WEB框架其优势为:大而全,框架本身集成了ORM.模型绑定.模板引擎.缓存.Session等诸多功能. ...
随机推荐
- iOS系统navigationBar背景色,文字颜色处理
- (void)setRightBarButtonItem { // Create done Button UIBarButtonItem *doneButton = [[UIBarButtonIte ...
- linux文件系统模拟
#include "stdio.h" #include <stdlib.h> //#include <conio.h> #include <strin ...
- 繁华模拟赛 Evensgn玩序列
#include<iostream> #include<cstdio> #include<string> #include<cstring> #incl ...
- codeforces 258div2 A Game With Sticks(DP)
题目链接:http://codeforces.com/contest/451/problem/A 解题报告:有n跟红色的棍子横着放,m根蓝色的棍子竖着放,它们形成n*m个交点,两个人轮流在里面选择交点 ...
- unity缓存和浏览器缓存
原地址:http://www.cnblogs.com/hisiqi/p/3203553.html <蒸汽之城>游戏中,为什么会黑屏?或者无法正常进入游戏? 我们在进行多项测试中发现少数用户 ...
- [codeforces 241]C. Mirror Box
[codeforces 241]C. Mirror Box 试题描述 Mirror Box is a name of a popular game in the Iranian National Am ...
- php email邮箱正则验证
国际域名格式如下: 域名由各国文字的特定字符集.英文字母.数字及“-”(即连字符或减号)任意组合而成, 但开头及结尾均不能含有“-”,“-”不能连续出现 . 域名中字母不分大小写.域名最长可达60个字 ...
- 【Spring】Spring系列6之Spring整合Hibernate
6.Spring整合Hibernate 6.1.准备工作 6.2.示例 com.xcloud.entities.book com.xcloud.dao.book com.xcloud.service. ...
- C# 代理/委托 Delegate
本文转载自努力,努力,努力 1. 委托的定义:委托是函数的封装,它代表一"类"函数.他们都符合一定的签名:拥有相同的参数列表,返回值类型.同时,委托也可以看成是对函数的抽象,是函数 ...
- Python多线程(1)——介绍
Python对多线程提供了很好的支持,Python中多线程相关的模块包括:thread,threading,Queue.可以方便地支持创建线程.互斥锁.信号量.同步等特性. 1. thread:多线程 ...