Topology的构建
- public class BlackListBolt extends BaseRichBolt{
- private static Logger logger = Logger.getLogger(BlackListBolt.class);
- private OutputCollector collector_;
- private Map<String,List<String>> blacklistMap_ = new ConcurrentHashMap<String,List<String>>();
- //实现了从数据库中获取黑名单基础数据的表,加载至内存作为比阿娘blacklistMap_维护 nextTuple()在每个tuple到达时被调用,这里主要实现了车牌在黑名单中的比对。
- public void prepare(Map stormConf,TopologyContext context,OutputCollector collector)
- {
- collector_ = collector;
- Connection con = null;
- Statement jjhmd_statement = null;
- ResultSet jjhmd_resultSet = null;
- String jjhmd_queryString = "select ID,CPHID,SFSSBJ,SFCL from JJHMD";
- try{
- con = DBUtil.getDataSource().getConnection();
- jjhmd_statement = con.createStatement();
- jjhmd_resultSet = jjhmd_statement.executeQuery(jjhmd_queryString);
- while(jjhmd_resultSet.next()){
- String jjhmd_cphid = jjhmd_resultSet.getString("CPHID");
- String jjhmd_sfssbj = jjhmd_resultSet.getString("SFSSBJ");
- String jjhmd_SFCL = jjhmd_resultSet.getString("SFCL");
- String jjhmd_id = jjhmd_resultSet.getString("ID");
- List<String> temp_info = new ArrayList<String>();
- temp_info.add(jjhmd_sfssbj);
- temp_info.add(jjhmd_SFCL);
- temp_info.add(jjhmd_id);
- blacklistMap_.put(jjhmd_cphid,temp_info);
- }
- jjhmd_resultSet.close();
- jjhmd_statement.close();
- }catch(SQLException e){
- e.printStackTrace();
- }finally{
- if(con!=null){
- try{
- con.close();
- }catch(SQLException e){
- logger.warn("",e);
- }
- }
- }
- }
- public void execute(Tuple tuple){
- String no = tuple.getStringByField("no");
- String location = tuple.getStringByFiled("location");
- String tpid = tuple.getStringByField(tpidl:);
- try{
- if(blacklistMap_.containsKey(no)){
- List<String>temp_info = blacklistMap_.get(no);
- if(temp_info.get(1).equals("否")){
- String msg = convertToMsg(tuple);
- conllector_.emit(new Values(msg));
- }
- }
- }catch(Excetption e){
- logger.error(e.getMessage());
- }finally{
- }
- }
- ……
- }
- public class BlackListTopology{
- //topicSpout接收来自JMS消息中间件的主题数据,且不设置并行度(这是由topic在JMS协议中的语义决定的)
- public static final String TOPIC_SPOUT = "topic_spout";
- //以随机分组的方式接收来自JmsSpout的数据,并行度被设置为2.
- public static final String BLACKLIST_BOLT = "blacklist_bolt";
- //下面这两个均以随机分组的方式接收来自BlackListBolt的数据,分别向消息中间件和数据库写入计算的结果数据.
- public static final String TOPIC_BOLT = "topic_bolt";
- public static final String DB_BOLT = "db_bolt";
- public static void main(String[] args) throws Exception{
- ConfigUtil cu = ConfigUtil.getInstance();
- JmsProvider jmsTopicProvbider_source = new ActiveMQProvider("failover:(tcp://"+cu.getMessage_ip()+":"+cu.getMessage_port+")",cu.getMessage_sb_topic(),"","");
- //消息中间件的IP地址、端口和主题名称,都是在配置文件中维护的,此处通过ConfigUtil对象从配置文件中获取的。
- JmsSpout topicSpout = new JmsSpout();
- topicSpout.setJmsProvider(jmsTopicProvider_source);
- topicSpout.setJmsTupleProducer(new SB_Beijing_TupleProducer());
- topicSpout.setJmsAcknowledgeMode(Session.AUTO_ACKNOWLEDGE);
- topicSpout.setDistributed(false);
- JmsProvider jmsTopicProvider_target = new ActiveMQProvider("failover:(tcp://"+cu.getMessage_ip()+":"+cu.getMessage_port()+")",cu.getMessage_ijhmdbj_topic(),"","")
- JmsBolt topicBolt = new JmsBolt();
- topicBolt.setJmsProvider(jmsTopicProvider_target);
- topicBolt.setJmsMessageProducer(new JsonMessageProducer());
- topicBolt.setJmsAcknowledgeMode(Session.AUTO_ACKNOWLEDGE);
- TopologyBuilder builder = new ToplogyBuilder();
- builder.setSpout(TOPIC_SPOUT,topicSpout;
- builder.setBolt(BLACKLIST_BOLT,new BlackListBolt(),2).shuffleGrouping(TOPIC_SPOUT);
- builder.setBolt(TOPIC_BOLT,topicBolt,1).shuffleGrouping(BLACKLIST_BOLT);
- RegisterBlackCarBolt dbBolt = new RegisterBlackCarBolt();
- builder.setBolt(DB_BOLT,dbBolt,1).shuffleGrouping(BLACKLIST_BOLT);
- Config conf = new Config();
- conf.setNumWorkers(2)
- if(args.length >0){
- conf.setDebug(false);
- StormSubmitter.submitTopology(args[0],conf,builder.createTopology());
- }else{
- conf.setDebug(true);
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology("storm-traffic-blcaklist",conf,builder.createTopology());
- Utils.sleep(6000000);
- cluster.killTopology("storm-traffic-blacklist");
- cluster.shutdown();
- }
- }
- }
- topicBolt是类JmsBolt的对象,它以随机分组的方式,也接受来自BlackListBolt的数据,即黑名单检索的即时结果,然后向消息中间件写入计算的结果数据
- public class JmsBolt extends BaseRichBolt{
- private static Logger LOG = LoggerFactory.getLogger(JmsBolt.class);
- private Connection connection;
- ……
- public void prepare(Map stormConf,TopologyContext context,OutputCollector collector){
- if(this.jmsProvider == null || this.producer == null){
- throw new IllegalStateException("JMS Provider and MessageProducer not set.");
- }
- this.collector = collector;
- try{
- ConnectionFactory cf = this.jmsProvider.connectionFactory();
- Destination dest = this.jmsProvider.destination();
- this.connection = cf.createConnection();
- this.session = connection.createSeesion(this.jmsTransactional,this.jmsAcknowledgeMode);
- this.messageProducer = session.createProducer(dest);
- connection.start();
- }
- catch(Exception e){
- LOG.warn("Error creating JMS connection.",e);
- }
- }
- public void execute(Tuple input){
- try{
- Message msg = this.producer.toMessage(this.session,input);
- if(msg!=null){
- if(msg.getJMSDestination()!=null){
- this.messageProducer.sen(msg.getJMSDestination(),msg);
- }else{
- this.messageProducer.send(msg);
- }
- }
- if(this.autoAck){
- LOG.debug("ACKing tuple:"+input);
- this.collector.ack(intput);
- }
- }catch(JMSException e){
- LOG.warn("Failing tuple:" + input + "Exception:" + e);
- this.collector.fail(input);
- }
- }
- ……
- }
- JmsSpout类继承了BaseRichSpout类并实现了MessageListener接口。作为JMS的客户端,JmsSpout实现了MessageListener接口,这里分析一下该接口声明的方法onMessage().方法onMessage()在Jms消息中间件向它推送一个消息时这里的实现是将得到的消息放入缓存队列queue对象中.
- public class JmsSpout extends BaseRichSpout implements MessageListener{
- private static final Logger LOG = LoggerFactory.getLogger(JmsSpout.class);
- private LinkedBlockingQueue<Message>queue;
- private ConcurrentHashMap<String,Message>pendingMessages;
- ……
- public void onMessage(Message msg){
- try{
- LOG.debug("Queuing msg ["+msg.getJMSMessageID()+"]");
- }catch(JMSException e){
- }
- this.queue.offer(msg);
- }
- //从queue对象获取数据,组织为tuple结构后发送(emit);
- public void nextTuple(){
- Message msg = this.queue.poll();
- if(msg == null){
- Utils.sleep(50);
- }else{
- LOG.debug("sending tuple:"+msg);
- try{
- Values vals = this.tupleProducer.toTuple(msg);
- if(this.isDurableSubscription()||(msg.getJMSDeliveryMode()!=Session.AUTO_ACKNOWLEDGE)){
- LOG.debug("Requesting acks.");
- this.collector.emit(vals,msg.getJMSMessageID());
- this.pendingMessages.put(msg.getJMSMessageID(),msg);
- }else{
- this.collector.emit(vals);
- }catch(JMSException e){
- LOG.warn("Unable to convert JMS message:"+msg);
- }
- }
- }
- //在tuple需要被确认处理成功时调用,这里的实现是从中间结果队列pendingMessages移除相应数据项,并对这条消息调用JMS的方法acknowledge()进行确认.
- public void ack(Object msgId){
- Message msg = this.pendingMessage.remove(msgId);
- if(msg!=null){
- try{
- msg.acknowledge();
- LOG.debug("JMS Message Scked:"+msgId);
- }catch(JMSException e){
- LOG.warn("Error acknowldging JMS message:" + msgId,e);
- }
- }else{
- LOG.warn("Couldn't acknowledge unknown JMS message ID:"+msgId);
- }
- }
- //在tuple需要被确认处理失败时调用,这里的实现是从中间结果队列pendingMessages移除相应数据项,并设置存在失败的标志位.
- public void fail(Object msgId){
- LOG.warn("Message failed:" + msgId);
- this.pendingMessages.remove(msgId);
- synchronized(this.recoveryMutex);{
- this.hasFailures = true;
- }
- }
- }
- ……
- }
- //dbBolt是类RegisterBlackCarBolt的对象,它以随机分组的方式,接受来自BlackListBolt的数据,也即黑名单检索的即时结果,然后向数据库写入计算的结果数据。
- public class RegisterBlackCarBolt implements IBasicBolt{
- private static Logger log = Logger.getLogger(RegisterBlackCarBolt.class);
- private Connection con = null;
- private String tableName = "JJHMDBJ";
- private void prepare(Map stormConf,TopologyContext context){
- try{
- con = DBUtil.getDataSource().getConnection();
- }catch(SQLException el){
- el.printStackTrace();
- }
- }
- public void execute(Tuple input,BasicOutputCollector collector){
- String json = (String)input.getValue(0);
- String[] tupleStrs = json.split(",");
- try{
- String stmt = "insert into "+tableName+"("+TPID+","+JCDID+","+HMDID+","+CPHID+","+LRSJ+","+primaryKey+","+FQH+") values(?,?,?,?,?,?,?)";
- PreparedStatment prepstmt = con.prepareStatement(stmt);
- if(tupleStrs.length==5){
- prepstmt.setString(1,tupleStrs[0]);
- prepstmt.setString(2,tupleStrs[1]);
- prepstmt.setString(3,tupleStrs[2]);
- prepstmt.setString(4,tupleStrs[3]);
- prepstmt.setTimestamp(5,new Timestamp((TimeUtil.string2datetime(tupleStrs[4])).getTime()));
- prepstmt.setInt(6,1);
- prepstmt.setInt(7,getPartNO(tupleStrs[4]));
- }else{
- log.error("tupple attribte size error!");
- }
- int r = prepstmt.executeUpdate();
- log.info("insert"+r+" row");
- }catch(Exception e){
- e.printStackTrace();
- }
- }
- ……
- }
Topology的构建的更多相关文章
- Storm入门教程 第二章 构建Topology[转]
2.1 Storm基本概念 在运行一个Storm任务之前,需要了解一些概念: Topologies Streams Spouts Bolts Stream groupings Reliability ...
- Storm概念学习系列之Topology拓扑
不多说,直接上干货! Hadoop 上运行的是 MapReduce 作业,而在 Storm 上运行的是拓扑 Topology,这两者之间是非常不同的.一个关键的区别是:一个MapReduce 作业 ...
- apache Storm学习之二-基本概念介绍
2.1 Storm基本概念 在运行一个Storm任务之前,需要了解一些概念: Topologies Streams Spouts Bolts Stream groupings Reliability ...
- Storm 01之 Storm基本概念及第一个demo
2.1 Storm基本概念 在运行一个Storm任务之前,需要了解一些概念: Topologies :[tə'pɑ:lədʒɪ]拓扑结构 Streams Spouts:[spaʊt]喷出; 喷射; 滔 ...
- Flume+Kafka+Storm+Redis 大数据在线实时分析
1.实时处理框架 即从上面的架构中我们可以看出,其由下面的几部分构成: Flume集群 Kafka集群 Storm集群 从构建实时处理系统的角度出发,我们需要做的是,如何让数据在各个不同的集群系统之间 ...
- Storm构建分布式实时处理应用初探
最近利用闲暇时间,又重新研读了一下Storm.认真对比了一下Hadoop,前者更擅长的是,实时流式数据处理,后者更擅长的是基于HDFS,通过MapReduce方式的离线数据分析计算.对于Hadoop, ...
- 《storm实战-构建大数据实时计算读书笔记》
自己的思考: 1.接收任务到任务的分发和协调 nimbus.supervisor.zookeeper 2.高容错性 各个组件都是无状态的,状态 ...
- Storm 实战:构建大数据实时计算
Storm 实战:构建大数据实时计算(阿里巴巴集团技术丛书,大数据丛书.大型互联网公司大数据实时处理干货分享!来自淘宝一线技术团队的丰富实践,快速掌握Storm技术精髓!) 阿里巴巴集团数据平台事业部 ...
- Topology and Geometry in OpenCascade-Edge
Topology and Geometry in OpenCascade-Edge eryar@163.com 摘要Abstract:本文简要介绍了几何造型中的边界表示法(BRep),并结合程序说明O ...
随机推荐
- leetcode 151. Reverse Words in a String --------- java
Given an input string, reverse the string word by word. For example,Given s = "the sky is blue& ...
- leetcode 118 Pascal's Triangle ----- java
Given numRows, generate the first numRows of Pascal's triangle. For example, given numRows = 5,Retur ...
- mysql 新建用户
//=============================================================================== //代理商子账户订单服务器为了安全 ...
- C++的异常处理
一.什么是异常处理 一句话:异常处理就是处理程序中的错误. 二.为什么需要异常处理,以及异常处理的基本思想 C++之父Bjarne Stroustrup在<The C++ Programming ...
- myeclipse10安装egit和使用
一.下载egit插件并安装到eclipse 下载egit插件包,然后解压放到Eclipse的dropins文件夹内或者直接放到对应的文件夹下 二.安装成功(window->preferences ...
- dede数据库文件导入失败的可能原因是数据表前缀不同,这里的失败指的是mysql添加了数据,但后台不显示
利用dede提供的数据备份还原功能,还原数据,出现失败的可能原因是数据表前缀不同,改过来就可以了
- Android 初阶自定义 View 字符头像
自己很少做自定义 View ,只有最开始的时候跟着郭神写了一个小 Demo ,后来随着见识的越来越多,特别是在开源社区看到很多优秀的漂亮的控件,都是羡慕的要死,但是拉下来的代码还是看不明白,而且当时因 ...
- 学习tornado:安全
http://blog.csdn.net/siddontang/article/details/18053915
- MYSQL 获取表的列信息
SELECT COLUMN_NAME as '列名' ,DATA_TYPE as '字段类型' ,COLUMN_TYPE as '长度加类型' FROM information_schema.`COL ...
- ASP.NET MVC 中实现View与Controller分离
一.开篇题外话 我经常会在博客园逛来逛去,看过很多大牛们的Blog,我很少在这块技术天地活动,之前有发表过几篇日志,好像大部分是和电商有关,作为一个多年的开发人员,很少在这里分享,之前一直在CSDN上 ...