【Kafka】实时看板案例

目录

• 项目需求
• 项目模型
• 实现步骤

---

项目需求

快速计算双十一当天的订单量和销售金额

项目模型

---

实现步骤

一、创建topic

bin/kafka-topics.sh --create --topic itcast_order --zookeeper node01:2181,node02:2181,node03:2181 --partitions 5 --replication-factor 2



二、创建maven项目并导入要依赖的jar包

<dependencies>
    <dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka-clients</artifactId>
        <version>0.10.0.0</version>
    </dependency>
    <dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>fastjson</artifactId>
        <version>1.2.41</version>
    </dependency>
    <dependency>
        <groupId>redis.clients</groupId>
        <artifactId>jedis</artifactId>
        <version>2.9.0</version>
    </dependency>

    <dependency>
        <groupId>log4j</groupId>
        <artifactId>log4j</artifactId>
        <version>1.2.17</version>
    </dependency>

</dependencies>

<build>
    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <configuration>
                <source>1.8</source>
                <target>1.8</target>
            </configuration>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>2.4.1</version>
            <executions>
                <execution>
                    <phase>package</phase>
                    <goals>
                        <goal>shade</goal>
                    </goals>
                    <configuration>
                        <transformers>
                            <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                <mainClass>cn.itcast.realboard.LogOperate</mainClass>
                            </transformer>
                        </transformers>
                    </configuration>
                </execution>
            </executions>
        </plugin>
         <plugin>
                 <artifactId> maven-assembly-plugin </artifactId>
                 <configuration>
                      <descriptorRefs>
                           <descriptorRef>jar-with-dependencies</descriptorRef>
                      </descriptorRefs>
                      <archive>
                           <manifest>
                                <mainClass>cn.itcast.realboard.LogOperate</mainClass>
                           </manifest>
                      </archive>
                 </configuration>
                 <executions>
                      <execution>
                           <id>make-assembly</id>
                           <phase>package</phase>
                           <goals>
                                <goal>single</goal>
                           </goals>
                      </execution>
                 </executions>
            </plugin>
    </plugins>
</build>

三、消费生产代码实现

• 1.创建订单实体类

  package cn.itcast.realboard;
  
  import com.alibaba.fastjson.JSONObject;
  
  import java.text.ParseException;
  import java.text.SimpleDateFormat;
  import java.util.Date;
  import java.util.Random;
  import java.util.UUID;
  
  public class PaymentInfo {
      private static final long serialVersionUID = -7958315778386204397L;
      private String orderId;//订单编号
      private Date createOrderTime;//订单创建时间
      private String paymentId;//支付编号
      private Date paymentTime;//支付时间
      private String productId;//商品编号
      private String productName;//商品名称
      private long productPrice;//商品价格
      private long promotionPrice;//促销价格
      private String shopId;//商铺编号
      private String shopName;//商铺名称
      private String shopMobile;//商品电话
      private long payPrice;//订单支付价格
      private int num;//订单数量
      /**
       * <Province>19</Province>
       * <City>1657</City>
       * <County>4076</County>
       */
      private String province; //省
      private String city; //市
      private String county;//县
      //102,144,114
      private String catagorys;
      public String getProvince() {
          return province;
      }
      public void setProvince(String province) {
          this.province = province;
      }
  
      public String getCity() {
          return city;
      }
  
      public void setCity(String city) {
          this.city = city;
      }
  
      public String getCounty() {
          return county;
      }
  
      public void setCounty(String county) {
          this.county = county;
      }
  
      public String getCatagorys() {
          return catagorys;
      }
  
      public void setCatagorys(String catagorys) {
          this.catagorys = catagorys;
      }
  
      public PaymentInfo() {
      }
  
      public PaymentInfo(String orderId, Date createOrderTime, String paymentId, Date paymentTime, String productId, String productName, long productPrice, long promotionPrice, String shopId, String shopName, String shopMobile, long payPrice, int num) {
          this.orderId = orderId;
          this.createOrderTime = createOrderTime;
          this.paymentId = paymentId;
          this.paymentTime = paymentTime;
          this.productId = productId;
          this.productName = productName;
          this.productPrice = productPrice;
          this.promotionPrice = promotionPrice;
          this.shopId = shopId;
          this.shopName = shopName;
          this.shopMobile = shopMobile;
          this.payPrice = payPrice;
          this.num = num;
      }
  
      public String getOrderId() {
          return orderId;
      }
  
      public void setOrderId(String orderId) {
          this.orderId = orderId;
      }
  
      public Date getCreateOrderTime() {
          return createOrderTime;
      }
  
      public void setCreateOrderTime(Date createOrderTime) {
          this.createOrderTime = createOrderTime;
      }
  
      public String getPaymentId() {
          return paymentId;
      }
  
      public void setPaymentId(String paymentId) {
          this.paymentId = paymentId;
      }
  
      public Date getPaymentTime() {
          return paymentTime;
      }
  
      public void setPaymentTime(Date paymentTime) {
          this.paymentTime = paymentTime;
      }
  
      public String getProductId() {
          return productId;
      }
  
      public void setProductId(String productId) {
          this.productId = productId;
      }
  
      public String getProductName() {
          return productName;
      }
  
      public void setProductName(String productName) {
          this.productName = productName;
      }
  
      public long getProductPrice() {
          return productPrice;
      }
  
      public void setProductPrice(long productPrice) {
          this.productPrice = productPrice;
      }
  
      public long getPromotionPrice() {
          return promotionPrice;
      }
  
      public void setPromotionPrice(long promotionPrice) {
          this.promotionPrice = promotionPrice;
      }
  
      public String getShopId() {
          return shopId;
      }
  
      public void setShopId(String shopId) {
          this.shopId = shopId;
      }
  
      public String getShopName() {
          return shopName;
      }
  
      public void setShopName(String shopName) {
          this.shopName = shopName;
      }
  
      public String getShopMobile() {
          return shopMobile;
      }
  
      public void setShopMobile(String shopMobile) {
          this.shopMobile = shopMobile;
      }
  
      public long getPayPrice() {
          return payPrice;
      }
  
      public void setPayPrice(long payPrice) {
          this.payPrice = payPrice;
      }
  
      public int getNum() {
          return num;
      }
  
      public void setNum(int num) {
          this.num = num;
      }
  
      @Override
      public String toString() {
          return "PaymentInfo{" +
                  "orderId='" + orderId + '\'' +
                  ", createOrderTime=" + createOrderTime +
                  ", paymentId='" + paymentId + '\'' +
                  ", paymentTime=" + paymentTime +
                  ", productId='" + productId + '\'' +
                  ", productName='" + productName + '\'' +
                  ", productPrice=" + productPrice +
                  ", promotionPrice=" + promotionPrice +
                  ", shopId='" + shopId + '\'' +
                  ", shopName='" + shopName + '\'' +
                  ", shopMobile='" + shopMobile + '\'' +
                  ", payPrice=" + payPrice +
                  ", num=" + num +
                  '}';
      }
  
      public String random() throws ParseException {
          this.orderId = UUID.randomUUID().toString().replaceAll("-", "");
          this.paymentId = UUID.randomUUID().toString().replaceAll("-", "");
          this.productPrice = new Random().nextInt(1000);
          this.promotionPrice = new Random().nextInt(500);
          this.payPrice = new Random().nextInt(480);
          this.shopId = new Random().nextInt(200000)+"";
  
          this.catagorys = new Random().nextInt(10000)+","+new Random().nextInt(10000)+","+new Random().nextInt(10000);
          this.province = new Random().nextInt(23)+"";
          this.city = new Random().nextInt(265)+"";
          this.county = new Random().nextInt(1489)+"";
  
          String date = "2015-11-11 12:22:12";
          SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
          try {
              this.createOrderTime = simpleDateFormat.parse(date);
          } catch (ParseException e) {
              e.printStackTrace();
          }
          JSONObject obj = new JSONObject();
          String jsonString = obj.toJSONString(this);
          return jsonString;
          //  return new Gson().toJson(this);
      }
  }
  

• 2.定义log4j.properties配置文件
  
  在/src/main/resources/目录下创建log4h.properties文件

  ### 设置###
  log4j.rootLogger = debug,stdout,D,E
  
  ### 输出信息到控制抬 ###
  log4j.appender.stdout = org.apache.log4j.ConsoleAppender
  log4j.appender.stdout.Target = System.out
  log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
  log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n
  
  ### 输出DEBUG 级别以上的日志到=E://logs/error.log ###
  log4j.appender.D = org.apache.log4j.DailyRollingFileAppender
  #log4j.appender.D.File = /Users/zhaozhuang/Desktop/logs/log.log
  log4j.appender.D.File = /export/servers/orderLogs/orderinfo.log
  log4j.appender.D.Append = true
  log4j.appender.D.Threshold = DEBUG
  log4j.appender.D.layout = org.apache.log4j.PatternLayout
  #log4j.appender.D.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss}  [ %t:%r ] - [ %p ]  %m%n
  log4j.appender.D.layout.ConversionPattern = %m%n
  
  ### 输出ERROR 级别以上的日志到=E://logs/error.log ###
  log4j.appender.E = org.apache.log4j.DailyRollingFileAppender
  #log4j.appender.E.File = /Users/zhaozhuang/Desktop/logs/error.log
  log4j.appender.E.File = /export/servers/orderLogs/ordererror.log
  log4j.appender.E.Append = true
  log4j.appender.E.Threshold = ERROR
  log4j.appender.E.layout = org.apache.log4j.PatternLayout
  #log4j.appender.E.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss}  [ %t:%r ] - [ %p ]  %m%n
  log4j.appender.E.layout.ConversionPattern =  %m%n
  

• 3.开发日志生产代码

  package cn.itcast.realboard;
  
  import org.apache.log4j.Logger;
  
  import java.text.ParseException;
  
  public class LogOperate {
      private static Logger printLogger = Logger.getLogger("printLogger");
      public static void main(String[] args) throws ParseException, InterruptedException {
          PaymentInfo paymentInfo = new PaymentInfo();
          while(true){
              String random = paymentInfo.random();
              System.out.println(random);
              printLogger.info(random);
              Thread.sleep(1000);
          }
      }
  }
  

• 4.将程序打包并上传服务器运行
  
  上传后执行以下命令
  
  java -jar day10_Project-1.0-SNAPSHOT-jar-with-dependencies.jar

• 5.开发flume配置文件，实现收集数据到kafka
  
  开发flume配置文件
  
  cd /export/servers/apache-flume-1.6.0-cdh5.14.0-bin/

vim file_kafka.conf

  #为我们的source channel  sink起名
  a1.sources = r1
  a1.channels = c1
  a1.sinks = k1
  #指定我们的source收集到的数据发送到哪个管道
  a1.sources.r1.channels = c1
  #指定我们的source数据收集策略
  a1.sources.r1.type = TAILDIR
  a1.sources.r1.positionFile = /var/log/flume/taildir_position.json
  a1.sources.r1.filegroups = f1
  a1.sources.r1.filegroups.f1 = /export/servers/orderLogs/orderinfo.log
  
  #指定我们的channel为memory,即表示所有的数据都装进memory当中
  a1.channels.c1.type = memory
  #指定我们的sink为kafka  sink，并指定我们的sink从哪个channel当中读取数据
  a1.sinks.k1.channel = c1
  a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
  a1.sinks.k1.kafka.topic = itcast_order
  a1.sinks.k1.kafka.bootstrap.servers = node01:9092,node02:9092,node03:9092
  a1.sinks.k1.kafka.flumeBatchSize = 20
  a1.sinks.k1.kafka.producer.acks = 1
  

  启动flume bin/flume-ng agent -c conf/ -f conf/file_kafka.conf -n a1

• 6.启动kafka，开始消费数据
  
  cd /export/servers/kafka_2.11-1.0.0/

bin/kafka-console-consumer.sh --from-beginning --bootstrap-server node01:9092,node02:9092,node03:9092 --topic itcast_order

四、消费消费代码实现

• 1.定义redis工具类

  package cn.itcast.realboard;
  
  import org.junit.Test;
  import redis.clients.jedis.Jedis;
  import redis.clients.jedis.JedisPool;
  import redis.clients.jedis.JedisPoolConfig;
  
  public class JedisUtils {
      private static JedisPool pool = null;
      /**
       * 获取jedis连接池
       **/
      public static JedisPool getPool(){
          if(pool == null){
              //创建jedis连接池配置
              JedisPoolConfig config = new JedisPoolConfig();
              //最大连接数
              config.setMaxTotal(20);
              //最大空闲连接
              config.setMaxIdle(5);
              //创建redis连接池
              pool = new JedisPool(config,"node01",6379,3000);
          }
          return pool;
      }
  
      /**
       * 获取jedis连接
       **/
      public static Jedis getConn(){
          return getPool().getResource();
      }
  
      @Test
      public void getJedisTest() throws Exception {
          Jedis jedis = getPool().getResource();
          jedis.incrBy("mine", 5);
          jedis.close();
      }
  }
  

• 2.开发Kafka消费代码

  package cn.itcast.realboard;
  
  import com.alibaba.fastjson.JSONObject;
  import org.apache.kafka.clients.consumer.ConsumerRecord;
  import org.apache.kafka.clients.consumer.ConsumerRecords;
  import org.apache.kafka.clients.consumer.KafkaConsumer;
  import org.apache.kafka.clients.consumer.OffsetAndMetadata;
  import org.apache.kafka.common.TopicPartition;
  import redis.clients.jedis.Jedis;
  import redis.clients.jedis.JedisPool;
  
  import java.util.*;
  
  public class MyKafkaConsumer {
      /**
       * 消费itcast_order里面的数据
       *
       * @param args
       */
      public static void main(String[] args) {
          Properties props = new Properties();
          props.put("bootstrap.servers", "node01:9092");
          props.put("group.id", "test");
          //以下两行代码 ---消费者自动提交offset值
          props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
          props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
          props.put("enable.auto.commit", "false");
          props.put("auto.commit.interval.ms", "1000");
          KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<String, String>(props);
          kafkaConsumer.subscribe(Arrays.asList("itcast_order"));
  
          while (true) {
              ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(3000);
              //获取所有分区
              Set<TopicPartition> partitions = consumerRecords.partitions();
              for (TopicPartition topicPartition : partitions) {
                  List<ConsumerRecord<String, String>> records = consumerRecords.records(topicPartition);
                  for (ConsumerRecord<String, String> record : records) {
                      //获取Jedis客户端
                      JedisPool jedisPool = JedisUtils.getPool();
                      Jedis jedis = jedisPool.getResource();
  
                      //获取json字符串
                      String value = record.value();
  
                      //把字符串转换成对象
                      PaymentInfo paymentInfo = JSONObject.parseObject(value, PaymentInfo.class);
  
                      //获取付款额度
                      long payPrice = paymentInfo.getPayPrice();
  
                      //redis中的Key一般是约定俗成的，实际工作中一般都有业务逻辑
  
                      //求取平台销售总额度
                      jedis.incrBy("itcast:order:total:price:date",payPrice);
                      //求取平台下单人数
                      jedis.incr("itcast:order:total:user:date");
                      //平台销售数量(简单认为一个订单一个商品)
                      jedis.incr("itcast:order:total:num:date");
  
                      //每个商品的销售额
                      jedis.incrBy("itcast:order:"+paymentInfo.getProductId()+"price:date",payPrice);
                      //每个商品的购买人数
                      jedis.incr("itcast:order:"+paymentInfo.getProductId()+":user:date");
                      //每个商品销售数量
                      jedis.incr("itcast:order:"+paymentInfo.getProductId()+":num:date");
  
                      //每个店铺的总销售额
                      jedis.incrBy("itcast:order:"+paymentInfo.getShopId()+":price:date",payPrice);
                      //每个店铺的购买人数
                      jedis.incr("itcast:order:"+paymentInfo.getShopId()+":user:date");
                      //每个店铺的销售数量
                      jedis.incr("itcast:order:"+paymentInfo.getShopId()+":num:date");
  
                      jedis.close();
  
                      //处理业务逻辑
                      /**
                       平台运维角度统计指标
                       平台总销售额度
                       redisRowKey设计  itcast:order:total:price:date
                       平台今天下单人数
                       redisRowKey设计  itcast:order:total:user:date
                       平台商品销售数量
                       redisRowKey设计  itcast:order:total:num:date
  
                       商品销售角度统计指标
                       每个商品的总销售额
                       Redis的rowKey设计itcast:order:productId:price:date
                       每个商品的购买人数
                       Redis的rowKey设计itcast:order:productId:user:date
                       每个商品的销售数量
                       Redis的rowKey设计itcast:order:productId:num:date
  
                       店铺销售角度统计指标
                       每个店铺的总销售额
                       Redis的rowKey设计itcast:order:shopId:price:date
                       每个店铺的购买人数
                       Redis的rowKey设计itcast:order:shopId:user:date
                       每个店铺的销售数量
                       Redis的rowKey设计itcast:order:shopId:num:date
                       */
  
                  }
                  long offset = records.get(records.size() - 1).offset();
                  Map<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataMap = Collections.singletonMap(topicPartition, new OffsetAndMetadata(offset));
                  kafkaConsumer.commitSync(topicPartitionOffsetAndMetadataMap);
              }
          }
      }
  }