下载

自定义sink(mysql)

1、ide打开下载后的源码

2、代码如下:

/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/ /**
* User: 过往记忆
* Date: 14-9-4
* Time: 下午13:16
* bolg: https://www.iteblog.com
* 本文地址:https://www.iteblog.com/archives/1109
* 过往记忆博客,专注于hadoop、hive、spark、shark、flume的技术博客,大量的干货
* 过往记忆博客微信公共帐号:iteblog_hadoop
*/
package com.pf;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import org.apache.flume.*;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.List; public class MysqlSink extends AbstractSink implements Configurable { private Logger LOG = LoggerFactory.getLogger(MysqlSink.class);
private String hostname;
private String port;
private String databaseName;
private String tableName;
private String user;
private String password;
private PreparedStatement preparedStatement;
private Connection conn;
private int batchSize; public MysqlSink() {
LOG.info("MysqlSink start...");
} @Override
public void configure(Context context) {
hostname = context.getString("hostname");
Preconditions.checkNotNull(hostname, "hostname must be set!!");
port = context.getString("port");
Preconditions.checkNotNull(port, "port must be set!!");
databaseName = context.getString("databaseName");
Preconditions.checkNotNull(databaseName, "databaseName must be set!!");
tableName = context.getString("tableName");
Preconditions.checkNotNull(tableName, "tableName must be set!!");
user = context.getString("user");
Preconditions.checkNotNull(user, "user must be set!!");
password = context.getString("password");
Preconditions.checkNotNull(password, "password must be set!!");
batchSize = context.getInteger("batchSize", 100);
Preconditions.checkNotNull(batchSize > 0, "batchSize must be a positive number!!");
} @Override
public void start() {
super.start();
try {
//调用Class.forName()方法加载驱动程序
Class.forName("com.mysql.jdbc.Driver");
} catch (ClassNotFoundException e) {
e.printStackTrace();
} String url = "jdbc:mysql://" + hostname + ":" + port + "/" + databaseName;
//调用DriverManager对象的getConnection()方法,获得一个Connection对象 try {
conn = DriverManager.getConnection(url, user, password);
conn.setAutoCommit(false);
//创建一个Statement对象
preparedStatement = conn.prepareStatement("insert into " + tableName +
" (content) values (?)"); } catch (SQLException e) {
e.printStackTrace();
System.exit(1);
} } @Override
public void stop() {
super.stop();
if (preparedStatement != null) {
try {
preparedStatement.close();
} catch (SQLException e) {
e.printStackTrace();
}
} if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
} @Override
public Status process() throws EventDeliveryException {
Status result = Status.READY;
Channel channel = getChannel();
Transaction transaction = channel.getTransaction();
Event event;
String content; List<String> actions = Lists.newArrayList();
transaction.begin();
try {
for (int i = 0; i < batchSize; i++) {
event = channel.take();
if (event != null) {
content = new String(event.getBody());
actions.add(content);
} else {
result = Status.BACKOFF;
break;
}
} if (actions.size() > 0) {
preparedStatement.clearBatch();
for (String temp : actions) {
preparedStatement.setString(1, temp);
preparedStatement.addBatch();
}
preparedStatement.executeBatch(); conn.commit();
}
transaction.commit();
} catch (Throwable e) {
try {
transaction.rollback();
} catch (Exception e2) {
LOG.error("Exception in rollback. Rollback might not have been" +
"successful.", e2);
}
LOG.error("Failed to commit transaction." +
"Transaction rolled back.", e);
Throwables.propagate(e);
} finally {
transaction.close();
} return result;
}
}

3、pom文件

<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent>
<artifactId>flume-ng-sinks</artifactId>
<groupId>org.apache.flume</groupId>
<version>1.7.0</version>
</parent>
<artifactId>pf</artifactId>
<packaging>jar</packaging> <name>pf</name>
<url>http://maven.apache.org</url> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties> <dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
</dependency> <dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-configuration</artifactId>
</dependency> <dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.25</version>
</dependency> <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency> <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<scope>test</scope>
</dependency>
</dependencies> <!-- <build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build> -->
</project>

4、建库建表

create database ngmonitor;

create table mysqltest(
id int(11) NOT NULL AUTO_INCREMENT,
content varchar(50000) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8;

5、打包--> lib目录下(含mysql的jar)

6、参考conf

# example2.conf:

# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1 # Describe/configure the source
a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /home/wwj/flumeSpool
a1.sources.r1.fileHeader = false
a1.sources.r1.maxBackoff = 60000
#a1.sources.r1.ignorePattern = ^[\s]*\n # Describe the sink
a1.sinks.k1.type = com.pf.MysqlSink
a1.sinks.k1.hostname = localhost
a1.sinks.k1.port = 3306
a1.sinks.k1.databaseName = ngmonitor
a1.sinks.k1.tableName = mysqltest
a1.sinks.k1.user = root
a1.sinks.k1.password= 123456 # Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100 # Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

7、启动

bin/flume-ng agent --conf conf --conf-file conf/example2.conf --name a1 -Dflume.root.logger=INFO,console

8、成功

flume学习的更多相关文章

  1. Flume学习总结

    Flume学习总结 flume是一个用来采集数据的软件,它可以从数据源采集数据到一个集中存放的地方. 最常用flume的数据采集场景是对日志的采集,不过,lume也可以用来采集其他的各种各样的数据,因 ...

  2. flume学习(三):flume将log4j日志数据写入到hdfs(转)

    原文链接:flume学习(三):flume将log4j日志数据写入到hdfs 在第一篇文章中我们是将log4j的日志输出到了agent的日志文件当中.配置文件如下: tier1.sources=sou ...

  3. Flume学习应用:Java写日志数据到MongoDB

    概述 Windows平台:Java写日志到Flume,Flume最终把日志写到MongoDB. 系统环境 操作系统:win7 64 JDK:1.6.0_43 资源下载 Maven:3.3.3下载.安装 ...

  4. Flume学习 & Kafka & Storm 等 & Log4J 配置

    正在学习这篇文章: http://blog.csdn.net/ymh198816/article/details/51998085 和工作中接触的电商.订单.分析,可以结合起来. 开宗明义,这幅图片: ...

  5. flume学习安装

    近期项目组有需求点击流日志须要自己收集,学习了一下flume而且成功安装了.相关信息记录一下. 1)下载flume1.5版本号  wget http://www.apache.org/dyn/clos ...

  6. flume学习以及ganglia(若是要监控hive日志,hive存放在/tmp/hadoop/hive.log里,只要运行过hive就会有)

    python3.6hdfs的使用 https://blog.csdn.net/qq_29863961/article/details/80291654 https://pypi.org/  官网直接搜 ...

  7. Flume学习——BasicTransactionSemantics

    org.apache.flume.channel.BasicTransactionSemantics An implementation of basic Transaction semantics ...

  8. Flume学习——Flume中事务的定义

    首先要搞清楚的问题是:Flume中的事务用来干嘛? Flume中的事务用来保证消息的可靠传递. 当使用继承自BasicChannelSemantics的Channel时,Flume强制在操作Chann ...

  9. Flume学习——Flume的架构

    Flume有三个组件:Source.Channel 和 Sink.在源码中对应同名的三个接口. When a Flume source receives an event, it stores it ...

随机推荐

  1. Linux 之安装文件

    1.首先要检查 rpm -q gcc glibc glibc-common rrdtool rrdtool-devel expat expat-devel pcre pcre-devel dejavu ...

  2. Redis学习笔记八:独立功能之二进制位数组

    Redis 提供了 setbit.getbit.bitcount.bitop 四个命令用于处理二进制位数组. setbit 命令用于为位数组指定偏移量上的二进制位设置值,偏移量从 0 开始计数. ge ...

  3. Effective Java 读书笔记之四 泛型

    泛型的本质是参数化类型.只对编译器有效. 一.请不要在新代码中使用原生态类型 1.泛型类和接口统称为泛型,有一个对应的原生态类型. 2.原生类型的存在是为了移植兼容性. 3.无限制通配类型和原生态类型 ...

  4. CentOS6系升级Python2.7版本

    安装前准备 本实例以CentOS6.7为例 [root@E tools]# uname -r 2.6.32-431.23.3.el6.x86_64 [root@E tools]# uname -m x ...

  5. Java BigDecimal详解

    借用<Effactive Java>这本书中的话,float和double类型的主要设计目标是为了科学计算和工程计算.他们执行二进制浮点运算,这是为了在广域数值范围上提供 较为精确的快速近 ...

  6. opencv png和jpg的叠加

    char *bgFile = "C:/C_Project/HandTraining/Debug/res/bg.jpg"; FILE *file = fopen(bgFile, &q ...

  7. Android架构:用消息机制获取网络数据

    网络请求,不管是什么协议,是长连接还是短连接,总是一个异步的请求,过程包括:加请求参数->发起请求->接收响应->解析数据->获得业务数据. 最挫的做法是,业务代码包揽所有这些 ...

  8. admin 后台 bs模板

    http://bootsnipp.com/ https://almsaeedstudio.com/preview

  9. Junit 测试 Spring

    在测试类上加上@RunWith,和@ContextConfiguration @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration ...

  10. HDU 4942 Game on S♂play(线段树、模拟、扩栈)

    比赛的时候想到这题的大概做法,但由于卡别的水题...就赛后做了... 题意:给一个二叉树,每个结点有一个w[i],有3种操作,0 x表示左旋x,1 x表示右旋x,3 x表示询问x结点的价值,其中,价值 ...