flume学习
自定义sink(mysql)
1、ide打开下载后的源码
2、代码如下:
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/ /**
* User: 过往记忆
* Date: 14-9-4
* Time: 下午13:16
* bolg: https://www.iteblog.com
* 本文地址:https://www.iteblog.com/archives/1109
* 过往记忆博客,专注于hadoop、hive、spark、shark、flume的技术博客,大量的干货
* 过往记忆博客微信公共帐号:iteblog_hadoop
*/
package com.pf;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import org.apache.flume.*;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.List; public class MysqlSink extends AbstractSink implements Configurable { private Logger LOG = LoggerFactory.getLogger(MysqlSink.class);
private String hostname;
private String port;
private String databaseName;
private String tableName;
private String user;
private String password;
private PreparedStatement preparedStatement;
private Connection conn;
private int batchSize; public MysqlSink() {
LOG.info("MysqlSink start...");
} @Override
public void configure(Context context) {
hostname = context.getString("hostname");
Preconditions.checkNotNull(hostname, "hostname must be set!!");
port = context.getString("port");
Preconditions.checkNotNull(port, "port must be set!!");
databaseName = context.getString("databaseName");
Preconditions.checkNotNull(databaseName, "databaseName must be set!!");
tableName = context.getString("tableName");
Preconditions.checkNotNull(tableName, "tableName must be set!!");
user = context.getString("user");
Preconditions.checkNotNull(user, "user must be set!!");
password = context.getString("password");
Preconditions.checkNotNull(password, "password must be set!!");
batchSize = context.getInteger("batchSize", 100);
Preconditions.checkNotNull(batchSize > 0, "batchSize must be a positive number!!");
} @Override
public void start() {
super.start();
try {
//调用Class.forName()方法加载驱动程序
Class.forName("com.mysql.jdbc.Driver");
} catch (ClassNotFoundException e) {
e.printStackTrace();
} String url = "jdbc:mysql://" + hostname + ":" + port + "/" + databaseName;
//调用DriverManager对象的getConnection()方法,获得一个Connection对象 try {
conn = DriverManager.getConnection(url, user, password);
conn.setAutoCommit(false);
//创建一个Statement对象
preparedStatement = conn.prepareStatement("insert into " + tableName +
" (content) values (?)"); } catch (SQLException e) {
e.printStackTrace();
System.exit(1);
} } @Override
public void stop() {
super.stop();
if (preparedStatement != null) {
try {
preparedStatement.close();
} catch (SQLException e) {
e.printStackTrace();
}
} if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
} @Override
public Status process() throws EventDeliveryException {
Status result = Status.READY;
Channel channel = getChannel();
Transaction transaction = channel.getTransaction();
Event event;
String content; List<String> actions = Lists.newArrayList();
transaction.begin();
try {
for (int i = 0; i < batchSize; i++) {
event = channel.take();
if (event != null) {
content = new String(event.getBody());
actions.add(content);
} else {
result = Status.BACKOFF;
break;
}
} if (actions.size() > 0) {
preparedStatement.clearBatch();
for (String temp : actions) {
preparedStatement.setString(1, temp);
preparedStatement.addBatch();
}
preparedStatement.executeBatch(); conn.commit();
}
transaction.commit();
} catch (Throwable e) {
try {
transaction.rollback();
} catch (Exception e2) {
LOG.error("Exception in rollback. Rollback might not have been" +
"successful.", e2);
}
LOG.error("Failed to commit transaction." +
"Transaction rolled back.", e);
Throwables.propagate(e);
} finally {
transaction.close();
} return result;
}
}
3、pom文件
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent>
<artifactId>flume-ng-sinks</artifactId>
<groupId>org.apache.flume</groupId>
<version>1.7.0</version>
</parent>
<artifactId>pf</artifactId>
<packaging>jar</packaging> <name>pf</name>
<url>http://maven.apache.org</url> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties> <dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
</dependency> <dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-configuration</artifactId>
</dependency> <dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.25</version>
</dependency> <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency> <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<scope>test</scope>
</dependency>
</dependencies> <!-- <build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build> -->
</project>
4、建库建表
create database ngmonitor; create table mysqltest(
id int(11) NOT NULL AUTO_INCREMENT,
content varchar(50000) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8;
5、打包--> lib目录下(含mysql的jar)
6、参考conf
# example2.conf: # Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1 # Describe/configure the source
a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /home/wwj/flumeSpool
a1.sources.r1.fileHeader = false
a1.sources.r1.maxBackoff = 60000
#a1.sources.r1.ignorePattern = ^[\s]*\n # Describe the sink
a1.sinks.k1.type = com.pf.MysqlSink
a1.sinks.k1.hostname = localhost
a1.sinks.k1.port = 3306
a1.sinks.k1.databaseName = ngmonitor
a1.sinks.k1.tableName = mysqltest
a1.sinks.k1.user = root
a1.sinks.k1.password= 123456 # Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100 # Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
7、启动
bin/flume-ng agent --conf conf --conf-file conf/example2.conf --name a1 -Dflume.root.logger=INFO,console
8、成功
flume学习的更多相关文章
- Flume学习总结
Flume学习总结 flume是一个用来采集数据的软件,它可以从数据源采集数据到一个集中存放的地方. 最常用flume的数据采集场景是对日志的采集,不过,lume也可以用来采集其他的各种各样的数据,因 ...
- flume学习(三):flume将log4j日志数据写入到hdfs(转)
原文链接:flume学习(三):flume将log4j日志数据写入到hdfs 在第一篇文章中我们是将log4j的日志输出到了agent的日志文件当中.配置文件如下: tier1.sources=sou ...
- Flume学习应用:Java写日志数据到MongoDB
概述 Windows平台:Java写日志到Flume,Flume最终把日志写到MongoDB. 系统环境 操作系统:win7 64 JDK:1.6.0_43 资源下载 Maven:3.3.3下载.安装 ...
- Flume学习 & Kafka & Storm 等 & Log4J 配置
正在学习这篇文章: http://blog.csdn.net/ymh198816/article/details/51998085 和工作中接触的电商.订单.分析,可以结合起来. 开宗明义,这幅图片: ...
- flume学习安装
近期项目组有需求点击流日志须要自己收集,学习了一下flume而且成功安装了.相关信息记录一下. 1)下载flume1.5版本号 wget http://www.apache.org/dyn/clos ...
- flume学习以及ganglia(若是要监控hive日志,hive存放在/tmp/hadoop/hive.log里,只要运行过hive就会有)
python3.6hdfs的使用 https://blog.csdn.net/qq_29863961/article/details/80291654 https://pypi.org/ 官网直接搜 ...
- Flume学习——BasicTransactionSemantics
org.apache.flume.channel.BasicTransactionSemantics An implementation of basic Transaction semantics ...
- Flume学习——Flume中事务的定义
首先要搞清楚的问题是:Flume中的事务用来干嘛? Flume中的事务用来保证消息的可靠传递. 当使用继承自BasicChannelSemantics的Channel时,Flume强制在操作Chann ...
- Flume学习——Flume的架构
Flume有三个组件:Source.Channel 和 Sink.在源码中对应同名的三个接口. When a Flume source receives an event, it stores it ...
随机推荐
- 第31天 mvp
interactor调用接口 Activity包含Presenter,这样界面上的操作就会通知到Presenter. Presenter调用view接口, Activity实现view接口,这样Pre ...
- springmvc之DispatcherServlet
1.作用:DispatcherServlet是前置控制器,配置在web.xml(因为DispatcherServlet是一个servelet)文件中的.拦截匹配的请求,Servlet拦截匹配规则要自已 ...
- maven之clean、install命令
1.进入到maven根目录,执行mvn compile命令会在根目录生成target文件(参照maven之helloworld案例),如下图: 2.执行mvn clean可将根目录下生成的target ...
- Ping CAP CTO、Codis作者谈redis分布式解决方案和分布式KV存储
此文根据[QCON高可用架构群]分享内容,由群内[编辑组]志愿整理,转发请注明出处. 苏东旭,Ping CAP CTO,Codis作者 开源项目Codis的co-author黄东旭,之前在豌豆荚从事i ...
- qt-4.8.5 显示图片居中笔记
已经太久没有写过qt的程序了,所以导致的后果就是一个很简单的程序写了老半天还没写完整. 今天想实现的功能在原来软件的基础上显示他的版本. 因为想在该界面显示一个logo,一开始在pc机上跑发现图片一直 ...
- mysql的安装以及基本操作
一.在Linux 下安装MySQL ubuntu 下可以直接使用apt-get . centos 下yum源有没有就不知道了. 1. sudo apt-get install mysql-server ...
- 获取客户端IP
function getIP(){ $ip = ""; if (getenv("HTTP_CLIENT_IP") && strcasecmp(g ...
- VS无法启动调试:“生成下面的模块时,启用了优化或没有调试信息“
调试项目遇到错误提示,Visual Studio 2010(或VS2008或VS2005)启动调试的时候,弹出提示信息: 生成下面的模块时,启用了优化或没有调试信息: C:\WINDOWS\Micro ...
- Windows下查看局域网内某台计算机的MAC地址
我们知道在局域网中,在Windows下,查看局域网内其他主机的ip和对应mac地址的命令是: arp -a 这样可以看到ip和mac地址的对应关系,还是比较方便的 但是有些时候使用arp命令并不能列出 ...
- selenium 配合sikuli script操作高德地图
会不会使用工具,是一般QA和高级QA的区别 ---To be crazy Java就是好,开源框架遍地都是,各种niubility的jar包,各种神器,真是不亦乐乎. 今天研究一下基于图片识别作为对象 ...