自定义http source

config

a1.sources.r1.type=http
a1.sources.r1.bind=localhost
a1.sources.r1.port=
a1.sources.r1.channels=c1
#自定义source Handler
a1.sources.r1.handler = org.apache.flume.sw.source.http.JSONHandler
a1.sources.r1.handler.configHome = /home/www/logs/datareport

handler

public class JSONHandler implements HTTPSourceHandler {

  private static final Logger LOG = LoggerFactory.getLogger(JSONHandler.class);

  public static final String PARA_SIGN = "sign";
public static final String PARA_PROJECT_ID = "projectId";
public static final String PARA_REPORT_MSG = "reportMsg"; private final Type mapType = new TypeToken<LinkedHashMap<String, Object>>() {}.getType();
private final Gson gson; //可以获取外部参数
private Context context = null; public JSONHandler() {
gson = new GsonBuilder().disableHtmlEscaping().create();
} /**
* {@inheritDoc}
*/
@Override
public List<Event> getEvents(HttpServletRequest request) throws Exception {
BufferedReader reader = request.getReader();
String charset = request.getCharacterEncoding();
//UTF-8 is default for JSON. If no charset is specified, UTF-8 is to
//be assumed.
if (charset == null) {
LOG.debug("Charset is null, default charset of UTF-8 will be used.");
charset = "UTF-8";
} else if (!(charset.equalsIgnoreCase("utf-8")
|| charset.equalsIgnoreCase("utf-16")
|| charset.equalsIgnoreCase("utf-32"))) {
LOG.error("Unsupported character set in request {}. "
+ "JSON handler supports UTF-8, "
+ "UTF-16 and UTF-32 only.", charset);
throw new UnsupportedCharsetException("JSON handler supports UTF-8, "
+ "UTF-16 and UTF-32 only.");
} /*
* Gson throws Exception if the data is not parseable to JSON.
* Need not catch it since the source will catch it and return error.
*/
LinkedHashMap<String, Object> map = new LinkedHashMap<String, Object>();
try {
map = gson.fromJson(reader, mapType);
} catch (JsonSyntaxException ex) {
throw new HTTPBadRequestException("Request has invalid JSON Syntax.", ex);
} String configHome = this.context.getString("configHome");
LOG.info(configHome);
String projectId = map.get(PARA_PROJECT_ID).toString();
String reportMsg = map.get(PARA_REPORT_MSG).toString();
Map<String, String> headers = new HashMap<String, String>();
headers.put(PARA_PROJECT_ID, projectId);
headers.put(PARA_SIGN, "");
JSONEvent jsonEvent = new JSONEvent();
jsonEvent.setHeaders(headers);
jsonEvent.setBody(reportMsg.getBytes()); return getSimpleEvents(jsonEvent);
} @Override
public void configure(Context context) {
this.context = context;
} private List<Event> getSimpleEvents(Event e) {
List<Event> newEvents = new ArrayList<Event>(1);
newEvents.add(EventBuilder.withBody(e.getBody(), e.getHeaders()));
return newEvents;
}
}

自定义Sink

config

#自定义Sink
a1.sinks.k1.type = org.apache.flume.sw.sink.RollingFileSink
a1.sinks.k1.channel = c1
a1.sinks.k1.sink.rollInterval = 15
a1.sinks.k1.sink.directory = D:/var/log/flume
#自定义pathManager类型
a1.sinks.k1.sink.pathManager = CUSTOM
#文件创建频率 (null or yyyyMMddHHmmss), 默认值null->不创建
a1.sinks.k1.sink.pathManager.dirNameFormatter = yyyyMMdd
a1.sinks.k1.sink.pathManager.prefix = log_
a1.sinks.k1.sink.pathManager.extension = txt

自定义RollingFileSink

    if(pathManagerType.equals("CUSTOM")) {
//如果外部配置的PathManager是CUSTOM,则直接new出自定义的SimplePathManager
pathController = new SimplePathManager(pathManagerContext);
} else {
pathController = PathManagerFactory.getInstance(pathManagerType, pathManagerContext);
}

自定义pathManager类型

public class SimplePathManager extends DefaultPathManager {
private static final Logger logger = LoggerFactory
.getLogger(SimplePathManager.class);
private final DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyyMMddHHmmss");
private DateTimeFormatter dirNameFormatter = null; private String lastRoll; public SimplePathManager(Context context) {
super(context); String dirNameFormatterStr = context.getString("dirNameFormatter");
if(dirNameFormatterStr == null || "null".equals(dirNameFormatterStr)){
dirNameFormatter = null;
} else {
dirNameFormatter = DateTimeFormat.forPattern(dirNameFormatterStr);
} } @Override
public File nextFile() {
LocalDateTime now = LocalDateTime.now();
StringBuilder sb = new StringBuilder();
String date = formatter.print(now);
if (!date.equals(lastRoll)) {
getFileIndex().set(0);
lastRoll = date;
}
sb.append(getPrefix()).append(date).append("-");
sb.append(getFileIndex().incrementAndGet());
if (getExtension().length() > 0) {
sb.append(".").append(getExtension());
} File dir = dirNameFormatter != null ? new File(getBaseDirectory(), dirNameFormatter.print(now)) :
getBaseDirectory(); try {
FileUtils.forceMkdir(dir);
currentFile = new File(dir, sb.toString());
} catch (IOException e) {
currentFile = new File(getBaseDirectory(), sb.toString());
logger.error(e.toString(), e);
} return currentFile;
} public static class Builder implements PathManager.Builder {
@Override
public PathManager build(Context context) {
return new SimplePathManager(context);
}
} }

Apache Flume 1.7.0 自定义输入输出的更多相关文章

  1. Apache Flume 1.7.0 发布,日志服务器

    Apache Flume 1.7.0 发布了,Flume 是一个分布式.可靠和高可用的服务,用于收集.聚合以及移动大量日志数据,使用一个简单灵活的架构,就流数据模型.这是一个可靠.容错的服务. 本次更 ...

  2. Apache Flume 1.7.0 源码编译 导入Eclipse

    前言 最近看了看Apache Flume,在虚拟机里跑了一下flume + kafka + storm + mysql架构的demo,功能很简单,主要是用flume收集数据源(http上报信息),放入 ...

  3. Apache Flume 1.6.0 发布,日志服务器

    Apache Flume 1.6.0 发布,此版本现已提供下载: http://flume.apache.org/download.html 更新日志和文档: http://flume.apache. ...

  4. Apache Flume 1.7.0 各个模块简介

    Flume简介 Apache Flume是一个分布式.可靠.高可用的日志收集系统,支持各种各样的数据来源,如http,log文件,jms,监听端口数据等等,能将这些数据源的海量日志数据进行高效收集.聚 ...

  5. Flume 1.5.0简单部署试用

    ================================================================================ 一.Flume简介 ========= ...

  6. Flume官方文档翻译——Flume 1.7.0 User Guide (unreleased version)中一些知识点

    Flume官方文档翻译--Flume 1.7.0 User Guide (unreleased version)(一) Flume官方文档翻译--Flume 1.7.0 User Guide (unr ...

  7. Flume官方文档翻译——Flume 1.7.0 User Guide (unreleased version)(二)

    Flume官方文档翻译--Flume 1.7.0 User Guide (unreleased version)(一) Logging raw data(记录原始数据) Logging the raw ...

  8. Flume官方文档翻译——Flume 1.7.0 User Guide (unreleased version)(一)

    Flume 1.7.0 User Guide Introduction(简介) Overview(综述) System Requirements(系统需求) Architecture(架构) Data ...

  9. Apache Spark 2.2.0 中文文档 - Spark Streaming 编程指南 | ApacheCN

    Spark Streaming 编程指南 概述 一个入门示例 基础概念 依赖 初始化 StreamingContext Discretized Streams (DStreams)(离散化流) Inp ...

随机推荐

  1. 获取window.location.href中传的值,并且转换成json数据使用

    做个记录保存一下,以免以后再次用到忘记了. function locVal(){ var url=window.location.href; if (url.indexOf('?')==-1)retu ...

  2. springBoot 使用拦截器 入坑

    近期使用SpringBoot 其中用到了拦截器 结果我的静态资源被全部拦截了,让我导致了好久才搞好: 看下图项目结构: 问题描述:上图划红框的资源都被拦截器给拦截了,搞得项目中不能访问:解决问题就是在 ...

  3. C# MVC验证Model

    .NET Core MVC3 数据模型验证的使用 这里我先粘贴一个已经加了数据验证的实体类PeopleModel,然后一一介绍. using System; using System.Collecti ...

  4. media 标签解释

    一:常用标签这句话是自动设置缩放,然而,它并不能完全适应所有的手机,并且你在用浏览器手机模式调试的时候可能正常,但是换到真实的手机端其实是不正常的.所以我们还要进行改动. <meta name= ...

  5. 微信如何实现自动跳转到用其他浏览器打开指定页面下载APP

    不管是app的下载链接还是普通文件的链接在微信内置浏览器或者QQ内置浏览器都会被屏蔽.这是微信对第三方下载域名实施的拦截政策.被拦截了用户在微信内打开就会提示“已停止访问该网页”. 那么当我们遇到这个 ...

  6. 消除blur属性的边框

    直接设置样式为:  outline:none <!DOCTYPE html> <html lang="en"> <head> <meta ...

  7. eclipse设置新建jsp文件默认字符编码为utf-8

    在使用Eclipse开发中,编码默认是ISO-8859-1,不支持中文.这样我们每次新建文件都要手动修改编码,非常麻烦.其实我们可以设置文件默认编码,今后再新建文件时就不用修改编码了. 1.打开Ecl ...

  8. Python学习过程中各个难点---数据类型篇

    ---恢复内容开始--- 当时在学习python的基本数据类型时,对于可变与不可变类型不是了解的很透彻,这篇是回过头来自己的一些理解. 可变的数据类型有列表,不可变的数据类型有字符串,数字和元组   ...

  9. Android中弹出dialog后无法捕捉back键

    一.需求 在Android开发过程中,弹出dialog后无法捕捉back键,点击back按键无响应. 二.解决方案 原因:弹出dialog后,activity失去焦点,dialog获得当前焦点. 解决 ...

  10. ByteArrayInputStream

    package org.example.io; import java.io.ByteArrayInputStream;import java.io.IOException; /** * ByteAr ...