自定义http source

config

a1.sources.r1.type=http
a1.sources.r1.bind=localhost
a1.sources.r1.port=
a1.sources.r1.channels=c1
#自定义source Handler
a1.sources.r1.handler = org.apache.flume.sw.source.http.JSONHandler
a1.sources.r1.handler.configHome = /home/www/logs/datareport

handler

public class JSONHandler implements HTTPSourceHandler {

  private static final Logger LOG = LoggerFactory.getLogger(JSONHandler.class);

  public static final String PARA_SIGN = "sign";
public static final String PARA_PROJECT_ID = "projectId";
public static final String PARA_REPORT_MSG = "reportMsg"; private final Type mapType = new TypeToken<LinkedHashMap<String, Object>>() {}.getType();
private final Gson gson; //可以获取外部参数
private Context context = null; public JSONHandler() {
gson = new GsonBuilder().disableHtmlEscaping().create();
} /**
* {@inheritDoc}
*/
@Override
public List<Event> getEvents(HttpServletRequest request) throws Exception {
BufferedReader reader = request.getReader();
String charset = request.getCharacterEncoding();
//UTF-8 is default for JSON. If no charset is specified, UTF-8 is to
//be assumed.
if (charset == null) {
LOG.debug("Charset is null, default charset of UTF-8 will be used.");
charset = "UTF-8";
} else if (!(charset.equalsIgnoreCase("utf-8")
|| charset.equalsIgnoreCase("utf-16")
|| charset.equalsIgnoreCase("utf-32"))) {
LOG.error("Unsupported character set in request {}. "
+ "JSON handler supports UTF-8, "
+ "UTF-16 and UTF-32 only.", charset);
throw new UnsupportedCharsetException("JSON handler supports UTF-8, "
+ "UTF-16 and UTF-32 only.");
} /*
* Gson throws Exception if the data is not parseable to JSON.
* Need not catch it since the source will catch it and return error.
*/
LinkedHashMap<String, Object> map = new LinkedHashMap<String, Object>();
try {
map = gson.fromJson(reader, mapType);
} catch (JsonSyntaxException ex) {
throw new HTTPBadRequestException("Request has invalid JSON Syntax.", ex);
} String configHome = this.context.getString("configHome");
LOG.info(configHome);
String projectId = map.get(PARA_PROJECT_ID).toString();
String reportMsg = map.get(PARA_REPORT_MSG).toString();
Map<String, String> headers = new HashMap<String, String>();
headers.put(PARA_PROJECT_ID, projectId);
headers.put(PARA_SIGN, "");
JSONEvent jsonEvent = new JSONEvent();
jsonEvent.setHeaders(headers);
jsonEvent.setBody(reportMsg.getBytes()); return getSimpleEvents(jsonEvent);
} @Override
public void configure(Context context) {
this.context = context;
} private List<Event> getSimpleEvents(Event e) {
List<Event> newEvents = new ArrayList<Event>(1);
newEvents.add(EventBuilder.withBody(e.getBody(), e.getHeaders()));
return newEvents;
}
}

自定义Sink

config

#自定义Sink
a1.sinks.k1.type = org.apache.flume.sw.sink.RollingFileSink
a1.sinks.k1.channel = c1
a1.sinks.k1.sink.rollInterval = 15
a1.sinks.k1.sink.directory = D:/var/log/flume
#自定义pathManager类型
a1.sinks.k1.sink.pathManager = CUSTOM
#文件创建频率 (null or yyyyMMddHHmmss), 默认值null->不创建
a1.sinks.k1.sink.pathManager.dirNameFormatter = yyyyMMdd
a1.sinks.k1.sink.pathManager.prefix = log_
a1.sinks.k1.sink.pathManager.extension = txt

自定义RollingFileSink

    if(pathManagerType.equals("CUSTOM")) {
//如果外部配置的PathManager是CUSTOM,则直接new出自定义的SimplePathManager
pathController = new SimplePathManager(pathManagerContext);
} else {
pathController = PathManagerFactory.getInstance(pathManagerType, pathManagerContext);
}

自定义pathManager类型

public class SimplePathManager extends DefaultPathManager {
private static final Logger logger = LoggerFactory
.getLogger(SimplePathManager.class);
private final DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyyMMddHHmmss");
private DateTimeFormatter dirNameFormatter = null; private String lastRoll; public SimplePathManager(Context context) {
super(context); String dirNameFormatterStr = context.getString("dirNameFormatter");
if(dirNameFormatterStr == null || "null".equals(dirNameFormatterStr)){
dirNameFormatter = null;
} else {
dirNameFormatter = DateTimeFormat.forPattern(dirNameFormatterStr);
} } @Override
public File nextFile() {
LocalDateTime now = LocalDateTime.now();
StringBuilder sb = new StringBuilder();
String date = formatter.print(now);
if (!date.equals(lastRoll)) {
getFileIndex().set(0);
lastRoll = date;
}
sb.append(getPrefix()).append(date).append("-");
sb.append(getFileIndex().incrementAndGet());
if (getExtension().length() > 0) {
sb.append(".").append(getExtension());
} File dir = dirNameFormatter != null ? new File(getBaseDirectory(), dirNameFormatter.print(now)) :
getBaseDirectory(); try {
FileUtils.forceMkdir(dir);
currentFile = new File(dir, sb.toString());
} catch (IOException e) {
currentFile = new File(getBaseDirectory(), sb.toString());
logger.error(e.toString(), e);
} return currentFile;
} public static class Builder implements PathManager.Builder {
@Override
public PathManager build(Context context) {
return new SimplePathManager(context);
}
} }

Apache Flume 1.7.0 自定义输入输出的更多相关文章

  1. Apache Flume 1.7.0 发布,日志服务器

    Apache Flume 1.7.0 发布了,Flume 是一个分布式.可靠和高可用的服务,用于收集.聚合以及移动大量日志数据,使用一个简单灵活的架构,就流数据模型.这是一个可靠.容错的服务. 本次更 ...

  2. Apache Flume 1.7.0 源码编译 导入Eclipse

    前言 最近看了看Apache Flume,在虚拟机里跑了一下flume + kafka + storm + mysql架构的demo,功能很简单,主要是用flume收集数据源(http上报信息),放入 ...

  3. Apache Flume 1.6.0 发布,日志服务器

    Apache Flume 1.6.0 发布,此版本现已提供下载: http://flume.apache.org/download.html 更新日志和文档: http://flume.apache. ...

  4. Apache Flume 1.7.0 各个模块简介

    Flume简介 Apache Flume是一个分布式.可靠.高可用的日志收集系统,支持各种各样的数据来源,如http,log文件,jms,监听端口数据等等,能将这些数据源的海量日志数据进行高效收集.聚 ...

  5. Flume 1.5.0简单部署试用

    ================================================================================ 一.Flume简介 ========= ...

  6. Flume官方文档翻译——Flume 1.7.0 User Guide (unreleased version)中一些知识点

    Flume官方文档翻译--Flume 1.7.0 User Guide (unreleased version)(一) Flume官方文档翻译--Flume 1.7.0 User Guide (unr ...

  7. Flume官方文档翻译——Flume 1.7.0 User Guide (unreleased version)(二)

    Flume官方文档翻译--Flume 1.7.0 User Guide (unreleased version)(一) Logging raw data(记录原始数据) Logging the raw ...

  8. Flume官方文档翻译——Flume 1.7.0 User Guide (unreleased version)(一)

    Flume 1.7.0 User Guide Introduction(简介) Overview(综述) System Requirements(系统需求) Architecture(架构) Data ...

  9. Apache Spark 2.2.0 中文文档 - Spark Streaming 编程指南 | ApacheCN

    Spark Streaming 编程指南 概述 一个入门示例 基础概念 依赖 初始化 StreamingContext Discretized Streams (DStreams)(离散化流) Inp ...

随机推荐

  1. String Match

    Finding length of longest common substring /*Finding length of longest common substring using DP * * ...

  2. Effective Java -- 使可变性最小化

    为了使类成为不可变的,应该遵循以下五条原则: 1. 不要提供任何会下盖对象状态的方法 2. 保证类不会被扩展 3. 使所有的域都是final的 4. 使所有的域都成为私有的 5. 确保对于任何可变组件 ...

  3. Fiddler-设置取消自动更新

    fiddler 启动时老弹出要更新,但不想更新,可以这样设置 Tools-Optons->General 把第一个√去掉

  4. 值得推荐的C/C++框架和库 (真的很强大)〔转〕

    http://m.blog.csdn.net/article/details?id=42541419

  5. [译]使用explain API摆脱ElasticSearch集群RED苦恼(转)

    "哔...哔...哗",PagerDuty的报警通知又来了. 可能是因为你又遭遇了节点宕机, 或者服务器机架不可用, 或者整个ElasticSearch集群重启了. 不管哪种情况, ...

  6. docker install

    1.安装必要工具集 sudo yum install -y yum-utils 2.安装Docker官方源 sudo yum-config-manager \ --add-repo \ https:/ ...

  7. 【python-时间戳】时间与时间戳之间的转换

    对于时间数据,如2016-05-05 20:28:54,有时需要与时间戳进行相互的运算,此时就需要对两种形式进行转换,在Python中,转换时需要用到time模块,具体的操作有如下的几种: 将时间转换 ...

  8. 读书笔记二 How Does the Internet work?

    原文链接:  https://web.stanford.edu/class/msande91si/www-spr04/readings/week1/InternetWhitepaper.htm 先写写 ...

  9. 谈谈一些有趣的CSS题目(十六)-- 奇妙的 background-clip: text

    开本系列,谈谈一些有趣的 CSS 题目,题目类型天马行空,想到什么说什么,不仅为了拓宽一下解决问题的思路,更涉及一些容易忽视的 CSS 细节. 解题不考虑兼容性,题目天马行空,想到什么说什么,如果解题 ...

  10. Monad Explained in One Picture

    The point of Monad is composability. In the green category, T -> Monad<U> and U -> Monad ...