logstash grok 分割匹配日志
使用logstash的时候，为了更细致的切割日志，会写一些正则表达式。 使用方法
input {
file {
type => "billin"
path => "/data/logs/product/result.log"
}
}
filter {
grok {
type => "billin"
pattern => "%{BILLINCENTER}"
patterns_dir => "/data/logstash/patterns/my_patterns"
}
}
output {
redis {
host => "192.168.50.13"
data_type =>"list"
key => "logstash:redis"
}
}
以下内容为正则表达式文件：cat my_patterns
TAB \t
META \-+
WZ ([^ ]*)
IPPORT %{IP}:%{POSINT}|%{META}
REQUEST (?:/[A-Za-z0-9$.+!*'(),~:#%_-]*)+\?[A-Za-z0-9$.+!*'(),~#%&/=:;_-]*
TY (?:(?<!\\)(?:"(?:\\.|[^\\"]+)*"
#EVERYURL ((\w+://)?([^\.]+)(\.[^/:]+)(:\d*)?([^#]*))|-
#EVERYURL (((\w+://)?([^\.]+)(\.[^/:]+)?([^#]*))+)|(\w+)|-
#EVERYURL ((\w+://)?([^\.]+)(\.[^/:]+)?([^#]*))+)|-
EVERYURL (http://+[\w\d:#@%/;$()~_?\+-=\\\.&]+)|(-)
#Logformat
########nginx access log example########
#122.137.199.113"122.137.199.113"www.xxxx.com172.16.10.110172.16.12.114:8018/Jun/2013:15:51:03 +0800GET /g/getSaleCounts.do?rnd=1371541857448&showStatus=true&goodsIds=215abd2e8fa95bc8 HTTP/1.120078"http://www.xxxx.com/goods-215abd2e8fa95bc8.html""Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; NP06)""a8fdb711-a695-43bd-abdd-a224fb07350d"
###############################
NGINXACCESSLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{IPPORT:upstrem_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:guid}
#picture p0.xxxx.com access log . 2012.07.19 add
PICLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}
#iis log format 20120618 add
###########iis log example###############
#2013-06-18 08:00:00 172.16.10.233 GET /js/functions.js - 80 - 117.136.34.2 Mozilla/5.0+(Linux;+U;+Android+4.1.2;+zh-CN;+LT22i+Build/6.2.A.0.400)+AppleWebKit/534.31+(KHTML,+like+Gecko)+UCBrowser/9.0.1.275+U3/0.8.0+Mobile+Safari/534.31 200 0 0 0
###################################
IISLOG %{DATE_EU:log_date} %{TIME:log_time} %{IP:server_ip} %{WORD:verb} %{URIPATH:uri_stem} %{WZ:uri_query} %{POSINT:s_port} %{WZ:cs_username} %{IP:c_ip} %{WZ:agent} %{POSINT:request} %{POSINT:substatus} %{POSINT:win32_status} %{POSINT:time_taken}
#2012/07/12 add
ZW \w+
###java date example
# 2012-11-27 14:52:42
############
JAVA_DATE %{DATE_EU} %{TIME}
EARTHLOG \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":"%{ZW:desc}","dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}\}\]
EAGLEUPDATE \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code},"orderId":"%{ZW:orderId}"\}\]
EAGLELOGIN \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}\}\]
#2012/10/23 add
LJF (-\s+-)
RESINLOG %{IP:remote_ip}%{SPACE}%{NUMBER}%{SPACE}%{LJF}%{SPACE}\[%{HTTPDATE:timestamp}\]%{SPACE}"%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER}"%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:session}
#RESINLOG %{IP:ip} %{NUMBER} - - \[%{HTTPDATE:time}\] "%{WORD:verb} %{WZ:request} HTTP/%{NUMBER}" %{NUMVER:response} %{NUMBER:bytes} %{QS:uri} %{QS:agent} %{QS:session}
#2012/11/13 add
DKH (\{.*\})
STOREGREP (\[\/\/\/ \- \] INFO \-)
DHMH ([^;|=]*)
CENTERLOG %{JAVA_DATE} %{STOREGREP} BId=%{NUMBER:bid};BR=%{DHMH:br};BP=%{DKH:bp}
#2012/11/20 add
JAVAGREP (\[\/\/\/ \- \])
ORDERCENTERERR %{JAVA_DATE} \[ RMI TCP Connection\(%{NUMER:threadid}\) -%{IP:ip}\] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{QS:message}
ORDERCENTERRESULT %{JAVA_DATE} \[ RMI TCP Connection\(%{NUMER:threadid}\) -%{IP:ip}\] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{DKH:message}
#2012/11/27 add
#####log example#######
#2013-06-18 15:28:12 INFO :{message:媒体传递的参数{"uid":["0"],"cid":["A100054947||0000"],"url":["http://www.xxxx.com/?from=lianmeng-weiyi"],"src":["weiyi"]}}
#
PARTNER %{JAVA_DATE:timestamp} %{WORD:level} :%{DKH:message}
#2012/11/28 add
PARTNERAPI %{JAVA_DATE:timestamp} %{WZ:level} :%{DKH:message}
#2013/06/18 add
#pattern all in the '[adskfjl }{\]'
FKH ([^;]*)
#######aether.log#####
#[2013-06-18 15:27:29] [INFO] [com.tuan.web.controller.IndexController] [{message:setHotStore#hot store size:5}]
AETHERLOG \[%{JAVA_DATE:timestamp}\] \[%{WZ:level}\] \[%{WZ:method}\] %{FKH:message}
USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
NUMBER (?:%{BASE10NUM})
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
POSINT \b(?:[1-9][0-9]*)\b
NONNEGINT \b(?:[0-9]+)\b
WORD \b\w+\b
NOTSPACE \S+
SPACE \s*
DATA .*?
GREEDYDATA .*
#QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
# Networking
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
IP (?:%{IPV6}|%{IPV4})
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
HOST %{HOSTNAME}
IPORHOST (?:%{HOSTNAME}|%{IP})
HOSTPORT (?:%{IPORHOST=~/\./}:%{POSINT})
# paths
PATH (?:%{UNIXPATH}|%{WINPATH})
UNIXPATH (?>/(?>[\w_%!$@:.,-]+|\\.)*)+
#UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
URIHOST %{IPORHOST}(?::%{POSINT:port})?
# uripath comes loosely from RFC1738, but mostly from what Firefox
# doesn't turn into %XX
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
# Months: January, Feb, 3, 03, 12, December
MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
MONTHNUM (?:0?[1-9]|1[0-2])
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
# Days: Monday, Tue, Thu, etc...
DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
# Years?
YEAR (?>\d\d){1,2}
# Time: HH:MM:SS
#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
# I'm still on the fence about using grok to perform the time match,
# since it's probably slower.
# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)?
HOUR (?:2[0123]|[01]?[0-9])
MINUTE (?:[0-5][0-9])
# '60' is a leap second in most time standards and thus is valid.
SECOND (?:(?:[0-5][0-9]|60)(?:[:.,][0-9]+)?)
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
ISO8601_SECOND (?:%{SECOND}|60)
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
DATE %{DATE_US}|%{DATE_EU}
DATESTAMP %{DATE}[- ]%{TIME}
TZ (?:[PMCE][SD]T|UTC)
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
# Syslog Dates: Month Day HH:MM:SS
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
PROG (?:[\w._/%-]+)
SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
SYSLOGHOST %{IPORHOST}
SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
# Shortcuts
QS %{QUOTEDSTRING}
# Log formats
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
COMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}
# Log Levels
LOGLEVEL ([A-a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
logstash grok 分割匹配日志的更多相关文章
- Logstash使用grok解析IIS日志
Logstash使用grok解析IIS日志 1. 安装配置 安装Logstash前请确认Elasticsearch已经安装正确,参见RedHat6.4安装Elasticsearch5.2.0. 下载链 ...
- logstash的grok正则匹配规则文件
文件路径:logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-4.1.0/patterns/grok-patterns 在线调试g ...
- filebeat+logstash+elasticsearch收集haproxy日志
filebeat用于是日志收集,感觉和 flume相同,但是用go开发,性能比较好 在2.4版本中, 客户机部署logstash收集匹配日志,传输到 kafka,在用logstash 从消息队列中抓取 ...
- elk系列7之通过grok分析apache日志【转】
preface 说道分析日志,我们知道的采集方式有2种: 通过grok在logstash的filter里面过滤匹配. logstash --> redis --> python(py脚本过 ...
- [elk]logstash grok原理
logstash语法 http://www.ttlsa.com/elk/elk-logstash-configuration-syntax/ https://www.elastic.co/guide/ ...
- logstash -grok插件语法介绍
介绍 logstash拥有丰富的filter插件,它们扩展了进入过滤器的原始数据,进行复杂的逻辑处理,甚至可以无中生有的添加新的 logstash 事件到后续的流程中去!Grok 是 Logsta ...
- logstash处理多行日志-处理java堆栈日志
logstash处理多行日志-处理java堆栈日志 一.背景 二.需求 三.实现思路 1.分析日志 2.实现,编写pipeline文件 四.注意事项 五.参考文档 一.背景 在我们的java程序中,经 ...
- 用Kibana和logstash快速搭建实时日志查询、收集与分析系统
Logstash是一个完全开源的工具,他可以对你的日志进行收集.分析,并将其存储供以后使用(如,搜索),您可以使用它.说到搜索,logstash带有一个web界面,搜索和展示所有日志. kibana ...
- grok 正则解析日志例子<1>
<pre name="code" class="html">下面是日志的样子 55.3.244.1 GET /index.html 15824 0. ...
随机推荐
- ASP.NET MVC学习（一）之路由篇Route
什么是路由 通过[路由]配置,路由可以规定URL的特殊格式,使其达到特殊效果. 在ASP.NET MVC框架中,通过路由配置URL,使用户的URL请求可以映射到Controller下的action方法 ...
- [整理]HTML5 WebSocket
vs2013 win7 iis7.5貌似不行,查阅了资料,好像得iis8支持 <ASP.NET SignalR系列>第一课 认识SignalR http://www.cnblogs.com ...
- 关于z-index这个层级的问题
z-index它可真是一个神奇的东西,它可以随意安排你的层的叠加问题. 如你想让红色矩形压在蓝色矩形上,正常布局先建立一个红色的再建议一个蓝色的,就可以了. 但如果我相反来建立,那么就得借助z-ind ...
- 【51Nod】1510 最小化序列 贪心+动态规划
[题目]1510 最小化序列 [题意]给定长度为n的数组A和数字k,要求重排列数组从而最小化: \[ans=\sum_{i=1}^{n-k}|A_i-A_{i+k}|\] 输出最小的ans,\(n \ ...
- iOS 根据生日得到生肖，星座，年龄的算法
根据用户生日,得到相应的年龄,星座和生肖.有的项目中可能会用到,贴出来共享. 得到年龄,这个很简单了: - (void)getAgeWith:(NSDate*)birthday{ //日历 NSCal ...
- 第7月第25天 xcode bundle calayer动画
1. http://www.jianshu.com/p/3755c35ac87f 2.calayer http://www.jianshu.com/p/aadfce72bf74
- kali更新失败
今天更新kali时失败,出现如下问题: root@kali:~# apt-get update Get: http://mirrors.aliyun.com/kali kali-rolling InR ...
- Linux环境下FTP工具的使用方法
在Windows环境下创建Ftp目录作为服务器根目录 在Linux端的操作: 从服务器端下载文件到Linux端: ftpget -u User -p Password ServerIP File Fi ...
- 谁在call我-backtrace的实现原理【转】
转自:http://www.xuebuyuan.com/1504689.html 显示函数调用关系(backtrace/callstack)是调试器必备的功能之一,比如在gdb里,用bt命令就可以查看 ...
- django-xadmin后台开发
先通过pip命令行安装django<=1.9版本 示例:pip install django==1.9 从https://github.com/sshwsfc/xadmin下载xadmin源码解 ...