logstash grok 分割匹配日志

使用logstash的时候，为了更细致的切割日志，会写一些正则表达式。 使用方法

        input {
        file {
                type => "billin"
                path => "/data/logs/product/result.log"
            }
        }
        filter  {
        grok    {
            type => "billin"
            pattern => "%{BILLINCENTER}"
            patterns_dir => "/data/logstash/patterns/my_patterns"
            }
        }
        output  {
        redis   {
            host => "192.168.50.13"
            data_type =>"list"
            key => "logstash:redis"
        }
        }

以下内容为正则表达式文件：cat my_patterns

        TAB \t
        META \-+
        WZ ([^ ]*)
        IPPORT %{IP}:%{POSINT}|%{META}
        REQUEST (?:/[A-Za-z0-9$.+!*'(),~:#%_-]*)+\?[A-Za-z0-9$.+!*'(),~#%&/=:;_-]*
        TY (?:(?<!\\)(?:"(?:\\.|[^\\"]+)*"
        #EVERYURL ((\w+://)?([^\.]+)(\.[^/:]+)(:\d*)?([^#]*))|-
        #EVERYURL (((\w+://)?([^\.]+)(\.[^/:]+)?([^#]*))+)|(\w+)|-
        #EVERYURL ((\w+://)?([^\.]+)(\.[^/:]+)?([^#]*))+)|-
        EVERYURL (http://+[\w\d:#@%/;$()~_?\+-=\\\.&]+)|(-)
        #Logformat
        ########nginx access log example########
        #122.137.199.113"122.137.199.113"www.xxxx.com172.16.10.110172.16.12.114:8018/Jun/2013:15:51:03 +0800GET /g/getSaleCounts.do?rnd=1371541857448&showStatus=true&goodsIds=215abd2e8fa95bc8 HTTP/1.120078"http://www.xxxx.com/goods-215abd2e8fa95bc8.html""Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; NP06)""a8fdb711-a695-43bd-abdd-a224fb07350d"
        ###############################
        NGINXACCESSLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{IPPORT:upstrem_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:guid}
        #picture p0.xxxx.com access log . 2012.07.19 add
        PICLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}
        #iis log format 20120618 add
        ###########iis log example###############
        #2013-06-18 08:00:00 172.16.10.233 GET /js/functions.js - 80 - 117.136.34.2 Mozilla/5.0+(Linux;+U;+Android+4.1.2;+zh-CN;+LT22i+Build/6.2.A.0.400)+AppleWebKit/534.31+(KHTML,+like+Gecko)+UCBrowser/9.0.1.275+U3/0.8.0+Mobile+Safari/534.31 200 0 0 0
        ###################################
        IISLOG %{DATE_EU:log_date} %{TIME:log_time} %{IP:server_ip} %{WORD:verb} %{URIPATH:uri_stem} %{WZ:uri_query} %{POSINT:s_port} %{WZ:cs_username} %{IP:c_ip} %{WZ:agent} %{POSINT:request} %{POSINT:substatus} %{POSINT:win32_status} %{POSINT:time_taken}
        #2012/07/12 add
        ZW \w+
        ###java date example
        #  2012-11-27 14:52:42
        ############
        JAVA_DATE %{DATE_EU} %{TIME}
        EARTHLOG \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":"%{ZW:desc}","dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}\}\]
        EAGLEUPDATE \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code},"orderId":"%{ZW:orderId}"\}\]
        EAGLELOGIN \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}\}\]
        #2012/10/23 add
        LJF (-\s+-)
        RESINLOG %{IP:remote_ip}%{SPACE}%{NUMBER}%{SPACE}%{LJF}%{SPACE}\[%{HTTPDATE:timestamp}\]%{SPACE}"%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER}"%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:session}
        #RESINLOG %{IP:ip} %{NUMBER} - - \[%{HTTPDATE:time}\] "%{WORD:verb} %{WZ:request} HTTP/%{NUMBER}" %{NUMVER:response} %{NUMBER:bytes} %{QS:uri} %{QS:agent} %{QS:session}
        #2012/11/13 add
        DKH (\{.*\})
        STOREGREP (\[\/\/\/ \- \] INFO  \-)
        DHMH ([^;|=]*)
        CENTERLOG %{JAVA_DATE} %{STOREGREP} BId=%{NUMBER:bid};BR=%{DHMH:br};BP=%{DKH:bp}
        #2012/11/20 add
        JAVAGREP (\[\/\/\/ \- \])
        ORDERCENTERERR %{JAVA_DATE} \[ RMI TCP Connection\(%{NUMER:threadid}\) -%{IP:ip}\] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{QS:message}
        ORDERCENTERRESULT %{JAVA_DATE} \[ RMI TCP Connection\(%{NUMER:threadid}\) -%{IP:ip}\] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{DKH:message}
        #2012/11/27 add
        #####log example#######
        #2013-06-18 15:28:12 INFO :{message:媒体传递的参数{"uid":["0"],"cid":["A100054947||0000"],"url":["http://www.xxxx.com/?from=lianmeng-weiyi"],"src":["weiyi"]}}
        #
        PARTNER %{JAVA_DATE:timestamp} %{WORD:level} :%{DKH:message}
        #2012/11/28 add
        PARTNERAPI %{JAVA_DATE:timestamp} %{WZ:level} :%{DKH:message}
        #2013/06/18 add
        #pattern all in the '[adskfjl }{\]'
        FKH ([^;]*)
        #######aether.log#####
        #[2013-06-18 15:27:29] [INFO] [com.tuan.web.controller.IndexController] [{message:setHotStore#hot store size:5}]
        AETHERLOG \[%{JAVA_DATE:timestamp}\] \[%{WZ:level}\] \[%{WZ:method}\] %{FKH:message}

        USERNAME [a-zA-Z0-9._-]+
        USER %{USERNAME}
        INT (?:[+-]?(?:[0-9]+))
        BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
        NUMBER (?:%{BASE10NUM})
        BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
        BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b

        POSINT \b(?:[1-9][0-9]*)\b
        NONNEGINT \b(?:[0-9]+)\b
        WORD \b\w+\b
        NOTSPACE \S+
        SPACE \s*
        DATA .*?
        GREEDYDATA .*
        #QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
        QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
        UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
        # Networking
        MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
        CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
        WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
        COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
        IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
        IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
        IP (?:%{IPV6}|%{IPV4})
        HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
        HOST %{HOSTNAME}
        IPORHOST (?:%{HOSTNAME}|%{IP})
        HOSTPORT (?:%{IPORHOST=~/\./}:%{POSINT})
        # paths
        PATH (?:%{UNIXPATH}|%{WINPATH})
        UNIXPATH (?>/(?>[\w_%!$@:.,-]+|\\.)*)+
        #UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+
        TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
        WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
        URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
        URIHOST %{IPORHOST}(?::%{POSINT:port})?
        # uripath comes loosely from RFC1738, but mostly from what Firefox
        # doesn't turn into %XX
        URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
        #URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
        URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
        URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
        URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?

        # Months: January, Feb, 3, 03, 12, December
        MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
        MONTHNUM (?:0?[1-9]|1[0-2])
        MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])

        # Days: Monday, Tue, Thu, etc...
        DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)

        # Years?
        YEAR (?>\d\d){1,2}
        # Time: HH:MM:SS
        #TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
        # I'm still on the fence about using grok to perform the time match,
        # since it's probably slower.
        # TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)?
        HOUR (?:2[0123]|[01]?[0-9])
        MINUTE (?:[0-5][0-9])
        # '60' is a leap second in most time standards and thus is valid.
        SECOND (?:(?:[0-5][0-9]|60)(?:[:.,][0-9]+)?)
        TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
        # datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
        DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
        DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
        ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
        ISO8601_SECOND (?:%{SECOND}|60)
        TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
        DATE %{DATE_US}|%{DATE_EU}
        DATESTAMP %{DATE}[- ]%{TIME}
        TZ (?:[PMCE][SD]T|UTC)
        DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
        DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
        # Syslog Dates: Month Day HH:MM:SS
        SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
        PROG (?:[\w._/%-]+)
        SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
        SYSLOGHOST %{IPORHOST}
        SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
        HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}

        # Shortcuts
        QS %{QUOTEDSTRING}

        # Log formats
        SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
        COMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}

        # Log Levels
        LOGLEVEL ([A-a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)