ELKStack入门篇（二）之Nginx、Tomcat、Java日志收集以及TCP收集日志使用

1、收集Nginx的json格式日志

1.1、Nginx安装

[root@linux-node1 ~]# yum install nginx -y
[root@linux-node1 ~]# vim /etc/nginx/nginx.conf
#修改日志格式为json格式，并创建一个nginxweb的网站目录
log_format access_json '{"@timestamp":"$time_iso8601",'
                           '"host":"$server_addr",'
                           '"clientip":"$remote_addr",'
                           '"size":$body_bytes_sent,'
                           '"responsetime":$request_time,'
                           '"upstreamtime":"$upstream_response_time",'
                           '"upstreamhost":"$upstream_addr",'
                           '"http_host":"$host",'
                           '"url":"$uri",'
                           '"domain":"$host",'
                           '"xff":"$http_x_forwarded_for",'
                           '"referer":"$http_referer",'
                           '"status":"$status"}';
    access_log  /var/log/nginx/access.log  access_json;
 
        location /nginxweb {
                root html;
                index index.html index.htm;
        }
[root@linux-node1 ~]# mkdir /usr/share/nginx/html/nginxweb
[root@linux-node1 ~]# echo "<h1> welcome to use Nginx" > /usr/share/nginx/html/nginxweb/index.html
[root@linux-node1 ~]# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
[root@linux-node1 ~]# systemctl start nginx

1.2、配置logstash

[root@linux-node1 ~]# vim /etc/logstash/conf.d/nginx-accesslog.conf
input{
    file {
        path => "/var/log/nginx/access.log"
        type => "nginx-access-log"
        start_position => "beginning"
        stat_interval => ""
    }
 
}
 
output{
   elasticsearch {
        hosts => ["192.168.56.11:9200"]
        index => "logstash-nginx-access-log-%{+YYYY.MM.dd}"
   }
   file {
        path => "/tmp/logstash-nginx-access-log-%{+YYYY.MM.dd}"
   }
}
[root@linux-node1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_access.conf -t
[root@linux-node1 ~]# systemctl restart logstash

1.3、配置Kibana展示

[root@linux-node1 ~]# ab -n1000 -c  http://192.168.56.11/nginxweb/index.html    #对页面压测
[root@linux-node1 ~]# tailf /var/log/nginx/access.log    #nginx的访问日志变成了json格式
{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}
{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}
{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}
{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}
{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}
{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}
{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}
{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}
{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}
{"@timestamp":"2017-12-27T16:38:17+08:00","host":"192.168.56.11","clientip":"192.168.56.11","size":,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.56.11","url":"/nginxweb/index.html","domain":"192.168.56.11","xff":"-","referer":"-","status":""}

Head插件查看：

Kibana查看：

2、Tomcat的json日志收集

2.1、下载tomcat

[root@linux-node2 ~]# wget http://apache.fayea.com/tomcat/tomcat-8/v8.5.20/bin/apache-tomcat-8.5.20.tar.gz
[root@linux-node2 ~]# tar -zxf apache-tomcat-8.5..tar.gz
[root@linux-node2 ~]# mv apache-tomcat-8.5. /usr/local/tomcat

2.2、修改tomcat日志格式

[root@linux-node2 ~]# cd /usr/local/tomcat/conf
[root@linux-node2 conf ]# cp server.xml{,.bak}
[root@linux-node2 conf ]# vim server.xml
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
               prefix="tomcat_access_log" suffix=".log"
               pattern="{"clientip":"%h","ClientUser":"%l","authenticated":"%u","AccessTime":"%t","method":"%r","status":"%s","SendBytes":"%b","Query?string":"%q","partner":"%{Referer}i","AgentVersion":"%{User-Agent}i"}"/>

2.3、增加tomcat网页

[root@linux-node2 conf ]# cd ../webapps/
[root@linux-node2 webapps ]# mkdir webdir && cd webdir
[root@linux-node2 webdir ]# echo "<h1>welcome to use tomcat</h1>" > index.html
[root@linux-node2 conf ]# ../bin/catalina.sh start
[root@linux-node2 conf ]# netstat -tulnp |grep
tcp6              :::                 :::*                    LISTEN      /java

2.4、压测页面，生成tomcat的访问日志

[root@linux-node2 conf ]# ab -n1000 -c100 http://192.168.56.12:8080/webdir/index.html
[root@linux-node2 ~]# tailf /usr/local/tomcat/logs/tomcat_access_log.--.log
{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}
{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}
{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}
{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}
{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}
{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}
{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}
{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:56 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}
{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}
{"clientip":"192.168.56.12","ClientUser":"-","authenticated":"-","AccessTime":"[28/Dec/2017:09:52:57 +0800]","method":"GET /webdir2/index.html HTTP/1.0","status":"","SendBytes":"","Query?string":"","partner":"-","AgentVersion":"ApacheBench/2.3"}

2.5、配置logstash

[root@linux-node2 ~]# vim /etc/logstash/conf.d/tomcat_accesslog.conf
input {
        file {
                path => "/usr/local/tomcat/logs/tomcat_access_log.*.log"
                type => "tomcat-accesslog"
                start_position => "beginning"
                stat_interval => ""
        }
}
 
output {
        elasticsearch {
                hosts => ["192.168.56.11:9200"]
                index => "logstash-tomcat5612-accesslog-%{+YYYY.MM.dd}"
        }
        file {
                path => "/tmp/logstash-tomcat5612-accesslog-%{+YYYY.MM.dd}"
        }
}

2.6、检查logstash的配置语法并重启logstash

[root@linux-node2 tomcat]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tomcat_accesslog.conf -t
OpenJDK -Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
Configuration OK
[root@linux-node2 tomcat]# systemctl restart logstash
[root@linux-node2 tomcat]# ps -ef |grep logstash
logstash         : ?        :: /bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction= -XX:+UseCMSInitiatingOccupancyOnly -XX:+DisableExplicitGC -Djava.awt.headless=true -Dfile.encoding=UTF- -XX:+HeapDumpOnOutOfMemoryError -Xmx1g -Xms256m -Xss2048k -Djffi.boot.library.path=/usr/share/logstash/vendor/jruby/lib/jni -Xbootclasspath/a:/usr/share/logstash/vendor/jruby/lib/jruby.jar -classpath : -Djruby.home=/usr/share/logstash/vendor/jruby -Djruby.lib=/usr/share/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main /usr/share/logstash/lib/bootstrap/environment.rb logstash/runner.rb --path.settings /etc/logstash
root          : pts/    :: grep --color=auto logstash

2.7、elasticsearch的head插件查看

数据浏览：

注：此处如果无法出现tomcat的数据索引，查看一下是否有权限访问日志。

[root@linux-node2 logs]# ll /usr/local/tomcat/logs/ -d
drwxr-x---  root root  12月  : /usr/local/tomcat/logs/
[root@linux-node2 logs]# chmod  /usr/local/tomcat/logs
[root@linux-node2 logs]# ll /usr/local/tomcat/logs/
总用量
-rw-r-----  root root    12月  : catalina.--.log
-rw-r-----  root root    12月  : catalina.out
-rw-r-----  root root       12月  : host-manager.--.log
-rw-r-----  root root     12月  : localhost.--.log
-rw-r-----  root root       12月  : manager.--.log
-rw-r-----  root root  12月  : tomcat_access_log.--.log
[root@linux-node2 logs]# chmod  /usr/local/tomcat/logs/

2.8、添加到Kibana

3、Java日志收集

使用codec的multiline插件实现多行匹配，这是一个可以将多行进行合并的插件，而且可以使用what指定将匹配到的行与前面的行合并还是和后面的行合并。
https://www.elastic.co/guide/en/logstash/6.0/plugins-codecs-multiline.html

语法例子：
input {
  stdin {
    codec => multiline {    #使用multiline插件
      pattern => "pattern, a regexp"   #正则匹配
      negate => "true" or "false"     #匹配是否成功
      what => "previous" or "next"   #和上面的还是和下面的内容合并
    }
  }
}
命令行测试输入输出
[root@linux-node1 ~]# /usr/share/logstash/bin/logstash -e 'input { stdin {codec => multiline { pattern => "^\[" negate => "true" what => "previous"} }} output { stdout {codec => rubydebug}}'
OpenJDK -Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
The stdin plugin is now waiting for input:
 
[
{
      "@version" => "",
          "host" => "linux-node1",
    "@timestamp" => --28T03::.663Z,
       "message" => "111111111\n22222222222\n333333333",   #会将[开头前面的进行合并
          "tags" => [
        [] "multiline"
    ]
}
 
[
{
      "@version" => "",
          "host" => "linux-node1",
    "@timestamp" => --28T03::.326Z,
       "message" => "[5555555555\n666666666666666666\n77777777777777777\n8888888888",
          "tags" => [
        [] "multiline"
    ]
}

3.1、举例

（1）查看elk集群日志

elk集群日志上都是以”[“开头并且每一个信息都是如此，寻找规律

[root@linux-node1 ~]# tailf /data/logs/elk-cluster.log
[--28T09::,][INFO ][o.e.c.s.MasterService    ] [elk-node1] zen-disco-node-join[{elk-node2}{CcF5fl9sRqCAGYYpT3scuw}{ncgZ1UsPRq-iz6zWHPl7PQ}{192.168.56.12}{192.168.56.12:}], reason: added {{elk-node2}{CcF5fl9sRqCAGYYpT3scuw}{ncgZ1UsPRq-iz6zWHPl7PQ}{192.168.56.12}{192.168.56.12:},}
[--28T09::,][INFO ][o.e.c.s.ClusterApplierService] [elk-node1] added {{elk-node2}{CcF5fl9sRqCAGYYpT3scuw}{ncgZ1UsPRq-iz6zWHPl7PQ}{192.168.56.12}{192.168.56.12:},}, reason: apply cluster state (from master [master {elk-node1}{Ulw9eIPlS06sl8Z6zQ_z4g}{HgJRMEAcQcqFOTn5ehHPdw}{192.168.56.11}{192.168.56.11:} committed version [] source [zen-disco-node-join[{elk-node2}{CcF5fl9sRqCAGYYpT3scuw}{ncgZ1UsPRq-iz6zWHPl7PQ}{192.168.56.12}{192.168.56.12:}]]])
[--28T09::,][WARN ][o.e.d.z.ElectMasterService] [elk-node1] value for setting "discovery.zen.minimum_master_nodes" is too low. This can result in data loss! Please set it to at least a quorum of master-eligible nodes (current value: [-], total number of master-eligible nodes used for publishing in this round: [])
[--28T09::,][INFO ][o.e.c.r.a.AllocationService] [elk-node1] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[.kibana][]] ...]).
[--28T09::,][INFO ][o.e.c.m.MetaDataCreateIndexService] [elk-node1] [logstash-tomcat5612-accesslog-2017.12.] creating index, cause [auto(bulk api)], templates [logstash], shards []/[], mappings [_default_]
[--28T09::,][INFO ][o.e.c.m.MetaDataMappingService] [elk-node1] [logstash-tomcat5612-accesslog-2017.12./YY4yqUQJRHa2mRUwmd2Y8g] create_mapping [tomcat-accesslog]
[--28T09::,][INFO ][o.e.c.r.a.AllocationService] [elk-node1] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[logstash-tomcat5612-accesslog-2017.12.][]] ...]).

（2）配置logstash

[root@linux-node1 ~]# vim /etc/logstash/conf.d/java.conf
input {
        file{
                path => "/data/logs/elk-cluster.log"
                type => "elasticsearch-java-log"
                start_position => "beginning"
                stat_interval => ""
                code => multiline {
                        pattern => "^\["    #以"["开头进行正则匹配，匹配规则
                        negate => "true"  #正则匹配成功，false匹配不成功
                        what => "previous"  #和前面的内容进行合并,如果是和下面的合并就是next
                }
        }
}
 
output {
        if [type] == "elasticsearch-java-log" {
                elasticsearch {
                        hosts => ["192.168.56.11:9200"]
                        index => "elasticsearch-jva-log-%{+YYYY.MM.dd}"
                }
        }
}
[root@linux-node1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/java.conf -t
OpenJDK -Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
Configuration OK
[root@linux-node1 ~]# systemctl restart logstash

（3）elasticsearch的head插件查看