ELK安装笔记
1、jdk安装
2、logstash安装使用
#命令方式
[root@ELK ELK]# tar xf logstash-5.3.2.tar.gz
[root@ELK logstash-5.3.2]# ./logstash-5.3.2/bin/logstash -e "input{stdin{}}output{stdout{codec=>rubydebug}}"
Sending Logstash's logs to /root/src/ELK/logstash-5.3.2/logs which is now configured via log4j2.properties
[2017-05-09T21:45:46,608][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.queue", :path=>"/root/src/ELK/logstash-5.3.2/data/queue"}
[2017-05-09T21:45:46,627][INFO ][logstash.agent ] No persistent UUID file found. Generating new UUID {:uuid=>"b9addbdf-c719-42a9-aa6b-22a757ae0c61", :path=>"/root/src/ELK/logstash-5.3.2/data/uuid"}
[2017-05-09T21:45:46,793][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>500}
[2017-05-09T21:45:46,816][INFO ][logstash.pipeline ] Pipeline main started
The stdin plugin is now waiting for input:
[2017-05-09T21:45:46,903][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
hello world
{
"@timestamp" => 2017-05-09T13:46:38.710Z,
"@version" => "1",
"host" => "ELK",
"message" => "hello world"
}
#配置文件
[root@ELK ELK]# mkdir conf
[root@ELK ELK]# vim conf/test.conf
input {
stdin{}
} output {
stdout {
codec => rubydebug{}
}
}
[root@ELK ELK]# ./logstash-5.3.2/bin/logstash -f conf/test.conf
Sending Logstash's logs to /root/src/ELK/logstash-5.3.2/logs which is now configured via log4j2.properties
[2017-05-09T21:59:31,733][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>500}
[2017-05-09T21:59:31,778][INFO ][logstash.pipeline ] Pipeline main started
The stdin plugin is now waiting for input:
[2017-05-09T21:59:31,828][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
hello world
{
"@timestamp" => 2017-05-09T13:59:42.881Z,
"@version" => "1",
"host" => "ELK",
"message" => "hello world"
} #后台运行
#======================================================
#方式1
nohup ./logstash-5.3.2/bin/logstash -f conf/test.conf &
tail -f nohup.out #方式2
yum install -y screen
screen
./logstash-5.3.2/bin/logstash -f conf/test.conf
ctrl+A+D
screen -ls
screen -r pid #方式3
通过supervisord管理 #file插件
[root@ELK ELK]# vim conf/test.conf
input {
file {
path => ["/usr/local/nginx/logs/access.log"]
type => "nginx"
start_position => "beginning" //beginning表示从文件开头读取，end表示从文件末尾读取
}
}
output {
stdout {
codec => rubydebug{}
}
}
[root@ELK ELK]# ./logstash-5.3.2/bin/logstash -f conf/test.conf
Sending Logstash's logs to /root/src/ELK/logstash-5.3.2/logs which is now configured via log4j2.properties
[2017-05-10T20:27:53,080][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>500}
[2017-05-10T20:27:53,179][INFO ][logstash.pipeline ] Pipeline main started
[2017-05-10T20:27:53,226][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
{
"path" => "/usr/local/nginx/logs/access.log",
"@timestamp" => 2017-05-10T12:27:57.235Z,
"@version" => "1",
"host" => "ELK",
"message" => "192.168.1.22 - - [10/May/2017:20:27:57 +0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0\"",
"type" => "nginx"
} #多行插件
[root@ELK ELK]# vim conf/test.conf
input {
stdin {
codec => multiline {
pattern => "^\["
negate => true
what => "previous"
}
}
}
output {
stdout {
codec => rubydebug{}
}
}
[root@ELK ELK]# ./logstash-5.3.2/bin/logstash -f conf/test.conf
Sending Logstash's logs to /root/src/ELK/logstash-5.3.2/logs which is now configured via log4j2.properties
[2017-05-10T20:55:15,045][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>500}
[2017-05-10T20:55:15,069][INFO ][logstash.pipeline ] Pipeline main started
The stdin plugin is now waiting for input:
[2017-05-10T20:55:15,126][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[ hello world
[ hello logstash
{
"@timestamp" => 2017-05-10T12:55:45.068Z,
"@version" => "1",
"host" => "ELK",
"message" => "[ hello world"
}
hello nihao
end
[ stop
{
"@timestamp" => 2017-05-10T12:56:00.774Z,
"@version" => "1",
"host" => "ELK",
"message" => "[ hello logstash\nhello nihao\nend",
"tags" => [
[0] "multiline"
]
}
[test
{
"@timestamp" => 2017-05-10T12:56:05.070Z,
"@version" => "1",
"host" => "ELK",
"message" => "[ stop"
} #grok & kv插件 官方提供的grok表达式
https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns grok在线调试
http://grokdebug.herokuapp.com/ 配置文件示例：直接调用grok表达式: %{IPV4:client} (%{USERNAME:ident}|-) (%{USERNAME:auth}|-) \[%{HTTPDATE:timestamp}\]
自定义正则：(?<client>\d{3}.\d{3}.\d{1}.\d{2})
kv {
source => "request" //要处理的字串
field_split => "&" //分隔符
value_split => "=" //键值对使用=对应
remove_field => [ "aw" ] //要删除的字段
}
#json & geoip插件
nginx 日志配置
log_format json '{ "@timestamp": "$time_local", '
'"@fields": { '
'"remote_addr": "$remote_addr", '
'"remote_user": "$remote_user", '
'"body_bytes_sent": "$body_bytes_sent", '
'"request_time": "$request_time", '
'"status": "$status", '
'"request": "$request", '
'"request_method": "$request_method", '
'"http_referrer": "$http_referer", '
'"body_bytes_sent":"$body_bytes_sent", '
'"http_x_forwarded_for": "$http_x_forwarded_for", '
'"http_user_agent": "$http_user_agent" } }';
access_log logs/access.log json;
logstash配置文件
input {
file {
path => ["/usr/local/nginx/logs/access.log"]
type => "nginx"
start_position => "end"
codec => json
}
}
filter {
geoip {
source => "client"
fields => ["ip","city_name","country_name","location"]
}
}
output {
stdout {
codec => rubydebug{}
}
}
#elasticsearch安装
[root@ELK ELK]# tar xf elasticsearch-5.3.2.tar.gz
[root@ELK ELK]# cd elasticsearch-5.3.2/
[root@ELK elasticsearch-5.3.2]# ./bin/elasticsearch
[2017-05-14T11:28:21,635][WARN ][o.e.b.ElasticsearchUncaughtExceptionHandler] [] uncaught exception in thread [main]
org.elasticsearch.bootstrap.StartupException: java.lang.RuntimeException: can not run elasticsearch as root
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:127) ~[elasticsearch-5.3.2.jar:5.3.2]
at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:114) ~[elasticsearch-5.3.2.jar:5.3.2]
at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:58) ~[elasticsearch-5.3.2.jar:5.3.2]
at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:122) ~[elasticsearch-5.3.2.jar:5.3.2]
at org.elasticsearch.cli.Command.main(Command.java:88) ~[elasticsearch-5.3.2.jar:5.3.2]
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91) ~[elasticsearch-5.3.2.jar:5.3.2]
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84) ~[elasticsearch-5.3.2.jar:5.3.2]
Caused by: java.lang.RuntimeException: can not run elasticsearch as root
at org.elasticsearch.bootstrap.Bootstrap.initializeNatives(Bootstrap.java:106) ~[elasticsearch-5.3.2.jar:5.3.2]
at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:204) ~[elasticsearch-5.3.2.jar:5.3.2]
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:360) ~[elasticsearch-5.3.2.jar:5.3.2]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:123) ~[elasticsearch-5.3.2.jar:5.3.2]
... 6 more
[root@ELK elasticsearch-5.3.2]#cd ..
[root@ELK ELK]# useradd elk
[root@ELK ELK]# chown -R elk.elk elasticsearch-5.3.2
[root@ELK ELK]# su - elk
[elk@ELK ~]$ cd /ELK/elasticsearch-5.3.2
[elk@ELK elasticsearch-5.3.2]$ ./bin/elasticsearch
[2017-05-14T11:37:38,166][INFO ][o.e.n.Node ] [] initializing ...
[2017-05-14T11:37:39,905][INFO ][o.e.e.NodeEnvironment ] [eHbK7Fh] using [1] data paths, mounts [[/ (rootfs)]], net usable_space [32.1gb], net total_space [37.4gb], spins? [unknown], types [rootfs]
[2017-05-14T11:37:39,907][INFO ][o.e.e.NodeEnvironment ] [eHbK7Fh] heap size [1.9gb], compressed ordinary object pointers [true]
[2017-05-14T11:37:39,913][INFO ][o.e.n.Node ] node name [eHbK7Fh] derived from node ID [eHbK7FhJTPyu9_1GNbXzmA]; set [node.name] to override
[2017-05-14T11:37:39,918][INFO ][o.e.n.Node ] version[5.3.2], pid[7121], build[3068195/2017-04-24T16:15:59.481Z], OS[Linux/3.10.0-327.el7.x86_64/amd64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/1.8.0_131/25.131-b11]
[2017-05-14T11:37:52,245][INFO ][o.e.p.PluginsService ] [eHbK7Fh] loaded module [aggs-matrix-stats]
[2017-05-14T11:37:52,266][INFO ][o.e.p.PluginsService ] [eHbK7Fh] loaded module [ingest-common]
[2017-05-14T11:37:52,267][INFO ][o.e.p.PluginsService ] [eHbK7Fh] loaded module [lang-expression]
[2017-05-14T11:37:52,267][INFO ][o.e.p.PluginsService ] [eHbK7Fh] loaded module [lang-groovy]
[2017-05-14T11:37:52,267][INFO ][o.e.p.PluginsService ] [eHbK7Fh] loaded module [lang-mustache]
[2017-05-14T11:37:52,267][INFO ][o.e.p.PluginsService ] [eHbK7Fh] loaded module [lang-painless]
[2017-05-14T11:37:52,268][INFO ][o.e.p.PluginsService ] [eHbK7Fh] loaded module [percolator]
[2017-05-14T11:37:52,268][INFO ][o.e.p.PluginsService ] [eHbK7Fh] loaded module [reindex]
[2017-05-14T11:37:52,269][INFO ][o.e.p.PluginsService ] [eHbK7Fh] loaded module [transport-netty3]
[2017-05-14T11:37:52,269][INFO ][o.e.p.PluginsService ] [eHbK7Fh] loaded module [transport-netty4]
[2017-05-14T11:37:52,271][INFO ][o.e.p.PluginsService ] [eHbK7Fh] no plugins loaded
[2017-05-14T11:38:21,270][INFO ][o.e.n.Node ] initialized
[2017-05-14T11:38:21,272][INFO ][o.e.n.Node ] [eHbK7Fh] starting ...
[2017-05-14T11:38:23,936][INFO ][o.e.t.TransportService ] [eHbK7Fh] publish_address {127.0.0.1:9300}, bound_addresses {[::1]:9300}, {127.0.0.1:9300}
[2017-05-14T11:38:23,962][WARN ][o.e.b.BootstrapChecks ] [eHbK7Fh] max file descriptors [4096] for elasticsearch process is too low, increase to at least [65536]
[2017-05-14T11:38:23,962][WARN ][o.e.b.BootstrapChecks ] [eHbK7Fh] max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
[2017-05-14T11:38:27,096][WARN ][o.e.m.j.JvmGcMonitorService] [eHbK7Fh] [gc][5] overhead, spent [582ms] collecting in the last [1s]
[2017-05-14T11:38:28,664][INFO ][o.e.c.s.ClusterService ] [eHbK7Fh] new_master {eHbK7Fh}{eHbK7FhJTPyu9_1GNbXzmA}{U5BQ8QlXSqarCAc4r4aU-A}{127.0.0.1}{127.0.0.1:9300}, reason: zen-disco-elected-as-master ([0] nodes joined)
[2017-05-14T11:38:29,539][INFO ][o.e.h.n.Netty4HttpServerTransport] [eHbK7Fh] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200}
[2017-05-14T11:38:29,556][INFO ][o.e.n.Node ] [eHbK7Fh] started
[2017-05-14T11:38:29,706][INFO ][o.e.g.GatewayService ] [eHbK7Fh] recovered [0] indices into cluster_state
[root@ELK ELK]# netstat -anpt
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 5329/nginx: master
tcp 0 0 192.168.122.1:53 0.0.0.0:* LISTEN 2502/dnsmasq
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 1224/sshd
tcp 0 0 127.0.0.1:631 0.0.0.0:* LISTEN 1227/cupsd
tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 2383/master
tcp 0 0 192.168.1.111:22 192.168.1.22:55363 ESTABLISHED 4739/sshd: root@pts
tcp 0 0 192.168.1.111:22 192.168.1.22:55300 ESTABLISHED 4605/sshd: root@pts
tcp 0 0 192.168.1.111:22 192.168.1.22:50543 ESTABLISHED 2962/sshd: root@pts
tcp6 0 0 127.0.0.1:9200 :::* LISTEN 7121/java
tcp6 0 0 ::1:9200 :::* LISTEN 7121/java
tcp6 0 0 127.0.0.1:9300 :::* LISTEN 7121/java
tcp6 0 0 ::1:9300 :::* LISTEN 7121/java
tcp6 0 0 :::22 :::* LISTEN 1224/sshd
tcp6 0 0 ::1:631 :::* LISTEN 1227/cupsd
#kibana安装
[root@ELK ELK]# tar xf kibana-5.3.2-linux-x86_64.tar.gz
[root@ELK ELK]# cd kibana-5.3.2-linux-x86_64/
[root@ELK kibana-5.3.2-linux-x86_64]# ./bin/kibana
log [03:42:22.544] [info][status][plugin:kibana@5.3.2] Status changed from uninitialized to green - Ready
log [03:42:22.757] [info][status][plugin:elasticsearch@5.3.2] Status changed from uninitialized to yellow - Waiting for Elasticsearch
log [03:42:22.778] [info][status][plugin:console@5.3.2] Status changed from uninitialized to green - Ready
log [03:42:23.999] [info][status][plugin:timelion@5.3.2] Status changed from uninitialized to green - Ready
log [03:42:24.058] [info][listening] Server running at http://localhost:5601
log [03:42:24.061] [info][status][ui settings] Status changed from uninitialized to yellow - Elasticsearch plugin is yellow
log [03:42:30.414] [info][status][plugin:elasticsearch@5.3.2] Status changed from yellow to yellow - No existing Kibana index found
log [03:42:32.770] [info][status][plugin:elasticsearch@5.3.2] Status changed from yellow to green - Kibana index ready
log [03:42:32.771] [info][status][ui settings] Status changed from yellow to green - Ready
#elk组合配置
elasticsearch
[root@ELK elasticsearch-5.3.2]# vim config/elasticsearch.yml
network.host: 192.168.1.111
logstash
[root@ELK ELK]# vim conf/test.conf
input {
file {
path => ["/usr/local/nginx/logs/access.log"]
type => "nginx"
start_position => "end"
codec => json
}
}
filter {
geoip {
source => "remote_addr"
#fields => ["ip","city_name","country_name","location"]
}
}
output {
#stdout {
# codec => rubydebug{}
#}
elasticsearch {
hosts => ["192.168.1.111:9200"]
index => "logstash-%{+YYYY.MM.dd}"
}
}
kibana
[root@ELK kibana-5.3.2-linux-x86_64]# vim config/kibana.yml
server.host: "192.168.1.111"
elasticsearch.url: "http://192.168.1.111:9200"
ip排名前5图表
状态码统计
统计响应时间
地图
仪表盘
ELK安装笔记的更多相关文章
- [2017-07-18]ELK安装笔记
ELK ElasticSearch LogStash Kibana Server:CentOS 7 采用RPM导入官方源方式进行安装 rpm --import https://artifacts.el ...
- ELK 安装笔记
logstash -noverify -javaagent:E:\svn\other\nn\jrebel6.0.0+crack\jrebel6.0.0-crack\jrebel.jar -Drebel ...
- Elk使用笔记(坑)（2017-02-17更新）
Elk使用笔记(坑)(2017-02-17更新) 作者: admin 时间: 2016-12-07 分类: 工具,数据 主要记录使用过程终于到的一些坑和需要注意的地方,有些坑想不起来了,以后再完善补上 ...
- MonoDevelop 4.2.2/Mono 3.4.0 in CentOS 6.5 安装笔记
MonoDevelop 4.2.2/Mono 3.4.0 in CentOS 6.5 安装笔记 说明 以root账户登录Linux操作系统,注意:本文中的所有命令行前面的 #> 表示命令行提示符 ...
- 基于Ubuntu14.04系统的nvidia tesla K40驱动和cuda 7.5安装笔记
基于Ubuntu14.04系统的nvidia tesla K40驱动和cuda 7.5安装笔记 飞翔的蜘蛛人 注1:本人新手,文章中不准确的地方,欢迎批评指正 注2:知识储备应达到Linux入门级水平 ...
- sublime 安装笔记
sublime 安装笔记 下载地址 安装package control 根据版本复制相应的代码到console,运行 按要求重启几次后再按crtl+shift+p打开命令窗口 输入pcip即可开始安装 ...
- docker在ubuntu14.04下的安装笔记
本文主要是参考官网教程进行ubuntu14.04的安装. 下面是我的安装笔记. 笔记原件完整下载: 链接: https://pan.baidu.com/s/1dEPQ8mP 密码: gq2p
- ArchLinux 安装笔记：续 --zz
续前话 在虚拟机里调试了几天,终于鼓起勇气往实体机安装了,到桌面环境为止的安装过程可以看我的前一篇文章<ArchLinux 安装笔记>.桌面环境我使用的是 GNOME,虽然用了很长一段时间 ...
- Hadoop1.x与2.x安装笔记
Hadoop1.x与2.x安装笔记 Email: chujiaqiang229@163.com 2015-05-09 Hadoop 1.x 安装 Hadoop1.x 集群规划 No 名称 内容 备注 ...
随机推荐
- cximage功能简介
CxImage是一个可以用于MFC 的C++图像处理类库类,它可以打开,保存,显示,转换各种常见格式的图像文件,比如BMP, JPEG, GIF, PNG, TIFF, MNG, ICO, PCX, ...
- elementUI MessageBox弹框 <el-dialog>弹框如果出现input的type属性为password。项目中用到日期组件的地方会报错
ElementUI:项目中如果用到MessageBox弹框的输入框input且type为password,以及用到<el-dialog>里面用到input且type为password.此时 ...
- APP信息获取接口
https://itunes.apple.com/lookup?id=APPID&callback=2 http://myapp.com/cgi-bin/mapp/mapp_info?type ...
- ModelDriven机制及其运用
ModelDriven 为什么需要ModelDriven 所谓ModelDriven ,意思是直接把实体类当成页面数据的收集对象.比如,有实体类User 如下: package cn.com.lead ...
- rman参数
rman 参数 RMAN> show all; 参数是存放在控制文件中的 改参数:(直接改) eg: CONFIGURE RETENTION POLICY TO REDUNDANCY 3 参数: ...
- Linxu基本指令
一.Linux权限的概念 Linux下有两种用户:普通用户和超级用户(). 普通用户:在linux下做有限的事情: 超级用户:可以在linux系统下做任何事情,不受限制. 普通用户的提示符是“$”,超 ...
- Webpack的作用（一个基础的打包编译工具在做什么?）
结论: 转换ES6语法成ES5 处理模块加载依赖 生成一个可以在浏览器加载执行的 js 文件 第一个问题,转换语法,其实我们可以通过babel来做.核心步骤也就是: 通过babylon生成AST 通过 ...
- 《鸟哥的Linux私房菜》读书笔记--第0章 计算机概论 硬件部分
一个下午看了不少硬件层面的知识,看得太多太快容易忘记.于是在博客上写下读书笔记. 有关硬件 个人计算机架构&接口设备 主板芯片组为“南北桥”的统称,南北桥用于控制所有组件之间的通信. 北桥连接 ...
- 紫书 习题 8-25 UVa 11175 （结论证明）（配图）
看了这篇博客https://blog.csdn.net/u013520118/article/details/48032599 但是这篇里面没有写结论的证明, 我来证明一下. 首先结论是对于E图而言, ...
- 【转】 C# DEBUG 调试信息打印及输出详解
[转] C# DEBUG 调试信息打印及输出详解 1.debug只在[debug模式下才执行](运行按钮后面的下拉框可选) 2.debug提供了许多调试指令,如断言 System.D ...