CentOS7安装elk

一、安装Elasticsearch

1.1 安装Java

# 系统版本
 cat /etc/redhat-release
CentOS Linux release 7.5. (Core) 

  uname -r
3.10.-.el7.x86_64

  yum -y install java
  java -version
openjdk version "1.8.0_212"
OpenJDK Runtime Environment (build 1.8.0_212-b04)
OpenJDK -Bit Server VM (build 25.212-b04, mixed mode)

1.2 安装Elasticsearch

• 关闭防火墙

systemctl disable firewalld

• 关闭selinux

vim /etc/sysconfig/selinux
SELINUX= Enforcing # <== 修改为 disabled, 重启系统生效

• 官方文档

https://www.elastic.co/guide/en/elasticsearch/reference/6.5/rpm.html#install-rpm

• 下载并安装公共签名秘钥

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

• 从RPM存储库安装

在/etc/yum.repos.d/创建一个elasticsearch.repo文件，文件内容如下：

vim /etc/yum.repos.d/elasticsearch.repo

[elasticsearch-.x]
name=Elasticsearch repository for .x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=
autorefresh=
type=rpm-md

• yum安装ElasticSearch

yum install elasticsearch -y

• 启动Elasticsearch

systemctl daemon-reload
systemctl enable elasticsearch.service

• 启动/停止Elasticsearch

systemctl start elasticsearch.service
systemctl stop elasticsearch.service

• Elasticsearch服务运行状态

systemctl status elasticsearch.service

Elasticsearch日志存储在/var/log/elasticsearch/目录下

• 检查Elasticsearch是否正在运行

[root@localhost ~]# curl http://localhost:9200
{
  "name" : "O2pObfg",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "uXvFw1fgR1iTduPf1d-MAw",
  "version" : {
    "number" : "6.7.1",
    "build_flavor" : "default",
    "build_type" : "rpm",
    "build_hash" : "2f32220",
    "build_date" : "2019-04-02T15:59:27.961366Z",
    "build_snapshot" : false,
    "lucene_version" : "7.7.0",
    "minimum_wire_compatibility_version" : "5.6.0",
    "minimum_index_compatibility_version" : "5.0.0"
  },
  "tagline" : "You Know, for Search"
}

1.3 配置Elasticsearch

Elasticsearch默认使用/etc/elasticsearch运行时配置

默认Elasticsearch从文件/etc/elasticsearch/elastisearch.yml加载配置。

[root@localhost ~]# ll -lrht /etc/elasticsearch/
total 36K
-rw-rw----.  root elasticsearch     Apr   : users_roles
-rw-rw----.  root elasticsearch     Apr   : users
-rw-rw----.  root elasticsearch   Apr   : roles.yml
-rw-rw----.  root elasticsearch   Apr   : role_mapping.yml
-rw-rw----.  root elasticsearch  13K Apr   : log4j2.properties
-rw-rw----.  root elasticsearch .6K Apr   : jvm.options
-rw-rw----.  root elasticsearch .9K Apr   : elasticsearch.yml
-rw-rw----.  root elasticsearch   Apr  : elasticsearch.keystore

• 配置Elasticsearch数据路径和日志目录：

官方文档：

https://www.elastic.co/guide/en/elasticsearch/reference/6.5/settings.html

https://www.elastic.co/guide/en/elasticsearch/reference/6.5/important-settings.html

# 创建Elasticsearch数据目录及日志目录
mkdir /opt/elasticsearch/{data,log}/ -pv
cd /opt/
chown -R elasticsearch:elasticsearch elasticsearch/

cp /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml_`date +%Y%m%d_bak`

vim /etc/elasticsearch/elasticsearch.yml
# Elasticsearch 节点名称
node.name: node-
#数据
path.data: /opt/elasticsearch/data/
# 日志
path.logs: /opt/elasticsearch/log/

# 内存锁定,将进程地址空间锁定在RAM中，防止任何Elasticsearch内存被换出
bootstrap.memory_lock: true

# 监听地址
network.host: 192.168.198.130
# 监听端口
http.port: 

# 查看修改了那些配置 grep '^[a-z]' /etc/elasticsearch/elasticsearch.yml
node.name: node-
path.data: /opt/elasticsearch/data/
path.logs: /opt/elasticsearch/log/
bootstrap.memory_lock: true
network.host: 192.168.198.130
http.port: 

• 配置JVM

Jvm配置文件/etc/elasticsearch/jvm.options

-Xms2g
-Xmx2g

• 文件句柄配置

官方文档：https://www.elastic.co/guide/en/elasticsearch/reference/6.5/setting-system-settings.html

ulimit -n
tail - /etc/security/limits.conf 

elasticsearch  -  nofile  

• 系统配置

官方文档：https://www.elastic.co/guide/en/elasticsearch/reference/6.5/setting-system-settings.html

vim /usr/lib/systemd/system/elasticsearch.service

[Service]        # <== 在此标签下添加
LimitMEMLOCK=infinity

# 重新加载
systemctl daemon-reload

• 虚拟内存

官方文档：

https://www.elastic.co/guide/en/elasticsearch/reference/6.5/vm-max-map-count.html

临时配置：

sysctl -w vm.max_map_count=

永久配置：/etc/sysctl.conf

tail - /etc/sysctl.conf
vm.max_map_count=
sysctl -p

二、安装Logstash

• 官方文档

https://www.elastic.co/guide/en/logstash/6.5/installing-logstash.html

• 下载并安装公共签名和密钥

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

• 在/etc/yum.repos.d/目录下新建logstash.repo文件，文件内容如下：

[logstash-.x]
name=Elastic repository for .x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=
autorefresh=
type=rpm-md

• yum安装logstash

yum -y install logstash

三、安装Kibana

• 官方文档

https://www.elastic.co/guide/en/kibana/6.5/rpm.html

• 下载并安装公共签名和密钥

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

• 在/etc/yum.repos.d/目录下，创建kibana.repo，文件内容如下：

[kibana-.x]
name=Kibana repository for .x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=
autorefresh=
type=rpm-md

• yum 安装kibana

yum install kibana

• 启动/停止kibana

systemctl daemon-reload
systemctl enable kibana.service
systemctl start kibana.service
systemctl stop kibana.service

• 配置kibana

Kibana配置文件在/etc/kibana/目录下的kibana.yml。默认kibana运行localhost:5601

[root@localhost ~]# grep '^[a-z]' /etc/kibana/kibana.yml
server.port:
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://192.168.198.130:9200"]
kibana.index: ".kibana"

四、安装Filebeat

• 官方文档

https://www.elastic.co/guide/en/beats/filebeat/5.5/setup-repositories.html

https://www.elastic.co/guide/en/beats/filebeat/6.5/configuring-output.html

• 下载并安装公共签名和密钥

rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch

• 在/etc/yum.repos.d/目录下创建filebeat.repo文件，文件内容如下：

[elastic-.x]
name=Elastic repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=
autorefresh=
type=rpm-md

• yum安装filebeat

yum -y install filebeat

#  filebeat记录读取文件位偏移 /var/lib/filebeat/registry

五、安装Redis

参考：https://www.cnblogs.com/hwlong/p/9330191.html

https://www.cnblogs.com/hwlong/p/6101019.html

wget http://download.redis.io/releases/redis-3.2.12.tar.gz
上传至 /usr/local
tar xzf redis-3.2..tar.gz
mv redis-3.2. redis
cd redis
make
src/redis-server &

六、Filebeat+Redis+Logstash+Elasticsearch+Kibana

6.1 配置Filebeat

filebeat.prospectors:

- input_type: log
  paths:
    - /data/logs/localhost.localdomain.*.log
  exclude_files: [".gz$"]
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
  multiline.negate: true
  multiline.match: after
  tags: ["localhost-localdomain"]
  document_type: localhost-localdomain

- input_type: log
  paths:
    - /data/logs/localhost.localdomain-error.*.log
  exclude_files: [".gz$"]
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
  multiline.negate: true
  multiline.match: after
  tags: ["localhost-localdomain-error"]
  document_type: localhost-localdomain-error

fields:
  ipaddr: '192.168.198.131'   # <== filebeat本机IP
fields_under_root: true

output.redis:
  hosts: ["192.168.198.130"]        # <== redis地址
  port:
  key: "default_list"
  keys:
    - key: "%{[type]}"                # <== 根据document_type类型写入不同的key中
      mapping:
      "localhost-localdomain": "localhost-localdomain"
      "localhost-localdomain-error": "localhost-localdomain-error"

  db:
  datatype: list
  timeout: 

processors:
  - drop_fields:
       fields: ["offset","beat","source","input"]

• 启动filebeat

systemctl status filebeat

6.2 配置logstash.conf

input {

  redis {
     host => "192.168.198.130"
     port => ""
     db => ""
     data_type => "list"
     key => "default_list"
     type => "default_list"
  }

  redis {
     host => "192.168.198.130"
     port => ""
     db => ""
     data_type => "list"
     key => "localhost-localdomain"
     type => "localhost-localdomain"
  }

  redis {
     host => "192.168.198.130"
     port => ""
     db => ""
     data_type => "list"
     key => "localhost-localdomain-error"
     type => "localhost-localdomain-error"
  }

}

filter {

}

output{
   if [type] == "localhost-localdomain" {

       elasticsearch {
           document_type => "localhost-localdomain"
           hosts => ["192.168.198.130:9200"]
           index => "localhost-localdomain.log"
       }
   }

   if [type] == "localhost-localdomain-error" {

       elasticsearch {
           document_type => "localhost-localdomain-error"
           hosts => ["192.168.198.130:9200"]
           index => "localhost-localdomain-error.log"
       }
   }

}

• 启动logstash

/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-logstash-es.conf  &