Graylog+elasticsearch+mongodb集群+nginx负载均衡前端

网上有张图画的很好，搜索有关它的配置文章，google里有几篇英文的，都是依靠haproxy等或别的什么实现，没有纯粹的Graylog+elasticsearch+mongodb集群，项目需要，只有自己摸索一下了，搞好发现其实也很简单。

整个结构大概就是上图样子，nginx的负载均衡可以参考我前面的文章。将后端的web集群换成graylog集群地址就可以了。

下面是4台集群的配置效果图，完成后的效果是随便访问那个IP，都可以访问到所有的日志，节点可以根据需求任意添加，无论graylog，es,还是mongodb都可以。

配置过程如下，难点在于graylog的集群，安装软件的过程可以参考前面的安装过程。

========================================elasticsearch=================================

#vim /etc/elasticsearch/elasticsearch.yml

node.master: true    #两台机器设置成master，其余节点此项不写

network.bind_host: 192.168.1.231  #绑定实际的服务器IP地址，不是集群的IP

cluster.name: graylog        #与/etc/graylog/server/server.conf 中的elasticsearch_cluster_name 集群名称必须相同

discovery.zen.minimum_master_nodes: 2  #4台机器就2个够了

discovery.zen.ping.multicast.enabled: false

discovery.zen.ping.unicast.hosts: ["192.168.1.231:9300","192.168.1.232:9300","192.168.1.233:9300","192.168.1.234:9300"]

=================================================mongodb========================================

vim /etc/mongod.conf   #所有节点配置文件相同

replication:

replSetName: graylog

systemLog:

destination: file

logAppend: true

path: /var/log/mongodb/mongod.log

storage:

dbPath: /var/lib/mongo

journal:

enabled: true

processManagement:

fork: true  # fork and run in background

pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile

net:

port: 27017

#mongo//进入数据库配置

rs.initiate()

rs.add("192.168.1.231:27017");

rs.add("192.168.1.232:27017");

rs.addArb("192.168.1.233:27017");

rs.add("192.168.1.234:27017");

graylog:PRIMARY>  rs.config();   rs.config();

{

"_id" : "graylog",

"version" : 4,

"protocolVersion" : NumberLong(1),

"members" : [

{

"_id" : 0,

"host" : "node231:27017",

"arbiterOnly" : false,

"buildIndexes" : true,

"hidden" : false,

"priority" : 1,

"tags" : {

},

"slaveDelay" : NumberLong(0),

"votes" : 1

},

{

"_id" : 1,

"host" : "192.168.1.232:27017",

"arbiterOnly" : false,

"buildIndexes" : true,

"hidden" : false,

"priority" : 1,

"tags" : {

},

"slaveDelay" : NumberLong(0),

"votes" : 1

},

{

"_id" : 2,

"host" : "192.168.1.234:27017",

"arbiterOnly" : false,

"buildIndexes" : true,

"hidden" : false,

"priority" : 1,

"tags" : {

},

"slaveDelay" : NumberLong(0),

"votes" : 1

},

{

"_id" : 3,

"host" : "192.168.1.233:27017",

"arbiterOnly" : true,

"buildIndexes" : true,

"hidden" : false,

"priority" : 1,

"tags" : {

},

"slaveDelay" : NumberLong(0),

"votes" : 1

}

],

"settings" : {

"chainingAllowed" : true,

"heartbeatIntervalMillis" : 2000,

"heartbeatTimeoutSecs" : 10,

"electionTimeoutMillis" : 10000,

"getLastErrorModes" : {

},

"getLastErrorDefaults" : {

"w" : 1,

"wtimeout" : 0

},

"replicaSetId" : ObjectId("578c49834bd6994271fcfad7")

}

}

============================graylog===================================

#vim /etc/graylog/server/server.conf

password_secret =0b4e7a0e5fe84ad35fb5f95b9ceeac790b4e7a0e5fe84ad35fb5f95b9ceeac79

root_password_sha2 =ed02457b5c41d964dbd2f2a609d63fe1bb7528dbe55e1abf5b52c249cd735797

elasticsearch_cluster_name = graylog

elasticsearch_discovery_zen_ping_multicast_enabled = false

elasticsearch_discovery_zen_ping_unicast_hosts = 192.168.1.231:9300,192.168.1.232:9300,192.168.1.233:9300,192.168.1.234:9300

mongodb_uri = mongodb://192.168.1.231:27017,192.168.1.232:27017,192.168.1.233:27017,192.168.1.234:27017/graylog

elasticsearch_shards = 4

elasticsearch_replicas = 1

mongodb_useauth = false       #默认为false,可不必添加，生产环境必须

=====================测试=============================

使用curl命令操作elasticsearch

# curl -X GET 'http://192.168.1.231:9200/_cat/nodes'

192.168.1.234 192.168.1.234  7 13 0.06 d * Marrina

192.168.1.233 192.168.1.233  7 21 0.12 d m Banshee

192.168.1.231 192.168.1.231  8 23 0.43 d m Katie Power

192.168.1.232 192.168.1.232  7 22 0.15 d m Dark Phoenix

192.168.1.231 192.168.1.231 20 23 0.43 c - graylog-8562414f-0f9b-48d9-b0c6-9faab367832e

]# curl -X GET 'http://192.168.1.231:9200/_cluster/health?pretty=true'

{

"cluster_name" : "graylog",

"status" : "green",

"timed_out" : false,

"number_of_nodes" : 4,

"number_of_data_nodes" : 3,

"active_primary_shards" : 4,

"active_shards" : 4,

"relocating_shards" : 0,

"initializing_shards" : 0,

"unassigned_shards" : 0,

"delayed_unassigned_shards" : 0,

"number_of_pending_tasks" : 0,

"number_of_in_flight_fetch" : 0,

"task_max_waiting_in_queue_millis" : 0,

"active_shards_percent_as_number" : 100.0

}

mongodb状态查询

graylog:PRIMARY> rs.status()

{

"set" : "graylog",

"date" : ISODate("2016-07-18T09:14:33.419Z"),

"myState" : 1,

"term" : NumberLong(2),

"heartbeatIntervalMillis" : NumberLong(2000),

"members" : [

{

"_id" : 0,

"name" : "node231:27017",

"health" : 1,

"state" : 1,

"stateStr" : "PRIMARY",

"uptime" : 19532,

"optime" : {

"ts" : Timestamp(1468833273, 2),

"t" : NumberLong(2)

},

"optimeDate" : ISODate("2016-07-18T09:14:33Z"),

"electionTime" : Timestamp(1468813741, 1),

"electionDate" : ISODate("2016-07-18T03:49:01Z"),

"configVersion" : 5,

"self" : true

},

{

"_id" : 1,

"name" : "192.168.1.232:27017",

"health" : 1,

"state" : 2,

"stateStr" : "SECONDARY",

"uptime" : 8272,

"optime" : {

"ts" : Timestamp(1468833271, 4),

"t" : NumberLong(2)

},

"optimeDate" : ISODate("2016-07-18T09:14:31Z"),

"lastHeartbeat" : ISODate("2016-07-18T09:14:31.802Z"),

"lastHeartbeatRecv" : ISODate("2016-07-18T09:14:31.765Z"),

"pingMs" : NumberLong(0),

"syncingTo" : "node231:27017",

"configVersion" : 5

},

{

"_id" : 2,

"name" : "192.168.1.234:27017",

"health" : 1,

"state" : 2,

"stateStr" : "SECONDARY",

"uptime" : 8257,

"optime" : {

"ts" : Timestamp(1468833271, 4),

"t" : NumberLong(2)

},

"optimeDate" : ISODate("2016-07-18T09:14:31Z"),

"lastHeartbeat" : ISODate("2016-07-18T09:14:31.736Z"),

"lastHeartbeatRecv" : ISODate("2016-07-18T09:14:31.802Z"),

"pingMs" : NumberLong(0),

"syncingTo" : "192.168.1.232:27017",

"configVersion" : 5

},

{

"_id" : 3,

"name" : "192.168.1.233:27017",

"health" : 1,

"state" : 7,

"stateStr" : "ARBITER",

"uptime" : 8092,

"lastHeartbeat" : ISODate("2016-07-18T09:14:31.802Z"),

"lastHeartbeatRecv" : ISODate("2016-07-18T09:14:32.548Z"),

"pingMs" : NumberLong(0),

"configVersion" : 5

}

],

"ok" : 1

}

graylog:PRIMARY>