kle 日志收集系统维护之清理索引及索引优化脚本

logstash每天往es建好索引，按天生成，就目前的需求，需要清理不需要的数据，以保证最新日志的速度展示，哈哈，瞎搞了这个脚本，路过的大神批评。

#!/usr/bin/env python
# coding:utf8
#author: shantuwqk@163.com
import sys,datetime
import os,time,commands
HOST = '192.168.35.125'
PORT = '9200'
#排除不需要删除的索引列表
NotDEL = ['g1_sms_collect']
def Del_Indices(last4):
    TMP_LIST = []
    DEL_DAY_LIST = []
    file_info = '/tmp/es_indices_list.txt'
    CURLCMD = 'curl http://%s:%s/_cat/indices?v > %s'%(HOST,PORT,file_info)
    CURLINDEX_INFO = "cat  /tmp/es_indices_list.txt | grep -v 'health status index'|egrep '*-%s'|awk '{print $3}'"%last4
    #DEL_DAY_INFO = "%s|awk '{print $3}'"%CURLINDEX_INFO
    s,v = commands.getstatusoutput(CURLCMD)
    if s == 0:
        with open(file_info,'r') as f:
        st,vi = commands.getstatusoutput(CURLINDEX_INFO)
        TMP_LIST.append(vi)
       #DEL_DAY_LIST.append(vi)
    else:
       print "\033[31;1m curl cmd execute faild\033[0m"
   # print DEL_DAY_LIST
    for i in TMP_LIST[0].split('\n'):
        DEL_DAY_LIST.append(i)
        #sd,vd = commands.getstatusoutput('curl -XDELETE http://%s:%s/%s'%(HOST,PORT,i))
        #print sd,vd,
    print  DEL_DAY_LIST
    try:
    for t in NotDEL:
            DEL_DAY_LIST.remove("%s-%s"%(t,last4))
    except Exception:
    pass
    for index in DEL_DAY_LIST:
        sd,vd = commands.getstatusoutput('curl -XDELETE http://%s:%s/%s'%(HOST,PORT,index))
if __name__ == '__main__':
    last4day = (datetime.date.today() - datetime.timedelta(days=3)).strftime('%Y.%m.%d')
    #print last4day
    Del_Indices(last4day)

最近由于es shard 老出问题， 在网上看有优化的功能，所以又扩展了以下脚本：做日常定时任务，主要保含清理和优化索引功能

#!/usr/bin/env python
# coding:utf8
#author:shantuwqk@163.com
 
import sys,datetime
import os,time,commands
import sys,os
 
HOST = 'localhost'
PORT = '9200'
#ES_ROOT = os.path.dirname(os.path.dirname(__file__))
ES_ROOT = '/'.join(os.getcwd().split('/')[:-1])
ES_BIN = ES_ROOT + '/bin'
ES_INDICES = ES_ROOT + '/data/elasticsearch/nodes/0/indices'
ES_OPTIMIZE = "_optimize"
ES_DEL_VAL = "_optimize?only_expunge_deletes=true"
print ES_ROOT
print ES_BIN
print ES_INDICES
 
def Indiceslist(date):
    indiceslist = []
    s,v = commands.getstatusoutput("ls %s -l| awk '{print $9}'|egrep \"*-%s\""%(ES_INDICES,date))
    #print "#########indices",s,v
    if s == 0:
        for i in v.split('\n'):
            indiceslist.append(i)
    return indiceslist
    #print indiceslist
 
def Del_Indices(date):
    NotDEL = ['g1_sms_collect']
    DEL_DAY_LIST = []
    print date,
    print Indiceslist(date)
    for i in Indiceslist(date):
        print i,
        DEL_DAY_LIST.append(i)
    try:
        for t in NotDEL:
                DEL_DAY_LIST.remove("%s-%s"%(t,date))
    except Exception:
        pass
    print DEL_DAY_LIST
    for index in DEL_DAY_LIST:
        sd,vd = commands.getstatusoutput('curl -XDELETE http://%s:%s/%s'%(HOST,PORT,index))
        if sd == 0:
            print "DEL indices Sucess !!"
        else:
            print "DEL indices Failed  !!"
    ds,dv = commands.getstatusoutput('curl -XPOST http://%s:%s/%s/%s'%(HOST,PORT,index,ES_DEL_VAL))
    if ds == 0:
        print "Cel indices Sucess !!"
    else:
        print "Cel indices Failed !!"
 
def es_optimize(date):
    #print Indiceslist(date)
    for i in Indiceslist(date):
        print "curl -XPOST \"http://%s:%s/%s/%s\""%(HOST,PORT,i,ES_OPTIMIZE)
        s,v = commands.getstatusoutput("curl -XPOST \"http://%s:%s/%s/%s\""%(HOST,PORT,i,ES_OPTIMIZE))
        print s,v,
        if s == 0:
            print "\033[32;1mES Optimize Success !!!\033[0m"
        else:
            print "\033[31;1m Es Optimze Failed!!!\033[0m"
 
def handle_cluster_status():
    clustet_status = "curl -XGET \'http://localhost:9200/_cluster/health?pretty=true\'"
 
def help_prompt():
    print """
This program prints files to the standard output.
Options include:
--version : Prints the version number
--help    : Helpful tips
--task    : To operate on missions [delindices|optimze]
sample    : python es_manage.py --task delindices
                : python es_manage.py --task optimze
"""
 
def error_prompt():
    print '\033[31;1mUnknown option.Please refer to the help...\033[0m'
    print "==============================================================="
    help_prompt()
    print "---------------------------------------------------------------"
    sys.exit(1)
 
if __name__ == '__main__':
 
    if len(sys.argv) == 1:
        print "\033[31;1m No action specified.\033[0m"
        help_prompt()
        sys.exit(1)
    if len(sys.argv) == 2 and sys.argv[1].startswith('--'):
        if sys.argv[1][2:] == 'version':
            print 'Version 1.0'
            sys.exit(1)
        elif sys.argv[1][2:] == 'help':
            help_prompt()
    if len(sys.argv) == 3 and sys.argv[1][2:] == 'task' and sys.argv[2] == "delindices":
        DelDay = (datetime.date.today() - datetime.timedelta(days=3)).strftime('%Y.%m.%d')
        Del_Indices(DelDay)
    elif len(sys.argv) == 3 and sys.argv[1][2:] == 'task' and sys.argv[2] == "optimze":
        yesterday = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y.%m.%d')
        today = (datetime.date.today() - datetime.timedelta(days=0)).strftime('%Y.%m.%d')
        print es_optimize(yesterday)
        es_optimize(today)
    else:
        error_prompt()
        exit(1)