LOG收集系统(一)：原日志至收集

Date: 20140207
Auth: Jin

设置一个LOG收集系统
1. 收集原生（不解析，不压缩）的业务日志和WEB日志（NGINX,PHP）
2. 提供给开发，测试直接阅读和下载

需求分析
原生日志，所以不需要其他程序介入，需要收集和阅读下载，具体分析开发测试人员拿到日子自行处理
1、收集可以通过FTP采集
2、阅读和下载可以通过WEB形式
3、收集周期可以通过crontab控制脚本实现, 初步设置为每小时
4、核心是收集日子的脚本，考虑到扩展和重复使用，我使用python编写d

步骤
一、搭建FTP服务器

1.install

yum install pure-ftpd

2.config

mkdir /data/ftproot/logs

chown webroot.webroot /data/ftproot/logs

vim /etc/pure-ftpd/pure-ftpd.conf

注意流量参数

Bind                      10.0.0.221,21
ChrootEveryone              yes
BrokenClientsCompatibility  yes
Daemonize                   yes
MaxClientsPerIP             20
VerboseLog                  yes
DisplayDotFiles             no
AnonymousOnly               no
NoAnonymous                 yes
SyslogFacility              none
DontResolve                 yes
MaxIdleTime                 15
LimitRecursion              10000 8
AnonymousCanCreateDirs      no
MaxLoad                     4
PassivePortRange          45000 50000
#AnonymousRatio                1 10
#UserRatio                 1 10
AntiWarez                   yes
#AnonymousBandwidth            200
UserBandwidth               8
Umask                       133:02
MinUID                      100
AllowUserFXP                no
AllowAnonymousFXP           no
ProhibitDotFilesWrite       no
ProhibitDotFilesRead        no
AutoRename                  yes
AnonymousCantUpload         yes
AltLog                     clf:/var/log/pureftpd.log
PureDB                     /etc/pure-ftpd/pureftpd.pdb
MaxDiskUsage               99
CreateHomeDir              no
CustomerProof              yes

3.start ftp seriver

/etc/init.d/pure-ftpd restart
Stopping pure-ftpd: [ OK ]
Starting pure-ftpd: [ OK ]

chkconfig pure-ftpd on

4.add account

# pure-pw useradd logsftp -u 9999 -g 9999 -d /data/ftproot/logs
Password:
Enter it again:

# pure-pw mkdb
# /etc/init.d/pure-ftpd restart
Stopping pure-ftpd: [ OK ]
Starting pure-ftpd: [ OK ]

5.test

on opensuse desktop test

# zypper install lftp

lftp 10.0.0.221
lftp 10.0.0.221:~> user logsftp
Password:
lftp logsftp@10.0.0.221:~> ls
ls: Login failed: 530 ��֤ʧ�ܣ���Ǹ
lftp logsftp@10.0.0.221:~>

二、搭建WEB服务器
DOCMENTROOT为

server {
        listen       80;
        server_name  logs.test.com;
        root   /data/ftproot/logs/;

        location / {
                index  index.htm index.html;
                autoindex on;
                autoindex_localtime on;
                auth_basic            "logs access";
                auth_basic_user_file   log-auth.conf;

                error_log off;
                access_log /var/log/nginx/log.test.com-access.log;
                #access_log off;
        }

}

注意点：

autoindex_localtime on;
参考：http://blog.chinaunix.net/uid-26719405-id-3508444.html
autoindex_localtime on;
默认为off，显示的文件时间为GMT时间。
改为on后，显示的文件时间为文件的服务器时间

三、代码编写

无LOG版本

加入计划任务

5 * * * * /home/gbin/logcollect.py >> /home/gbin/logcollect.log 2>&1

计划任务报错 登录用户问题

删除userName = os.getlogin()

• 配置文件logcollect.ini

[global]
ip = 127.0.0.1
username = logsftp
password = pwd
mode = 0

[applogs]
dms-log = /home/dms/logs/service.log
store-logdir = /home/store/logs/
stock-logdir = /home/stock/logs/

[weblogs]
nginxlog = /var/log/nginx.log
phplog = /var/log/php-fpm/php-fpm.log

• 函数和对象：gbopt.py

  #!/usr/bin/env python
  # coding=utf8
  '''
  Created on 2014-03-05                                                                                                                                                                           
  
  @author: Jin                                                                                                                                                                                    
  
  '''                                                                                                                                                                                             
  
  __version__ = "0.0.1#date:2014-03-05"                                                                                                                                                           
  
  __all__ = ['readConfig',
             'ftpPutGb',
             'OptFile'
          ]                                                                                                                                                                                       
  
  import os
  import sys
  import ConfigParser
  import ftplib
  import time
  import socket
  from ftplib import FTP
  from time import strftime as printtime
  from time import sleep as wait
  
  progName = 'logcollect'
  userName = os.getlogin()
  hostName = socket.gethostname()
  pwdDir   = os.getcwd()
  workDir  = os.path.dirname(sys.argv[0])
  
  configFile = progName+'.ini'
  todayDate  = time.strftime("%Y%m%d", time.localtime())
  
  if workDir ==  '' or workDir == '.':
     progDir = pwdDir
  elif workDir.startswith('/'):
     progDir = workDir
  elif workDir.startswith('./'):
     progDir = pwdDir+workDir.lstrip('.')
  else:
     progDir = pwdDir+'/'+workDir
  
  confPath  = progDir+'/'+configFile
  logPath   = progDir+'/'+progName+'.log'
  
  def readConfig(filename='',section=''):
      '''read ini config return dict'''
      Call = 'Call '+sys._getframe().f_code.co_name
      if not os.path.isfile(filename):
          print "%s: Error, File %s is not exists,Please check it!" % (Call,filename)
          sys.exit(10)
      else:
          try:
              Config = ConfigParser.ConfigParser()
              Config.read(filename)
              sections = Config.sections()
              configDict = {}
              for conf in Config.items(section):
                  ckey   = conf[0]
                  cvalue = conf[1]
                  configDict[ckey] = cvalue
          except ConfigParser.ParsingError,e:
              print "%s: Read<%s> Section<%s> Parsing Error with reason<%s>!" % (Call,filename,section,''.join( repr(e).split('\n')))
              sys.exit(11)
          except Exception, e:
              print "%s: Read<%s> Section<%s> Exception error with reason<%s>!" % (Call,filename,section,''.join( repr(e).split('\n')))
              sys.exit(12)
          else:
              return configDict
  
  def ftpPutGb(filename,inittime,server=None,username=None,password=None,localdir=None,remotedir='testdir',debuglevel=0,model=1,retrytime=10,timeout=120):
      """Put file to FTP server,use local path
      exitcode introduce:
      timeout | 99
      connect failed | 21
      user or passwd error | 22
      create dir fails | 23
      change dir fails | 24
      read file fails | 25
      transfer file fails | 26
      any ftp transfer fails | 27
      server or username or password not exist | 28
      call ftpPutGb('/home/jin/code/python/tt.ini','127.0.0.1','logsftp','passwd',remotedir='testdir1/testdir2/testdir3',model=0)"""
      Call = 'Call '+sys._getframe().f_code.co_name
      if not os.path.isfile(filename):
          print "Notice: %s is not exists!" % filename
          return False
      if not localdir:
          localdir=os.path.dirname(filename)
          filename=os.path.basename(filename)
      uploadfile='STOR '+filename
      ftp=FTP()
      bufsize=1024
      os.chdir(localdir)
      ftp.set_debuglevel(debuglevel)
      remotedirList=remotedir.split('/')
      appName=remotedirList[0]
      machineName=remotedirList[1]
      dayName=remotedirList[2]
      if server and username and password:
          print printtime('%Y-%m-%d %H:%M:%S'),"Notice: FTPuser(%s) Put %s to FTP(%s:/%s) Begin." % (username,filename,server,remotedir)
          try:
              ftp.connect(server)
          except socket.error:
              waittime=inittime
              wait(retrytime)
              waittime+=retrytime
              if waittime==timeout:
                  print "%s: Put %s to FTP(%s) timeout(%s),exit!" % (Call,filename,server,timeout)
                  sys.exit(20)
              print "Notice: Retry Put %s to FTP(%s) again after %s s" % (filename,server,retrytime)
              ftpPutGb(filename,waittime,server,username,password,localdir,remotedir,debuglevel,model,retrytime,timeout)
          except ftplib.error_perm,e:
              print "Error: Connect to FTP(%s) failed with<%s>" % (server,e)
              sys.exit(21)
          else:
              try:
                  ftp.login(username,password)
                  ftp.set_pasv(model)
              except ftplib.error_perm,e:
                  print "Error: User(%s) Login FTP(%s) failed with<%s>" % (username,server,e)
                  sys.exit(22)
              else:
                  try:
                      try:
                          ftp.mkd(appName)
                      except ftplib.error_perm,e:
                          pass
                      finally:
                          ftp.cwd(appName)
                          try:
                              ftp.mkd(machineName)
                          except ftplib.error_perm,e:
                              pass
                          finally:
                              ftp.cwd(machineName)
                              try:
                                  ftp.mkd(dayName)
                              except ftplib.error_perm,e:
                                  pass
                              finally:
                                  ftp.cwd(dayName)
                                  try:
                                      file_handler = open(filename,'rb')
                                  except ftplib.error_perm,e:
                                      print "Error: User(%s) on FTP(%s) Open file(%s) failed with %s!" % (username,server,filename,e)
                                      sys.exit(25)
                                  else:
                                      try:
                                          ftp.storbinary(uploadfile,file_handler,bufsize)
                                      except ftplib.error_perm,e:
                                          print "Error: User(%s) on FTP(%s) storbinary file(%s) failed with %s!" % (username,server,filename,e)
                                          sys.exit(26)
                  except Exception, e:
                      print  "Error: User(%s) Put file(%s) to FTP(%s:%s) failed exit with %s!" % (username,filename,server,remotedir,e)
                      sys.exit(27)
                  else:
                      print printtime('%Y-%m-%d %H:%M:%S'),"Notice: FTPuser(%s) Put %s to FTP(%s:/%s) successful." % (username,filename,server,remotedir)
                      return True
          finally:
              ftp.set_debuglevel(0)
              ftp.close()
      else:
          print "Error: server(%s) or username(%s) or password(%s) is not exist!" % (server,username,password)
          sys.exit(28)
  
  class OptFile(file):
      '''log file object'''
      def __init__(self,path,remotedir):
          self.filePath = path
          self.remotedir = remotedir
          globalDict = readConfig(confPath,'global')
          self.ftpIp = globalDict['ip']
          self.ftpUser = globalDict['username']
          self.ftpPass = globalDict['password']
          self.ftpMode = globalDict['mode']
          file.__init__
      def put(self):
          ftpPutGb(self.filePath,0,self.ftpIp,self.ftpUser,self.ftpPass,remotedir=self.remotedir,model=self.ftpMode)
  
  def main():
      pass
  if __name__ == '__main__':
      main()

• 调用脚本：logcollect.py

#!/usr/bin/env python
# coding=utf8
'''
Created on 2014-03-05

@author: Jin

'''
import os
import sys
import gbopt

def collectLogs(logtype):
    applogsDict=gbopt.readConfig(gbopt.confPath,logtype)
    for l in applogsDict:
    appName=l.split('-')[0]
    hostName=gbopt.hostName
    dayName=gbopt.todayDate
    remoteDir=appName+'/'+hostName+'/'+dayName
        if l.endswith('dir'):
            locallogPath=applogsDict[l]+gbopt.todayDate+'.log'
    else:
            locallogPath=applogsDict[l]
    logfile=gbopt.OptFile(locallogPath,remoteDir)
    logfile.put()

def main():
    for i in ['applogs','weblogs']:
    collectLogs(i)

if __name__ == '__main__':
    main()