email.py

import os
import argparse
import yaml
import smtplib
import csv
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from urllib import unquote
import logging
import logging.handlers

logger = logging.getLogger()
error_list = []
all_error_file_list=[]

def extension(filename):
    """
    get file extension
    :param filename:
    :return:
    """
    ext = os.path.splitext(filename)[1]
    if ext.startswith('.'):
        # os.path.splitext retains . separator
        ext = ext[1:]
    return ext

def list_file(file_name, list):
    """
    get all failed file list ,contains sub folder
    :param file_name: file name or folder name
    :param list:
    :return:
    """
    if os.path.isdir(file_name):
        for s in os.listdir(file_name):
            list_file(os.path.join(file_name, s), list)
    else:
        ext = extension(file_name)
        if ext == "failed":
            list.append(file_name)

def read_email_record(faild_path):
    """
    read last failed file record
    :param faild_path:
    :return:
    """
    record = os.path.join(faild_path, "email_record.txt")
    if not os.path.isfile(record):
        return {}
    try:
        return yaml.load(open(record))
    except Exception as err:
        return {}

def rewrite_record(args):
    """
    save record file
    :param args:
    :return:
    """
    if not error_list:
        return
    record = os.path.join(args.failed_path, "email_record.txt")
    map = {}
    for s in all_error_file_list:
        map[s] = "emailed"
    f = open(record, "w")
    yaml.dump(map, f, default_flow_style=False)
    f.close()

def process(args):
    """
    process :
        * list all failed files
        * read record list
        * remove duplicate data
    :param args:
    :return:
    """
    faild_path = args.failed_path
    if not os.path.isdir(faild_path):
        logger.error("failed path:%s is not exist, exist", faild_path)
        return
    list_file(faild_path, all_error_file_list)
    record_map = read_email_record(faild_path)
    if not record_map:
        error_list.extend(all_error_file_list)
    else:
        for file in all_error_file_list:
            if not record_map.has_key(file):
                error_list.append(file)

def generate_body(s3path):
    """
    generate email body
    :return:
    """
    body = """<h4>The following billing csv files are format failed,please check</h4>
            """
    logger.info("failed file list as below:")
    # list all failed list which need send email
    for s in error_list:
        # define line number from 0
        line_num = 0
        # to get error file name and path , just support {orgid}/{failed_file} format
        ps = os.path.split(s)
        # to regenerate new web path, such as s3://{bucket}/xxx/{orgid}/{fail_file}
        s3 = os.path.join(s3path, os.path.split(ps[0])[1], ps[1])
        # generate html body
        body += "<hr>"
        # add file name with link
        body += """<div>File: <a href="%s">%s<a></div>""" % (s3, ps[1])
        # add URL which can copy
        body += "<div>URL: %s</div>" % (s3)
        # add logger info
        logger.info("-"*50)
        logger.info("file:[%s]",s)
        # lines which read each time
        total_rows_each_read=10000000
        # open failed file
        with open(s, 'r') as f:
            # read file in loop
            while 1:
                # get lines array list
                lines = f.readlines(total_rows_each_read)
                if not lines:
                    break
                # check each line with csv , when failed ,
                # generate new array from failed index and re-check until finished
                while len(lines)>0:
                    # check csv format with strict = True
                    reader=csv.reader(lines, strict=True)
                    # define array index
                    i = 0
                    try:
                        # get rows of each line
                        for row in reader:
                            # when row is csv format , array index will ++
                            i += 1
                            line_num += 1
                            for item in row:
                                # check each column whether with \n , if yes ,will add line number
                                if str(item).find('\n')>0:
                                    i += 1
                                    line_num += 1
                        # when all lines is correct , array will set to empty to break loop
                        lines=[]
                    # get Exceptions
                    except Exception as err:
                        # when error occur , line number will increase 1
                        line_num += 1
                        #line_num += reader.line_num
                        line = lines[i]
                        # generate each error line information
                        body += "<div></div><br><br>"
                        body += "<li><div>Error line number: [#%s] </div></li>" \
                                "<div>#Exceptions: [%s]</div>" % (line_num, err)
                        body += """<div>#Error row data as below:</div>
                                   <div>%s</div>""" % line
                        if (i+1) == len(lines):
                            lines=[]
                        else:
                            tmp=lines[i+1:]
                            lines = tmp
    body += "<hr>"
    return body

def email_sender(args):
    """
    send email ,iterator each receiver then send email
    :param args:
    :return:
    """
    subject = "Error Pipeline CSV File List"
    #if not error_list:
    #    return False
    #body = generate_body(args.s3url)
    body = "test"
    emails = args.email
    smtp_host = args.smtpHost
    smtp_port = args.smtpPort
    username = args.username
    password = args.password
    sent_from = args.sentFrom

    if emails:
        mail_server = smtplib.SMTP()
        try:
            msg = MIMEMultipart()
            msg['From'] = sent_from
            msg['Subject'] = subject
            msg.attach(MIMEText(body, 'html'))
            mail_server.connect(smtp_host, smtp_port)
            mail_server.ehlo()
            mail_server.starttls()
            mail_server.ehlo()
            if password and smtp_host != 'eng-smtp.calix.local':
                mail_server.login(username, unquote(password))
            for recipient in emails:
                logger.info("send email to %s", recipient)
                msg['To'] = recipient
                mail_server.sendmail(sent_from, recipient, msg.as_string())
        except Exception as err:
            logger.error("send email failed:%s", err)
            return False
        finally:
            if mail_server:
                mail_server.close()
        return True
    return False

def init_log(log_file):
    """
    init logger
    :param log_file:
    :return:
    """
    log_path = os.path.split(log_file)[0]
    if not os.path.isdir(log_path):
        os.makedirs(log_path)
    logger.setLevel(logging.DEBUG)
    handler = logging.handlers.RotatingFileHandler(log_file,
                                                   maxBytes=20 * 1024 * 1024,
                                                   backupCount=5,
                                                   )
    fmt = '%(asctime)s-[%(filename)s:%(lineno)s]-[%(levelname)s]- %(message)s'
    formatter = logging.Formatter(fmt)
    handler.setFormatter(formatter)
    logger.addHandler(handler)

def main():
    parser = argparse.ArgumentParser(description=__doc__,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('--failed_path', default='/var/tmp/billing/', help='store failed files')
    parser.add_argument('-e', '--email', action='append',
                        help="To send multiple emails --email <email-address-1> --email <email-address-2> ...")
    parser.add_argument('--smtpHost', type=str, help="Host of SMTP server", required=False,
                        default="outlook.office365.com")
    parser.add_argument('--smtpPort', type=int, help="Port of SMTP server", required=False, default=587)
    parser.add_argument('--username', type=str, help="outlook username", required=False)
    parser.add_argument('--password', type=str, help="outlook password", required=False)
    parser.add_argument('--sentFrom', type=str, help="outlook email", required=False,
                        default="noreply-compass-fa@calix.com")
    parser.add_argument('--logfile', help='logger file name', required=False,
                        default='/var/log/sxadp-api-server/scan_failed_email.log')
    parser.add_argument('--s3url',type=str,help="billing s3 URL ",required=False)
    args = parser.parse_args()
    init_log(args.logfile)
    #process(args)
    if email_sender(args):
        print "done"
        #rewrite_record(args)

if __name__ == '__main__':
    main()