python单线程，多线程和协程速度对比

在某些应用场景下，想要提高python的并发能力，可以使用多线程，或者协程。比如网络爬虫，数据库操作等一些IO密集型的操作。下面对比python单线程，多线程和协程在网络爬虫场景下的速度。

一，单线程。

　　单线程代

 1 #!/usr/bin/env 
 2 # coding:utf8 
 3 # Author: hz_oracle

 import MySQLdb
 import gevent
 import requests
 import time

 class DbHandler(object):
     def __init__(self, host, port, user, pwd, dbname):
         self.host = host
         self.port = port
         self.user = user
         self.pwd = pwd
         self.db = dbname

     def db_conn(self):
         try:
             self.conn = MySQLdb.connect(host=self.host, port=self.port, user=self.user, passwd=self.pwd, db=self.db, charset="utf8")
             self.cursor = self.conn.cursor()
             return 1
         except Exception as e:
             return 0

     def get_urls(self, limitation):
         sql = """select pic  from  picurltable limit  %s""" % limitation
         urls_list = list()
         try:
             self.cursor.execute(sql)
             fetchresult = self.cursor.fetchall()
             for line in fetchresult:
                 urls_list.append(line[0])
             print len(urls_list)
         except Exception as e:
             print u"数据库查询失败:%s"  % e
             return []
         return urls_list

     def db_close(self):
         self.conn.close()

 def get_pic(url):
     try:
         pic_obj = requests.get(url).content
     except Exception as e:
         print u"图片出错"
         return ""
     filename = url.split('/')[-2]
     file_path = "./picture/" + filename + '.jpg'
     fp = file(file_path, 'wb')
     fp.write(pic_obj)
     fp.close()
     return "ok"

 def main():
     start_time = time.time()
     db_obj = DbHandler(host='127.0.0.1', port=3306, user='root', pwd='123456', dbname='pic')
     db_obj.db_conn()
     url_list = db_obj.get_urls(100)
     map(get_pic, url_list)
     #for url in url_list:
     #    get_pic(url)
     end_time = time.time()
     costtime = float(end_time) - float(start_time)
     print costtime
     print "download END"

 if __name__ == "__main__":
     main()

　　运行结果

100
45.1282339096
download END

单线程情况下，下载100张图片花了45秒。

再来看多线程的情况下。

#!/usr/bin/env python
# coding:utf8
# Author: hz_oracle

import MySQLdb
import gevent
import requests
import time
import threading
import Queue

lock1 = threading.RLock()
url_queue = Queue.Queue()
urls_list = list()

class DbHandler(object):
    def __init__(self, host, port, user, pwd, dbname):
        self.host = host
        self.port = port
        self.user = user
        self.pwd = pwd
        self.db = dbname

    def db_conn(self):
        try:
            self.conn = MySQLdb.connect(host=self.host, port=self.port, user=self.user, passwd=self.pwd, db=self.db, charset="utf8")
            self.cursor = self.conn.cursor()
            return 1
        except Exception as e:
            return 0

    def get_urls(self, limitation):
        sql = """select pic  from  picurltable limit  %s""" % limitation
        try:
            self.cursor.execute(sql)
            fetchresult = self.cursor.fetchall()
            for line in fetchresult:
                url_queue.put(line[0])
        except Exception as e:
            print u"数据库查询失败:%s"  % e
            return 0
        return 1

    def db_close(self):
        self.conn.close()

class MyThread(threading.Thread):
    def __init__(self):
        super(MyThread, self).__init__()

    def run(self):
        url = url_queue.get()
        try:
            pic_obj = requests.get(url).content
        except Exception as e:
            print u"图片出错"
            return ""
        filename = url.split('/')[-2]
        file_path = "./picture/" + filename + '.jpg'
        fp = file(file_path, 'wb')
        fp.write(pic_obj)
        fp.close()

def main():
    start_time = time.time()
    db_obj = DbHandler(host='127.0.0.1', port=3306, user='root', pwd='', dbname='pic')
    db_obj.db_conn()
    db_obj.get_urls(100)
    for i in range(100):
        i = MyThread()
        i.start()
    while True:
        if threading.active_count()<=1:
            break
    end_time = time.time()
    costtime = float(end_time) - float(start_time)
    print costtime
    print "download END"

if __name__ == "__main__":
    main()

运行结果

15.408192873
download END

启用100个线程发现只要花15秒即可完成任务，100个线程可能不是最优的方案，但较单线程有很明显的提升。接着再来看协程。

协程代码

#!/usr/bin/env python
# coding:utf8
# Author: hz_oracle

import MySQLdb
import requests
import time
import threading
import Queue

from gevent import monkey; monkey.patch_all()
import gevent

class DbHandler(object):
    def __init__(self, host, port, user, pwd, dbname):
        self.host = host
        self.port = port
        self.user = user
        self.pwd = pwd
        self.db = dbname

    def db_conn(self):
        try:
            self.conn = MySQLdb.connect(host=self.host, port=self.port, user=self.user, passwd=self.pwd, db=self.db, charset="utf8")
            self.cursor = self.conn.cursor()
            return 1
        except Exception as e:
            return 0

    def get_urls(self, limitation):
        urls_list = list()
        sql = """select pic  from  picurltable limit  %s""" % limitation
        try:
            self.cursor.execute(sql)
            fetchresult = self.cursor.fetchall()
            for line in fetchresult:
                urls_list.append(line[0])
        except Exception as e:
            print u"数据库查询失败:%s"  % e
            return []
        return urls_list

    def db_close(self):
        self.conn.close()

def get_pic(url):
    try:
        pic_obj = requests.get(url).content
    except Exception as e:
        print u"图片出错"
        return ""
    filename = url.split('/')[-2]
    file_path = "./picture/" + filename + '.jpg'
    fp = file(file_path, 'wb')
    fp.write(pic_obj)
    fp.close()
    return "ok"

def main():
    start_time = time.time()
    db_obj = DbHandler(host='127.0.0.1', port=3306, user='root', pwd='123456', dbname='pic')
    db_obj.db_conn()
    url_list = db_obj.get_urls(100)
    gevent.joinall([gevent.spawn(get_pic,url) for url in url_list])

    end_time = time.time()
    costtime = float(end_time) - float(start_time)
    print costtime
    print "download END"

if __name__ == "__main__":
    main()

运行结果

10.6234440804
download END

使用协程发现只花了10秒多，也就是三种方法中最快的。

总结：

三种方法中，单线程最慢，多线程次之，而协程最快。 不过如果对多线程进行优化，也可能变快，这里不讨论。