python多线程批量下载远程图片

python多线程使用场景：多线程采集, 以及性能测试等 。

数据库驱动类-简单封装下

mysqlDriver.py

#!/usr/bin/python3
#-*- coding: utf-8 -*-
# author:zhouchao
# mysql 驱动模型类
import pymysql;
import traceback; 
 
class mysqlDriver:
    host="localhost";
    user="root";
    password="";
    database="fitcmoe_boke";
    charset="GBK";
    db = "false";
 
    # 连接数据库
    def connect(self, host="", user="",password="",database="",charset = ""):
        if host and user :
            self.host = host;
            self.user = user;
            self.password = password;
            self.database = database;
            self.charset = charset;
        self.db=pymysql.connect(host=self.host,user=self.user,password=self.password,database=self.database,charset=self.charset);
 
    # 查询多条
    def select(self,sql,params = []):
        if params :
            sql = self.bindParams(sql, params);
        # print(sql);return;
        cursor = self.db.cursor();
        cursor.execute(sql);
        data = cursor.fetchall();
        self.db.close();
        return data;
 
    # 查询单条
    def find(self,sql,params = []):
        if params :
            sql = self.bindParams(sql, params);
        cursor = self.db.cursor();
        cursor.execute(sql);
        data = cursor.fetchone();
        self.db.close();
        return data;
 
    # 新增
    def add(self,sql,params=[]):
        if params :
            sql = self.bindParams(sql, params);
        try:
            cursor = self.db.cursor();
            cursor.execute(sql);
            newId = self.db.insert_id();
            self.db.commit();
            self.db.close();
            return newId;
        except "involid level":
           # 发生错误时回滚
           print("发生异常","involid level");
           self.db.rollback();
           self.db.close();
 
    # 更新
    def save(self,sql,params=[]):
        if params :
            sql = self.bindParams(sql, params);
        try:
            cursor = self.db.cursor();
            resultNum=cursor.execute(sql);
            self.db.commit();
            self.db.close();
            return resultNum;
        except "involid level":
           # 发生错误时回滚
           print("发生异常","involid level");
           self.db.rollback();
           self.db.close();
 
    def delete(self,sql,params=[]):
        if params :
            sql = self.bindParams(sql, params);
            # self.file_put_contents('rt.txt',sql,'append');
            # print(sql);return;
        try:
            cursor = self.db.cursor();
            resultNum=cursor.execute(sql);
            self.db.commit();
            self.db.close();
            return resultNum;
        except "involid level":
           # 发生错误时回滚
           print("发生异常","involid level");
           self.db.rollback();
           self.db.close();
 
    # 参数绑定  空格好奇怪啊
    def bindParams(self,sql, params):
        finalSql = "";
        length = len(sql);
        replaceNum = 0;
        for x in sql:
            if x == "?":
               x = params[replaceNum];
               typeX = type(x);
               if typeX.__name__ == "list":
                    temp = '(';
                    for t in x:
                        temp +=  "'" +str(t)+"',";
                    x=temp.rstrip(",");
                    x+=")";
               else :
                    if str(x).isdigit() :
                        x = int(x);
                    else:
                        x = "'" +str(x) +"'";
 
               replaceNum = replaceNum + 1;
            finalSql += str(x);
            # finalSql += x;
        return finalSql;
 
    # 将sql 写入本地文件
    # r+ = pappend 向前追加
    # w+ = 覆盖添加
    # a+ = append  向后追加
    def file_put_contents(self,fileName,content,modeName =''):
        if(modeName == 'pappend'):
          mode = "r+";
        elif(modeName == 'append'):
         mode = "a+";
        else:
         mode = "w+";
 
        fileObject = open(fileName,mode);
        fileObject.write(content);
        fileObject.close();
 
# 使用说明 异常处理类问题？？
# ? 代表参数
# ? 可以是列表或数字  如  [1,3,5,7]  和 3
# # 参数位置是按照顺序的 
 
# 简单使用
# mysqlD = mysqlDriver();
# mysqlD.connect();
# data = mysqlD.select("select from t_article where id in ? limit ? ",[[33,34,35,38],3]);
# data = mysqlD.delete("delete from t_article where aid in ?",[[43,44,270,280]]);

文件下载函数

file_get_contents.py

#!/usr/bin/python3
#-*- coding: utf-8 -*-
# author:zhouchao
# 文件处理函数
 
def file_get_contents(path,mode="r+"):
    fileObject = open(path,mode);
    content = fileObject.read();
    fileObject.close();
    return content;
 
# r+ = pappend 向前追加
# w+ = 覆盖添加
# a+ = append  向后追加
# wb+ 直接使用二进制添加
def file_put_contents(fileName,content,modeName =''):  
 
    if(modeName == 'pappend'):
      mode = "r+";
    elif(modeName == 'append'):
     mode = "a+";
    elif(modeName == 'wb+'):
     mode = "wb+";
    else:
     mode = "w+";
 
    fileObject = open(fileName,mode);
    fileObject.write(content);
    fileObject.close();
 
# file_put_contents('rt.txt','da ge');

批量远程图片下载

multiPool.py

#!/usr/bin/python3
#-*- coding: utf-8 -*-
# author:zhouchao
# 功能：多线程 批量下载远程图片
 
import random
import time
import os
import math
from multiprocessing.dummy import Pool as ThreadPool
import urllib.request
import requests
import sys
sys.path.append(r"../db")
sys.path.append(r"../function")
from file_get_contents import *
from mysqlDriver import *
 
sql = "select img from images limit ?";
mysqlD = mysqlDriver();
mysqlD.connect();
data = mysqlD.select(sql, [100])
 
saveDir = "D:/images/";
def  downImg(url):
    url = url[0];
    ir = requests.get(url)
    if ir.status_code == 200:
        # wb+  保存二进制
        fileName = str(time.time()) +'.jpg';
        file_put_contents(saveDir+fileName,ir.content,"wb+");
 
startTime = time.time();
 
# 不开启多线程
# length = len(data);
# for x in range(length):
#     downImg(data[x][0])
 
# 开启多线程
pool = ThreadPool(8) # Sets the pool size to 4
results = pool.map(downImg,data);
pool.close();
pool.join();
 
endTime = time.time();
consumeTime = endTime - startTime
print("程序运行时间："+str(consumeTime)+" 秒")

注意：

sys.path.append(r"../db")
sys.path.append(r"../function")
请引用自己保存文件路径
 
没有开启和开启多线程所耗时间：