Python-爬取妹子图(单线程和多线程版本)

一、参考文章

Python爬虫之——爬取妹子图片

上述文章中的代码讲述的非常清楚，我的基本能思路也是这样，本篇文章中的代码仅仅做了一些异常处理和一些日志显示优化工作，写此文章主要是当做笔记，方便以后查阅，修改的地方如下：

1、异常处理下面在代码中会单独标红

2、多线程版使用了multiprocessing这个库，需要在main函数开始调用freeze_support()，防止打包成exe之后，运行时创建线程失败

3、多线程版本加了一个命令行自定义线程个数功能

二、单线程版本

 #coding=utf-8
 import requests
 from bs4 import BeautifulSoup
 import os

 all_url = 'http://www.mzitu.com'

 #http请求头
 Hostreferer = {
     'User-Agent':'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
     'Referer':'http://www.mzitu.com'
                }
 Picreferer = {
     'User-Agent':'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
     'Referer':'http://i.meizitu.net'
 }
 #此请求头破解盗链

 start_html = requests.get(all_url, headers = Hostreferer)

 #保存地址
 path = os.getcwd() + '/mzitu/'

 #找寻最大页数
 soup = BeautifulSoup(start_html.text, "html.parser")
 page = soup.find_all('a', class_='page-numbers')
 max_page = page[-2].text

 same_url = 'http://www.mzitu.com/page/'
 for n in range(0, int(max_page)+1):#遍历页面数
     ul = same_url+str(n)
     start_html = requests.get(ul, headers = Hostreferer)
     soup = BeautifulSoup(start_html.text, "html.parser")
     all_a = soup.find('div', class_ = 'postlist').find_all('a', target = '_blank')
     for a in all_a:#每个页面包含的妹子数
         title = a.get_text() #提取文本
         if(title != ''):
             print("准备扒取：" + title)

             #win不能创建带？的目录
             if(os.path.exists(path+title.strip().replace('?', ''))):
                     #print('目录已存在')
                     flag = 1
             else:
                 os.makedirs(path+title.strip().replace('?', ''))
                 flag = 0
             os.chdir(path + title.strip().replace('?', ''))
             href = a['href']
             html = requests.get(href, headers = Hostreferer)
             mess = BeautifulSoup(html.text, "html.parser")
             pic_max = mess.find_all('span')
             pic_max = pic_max[10].text #最大页数
             if(flag == 1 and len(os.listdir(path+title.strip().replace('?', ''))) >= int(pic_max)):
                 print('已经保存完毕，跳过')
                 continue
             for num in range(1, int(pic_max) + 1):#每个妹子的所有照片
                 pic = href+'/'+str(num)
                 html = requests.get(pic, headers = Hostreferer)
                 mess = BeautifulSoup(html.text, "html.parser")
                 pic_url = mess.find('img', alt = title)

                 if 'src' not in pic_url.attrs:#有些pic_url标签没有src这个属性，导致操作异常，在次进行过滤
                     continue
                 print(pic_url['src'])
                 #exit(0)
                 html = requests.get(pic_url['src'],headers = Picreferer)
                 file_name = pic_url['src'].split(r'/')[-1]
                 f = open(file_name, 'wb')
                 f.write(html.content)
                 f.close()
             print('完成')
     print('第',n,'页完成')

三、多线程版本

 #coding=utf-8
 import requests
 from bs4 import BeautifulSoup
 import os
 from multiprocessing import Pool
 from multiprocessing import freeze_support
 import sys

 header = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 UBrowser/6.1.2107.204 Safari/537.36',
     'Referer':'http://www.mzitu.com'
     }
 Picreferer = {
     'User-Agent':'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
     'Referer':'http://i.meizitu.net'
 }

 def find_MaxPage():
     all_url = 'http://www.mzitu.com'
     start_html = requests.get(all_url, headers = header)
     #找寻最大妹子页面数
     soup = BeautifulSoup(start_html.text, "html.parser")
     page = soup.find_all('a', class_ = 'page-numbers')
     max_page = page[-2].text
     return max_page

 def Download(href, title, path):
     html = requests.get(href, headers = header)
     soup = BeautifulSoup(html.text, 'html.parser')
     pic_max = soup.find_all('span')
     pic_max = pic_max[10].text  # 最大页数
     if(os.path.exists(path+title.strip().replace('?', ''))
     and len(os.listdir(path+title.strip().replace('?', ''))) >= int(pic_max)):
         print('妹子已待命，继续准备下一个妹子' + title)
         return 1
     print(f"发现妹子资源{pic_max}个，准备中：" + title)
     os.makedirs(path + title.strip().replace('?', ''))
     os.chdir(path + title.strip().replace('?', ''))
     for num in range(1, int(pic_max) + 1):
         pic = href + '/' + str(num)
         html = requests.get(pic, headers = header)
         mess = BeautifulSoup(html.text, "html.parser")
         pic_url = mess.find('img', alt = title)
         if 'src' not in pic_url.attrs:#有些pic_url标签没有src属性，导致操作异常，在次进行过滤
             continue
         print(f"{title}：{pic_url['src']}")
         html = requests.get(pic_url['src'], headers = header)
         file_name = pic_url['src'].split(r'/')[-1]
         f = open(file_name,'wb')
         f.write(html.content)
         f.close()
     print('妹子已就绪，客官请慢用：' + title)

 if __name__ == '__main__':
     freeze_support()#防止打包后 运行exe创建进程失败

     #线程池中线程数
     count = 1
     if len(sys.argv) >=2:
         count = int(sys.argv[1])

     pool = Pool(count)
     print(f'初始化下载线程个数${count}')

     # http请求头
     path = os.getcwd() + '/mzitu_mutil/'
     max_page = find_MaxPage() #获取最大页数  即生成的文件夹数量
     print(f'捕获{max_page}页妹子，请耐心等待下载完成')
     same_url = 'http://www.mzitu.com/page/'

     for n in range(1, int(max_page) + 1):
         each_url = same_url + str(n)
         start_html = requests.get(each_url, headers = header)#请求一页中的所有妹子
         soup = BeautifulSoup(start_html.text, "html.parser")
         all_a = soup.find('div', class_ = 'postlist').find_all('a', target = '_blank')
         for a in all_a:#遍历每一页中的妹子
             title = a.get_text()  # 提取文本
             if (title != ''):
                 href = a['href']#请求妹子的所有图集
                 pool.apply_async(Download, args = (href, title, path))
     pool.close()
     pool.join()
     print('所有妹子已就绪，客官请慢用')

四、资源下载

　　资源下载地址：Python爬取妹子图-单线程和多线程版本

转载声明：本站文章无特别说明，皆为原创，版权所有，转载请注明：朝十晚八