妹子图爬取__RE__BS4

妹子图爬取

页面链接

感谢崔大佬: 原文链接

正则实现代码:

 import requests
 import re
 import os
 import random

 class mzitu():

     def __init__(self):
         self.headers = {
             'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
             'Referer': 'http://i.meizitu.net'}
         self.folder_path = str(os.getcwd())

     def all_url(self, url):
         html = self.request(url).text
         all_a_re = r'<a\shref="(http://www.mzitu.com/\d+)"\starget="_blank"'
         pattern = re.compile(all_a_re)
         all_a_list = re.findall(pattern, html)
         # for href in all_a_list:
         #     self.html(href)
         self.html(random.choice(all_a_list))

     def html(self, href):
         html = self.request(href).text
         title_pattern = re.compile(r'<h2\sclass="main-title">(.*?)</h2>', re.S)
         title = re.findall(title_pattern, html)
         self.mkdir(str(title))
         max_pattern = re.compile(r'<span>(\d*?)</span>', re.S)
         max_num = re.findall(max_pattern, html)
         max_num = int(max_num[-1])
         for i in range(max_num):
             page_url = href + "/" + str(i)
             self.img(page_url)

     def mkdir(self, folder):
         path = str(folder.strip())
         os.makedirs(os.path.join(self.folder_path, path))
         print(str(folder) + "------已创建")
         os.chdir(self.folder_path + str("/") + path)

     def img(self, page_url):
         img_html = self.request(page_url).text
         img_url_re = r'<img\ssrc="(.*?)"\salt=".*?"'
         pattern = re.compile(img_url_re, re.S)
         img_all = re.findall(pattern, img_html)
         img_url = img_all[0]
         self.save(img_url)

     def save(self, img_url):
         name = img_url[-6:]
         print(name + "正在下载")
         img = self.request(img_url)
         f = open(name, 'wb')
         f.write(img.content)
         f.close()

     def request(self, url):  # 这个函数获取网页的response 然后返回
         response = requests.get(url, headers=self.headers)
         return response

 Mzitu = mzitu()  # 实例化
 Mzitu.all_url('http://www.mzitu.com/all')

BS4实现代码:

 import requests
 from bs4 import BeautifulSoup
 import os
 import random

 class mzitu():

     def __init__(self):
         self.headers = {
             'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
             'Referer': 'http://i.meizitu.net'}
         self.folder_path = str(os.getcwd())

     def all_url(self, url):
         html = self.request(url)  # 调用request函数把套图地址传进去会返回给我们一个response
         all_a = BeautifulSoup(html.text, 'lxml').find(
             'div', class_='all').find_all('a')

         # for i in range(1, len(all_a)):
         #     a = all_a[i]
         a = random.choice(all_a)
         title = a.get_text()
         print(u'开始保存：', title)  # 加点提示不然太枯燥了
         # 我注意到有个标题带有 ？  这个符号Windows系统是不能创建文件夹的所以要替换掉
         path = str(title).replace("?", '_')
         self.mkdir(path)  # 调用mkdir函数创建文件夹！这儿path代表的是标题title哦！！！！！不要糊涂了哦！
         href = a['href']
         # 调用html函数把href参数传递过去！href是啥还记的吧？ 就是套图的地址哦！！不要迷糊了哦！
         self.html(href)

     def html(self, href):  # 这个函数是处理套图地址获得图片的页面地址
         html = self.request(href)
         self.headers['referer'] = href
         max_span = BeautifulSoup(html.text, 'lxml').find(
             'div', class_='pagenavi').find_all('span')[-2].get_text()
         for page in range(1, int(max_span) + 1):
             page_url = href + '/' + str(page)
             self.img(page_url)  # 调用img函数

     def img(self, page_url):  # 这个函数处理图片页面地址获得图片的实际地址
         img_html = self.request(page_url)
         img_url = BeautifulSoup(img_html.text, 'lxml').find(
             'div', class_='main-image').find('img')['src']
         self.save(img_url)

     def save(self, img_url):  # 这个函数保存图片
         name = img_url[-6:]
         print(name + "正在下载")
         img = self.request(img_url)
         f = open(name, 'ab')
         f.write(img.content)
         f.close()

     def mkdir(self, path):  # 这个函数创建文件夹
         path = path.strip()
         isExists = os.path.exists(os.path.join(self.folder_path, path))
         if not isExists:
             print(path)
             os.makedirs(os.path.join(self.folder_path, path))
             os.chdir(self.folder_path + str("/") + path)  # 切换到目录
             return True
         else:
             print(u'名字叫做', path, u'的文件夹已经存在了！')
             return False

     def request(self, url):  # 这个函数获取网页的response 然后返回
         content = requests.get(url, headers=self.headers)
         return content

 Mzitu = mzitu()  # 实例化
 Mzitu.all_url('http://www.mzitu.com/all')  # 给函数all_url传入参数  你可以当作启动爬虫（就是入口）