python学习笔记（12）--爬虫下载煎蛋网图片

说明：

1. 这个其实是在下载漫画之前写的，比那个稍微简单点，之前忘放到博客备份了。

2. 不想说啥了，总结放到漫画那个里面吧！

 import urllib.request
 import re
 import os

 # http://jandan.net/ooxx/page-2381#comments
 # <span class="current-comment-page">[2381]</span>
 # <img src="//wx4.sinaimg.cn/orj360/797ccd21gy1fdcjecuo1jj20qo0usacj.jpg" style="max-width: 480px; max-height: 750px; background-color: rgb(246, 161, 181);">
 # <a href="//ww1.sinaimg.cn/large/6715afcfgw1ef4zrjdaswj20js0qotag.jpg" target="_blank" class="view_img_link">[查看原图]</a>
 url = "http://jandan.net/ooxx/page-2381#comments"
 headers = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0"}

 # 打开主网站url，获取整个html字符串
 req = urllib.request.Request(url=url,headers=headers)
 response = urllib.request.urlopen(req)
 html = response.read().decode("utf-8")
 # print(html[0:1000])
 # 分析html字符串，找出页数和图片地址
 page = html.find("current-comment-page")
 page = html[page+23:page+27]
 # print(page)
 htmlPages = ""
 for i in range(int(page)-10,int(page)):
     urlPage = "http://jandan.net/ooxx/page-"+str(i)+"#comments"
     reqPage = urllib.request.Request(url=urlPage,headers=headers)
     responsePage = urllib.request.urlopen(reqPage)
     htmlPages += responsePage.read().decode("utf-8")
 regImg = r"//[0-9a-z]+\.sinaimg\.cn/large/[0-9a-z]+\.jpg"
 imgUrl = re.findall(regImg,htmlPages)
 # print(imgUrl)
 imgNum = len(imgUrl)
 # print(imgNum)
 # 创建文件夹
 os.mkdir("test")
 # 切换到这个文件夹
 os.chdir("test")

 # 打开每个图片地址，保存图片到本地
 for i in range(imgNum):
     req = urllib.request.Request(url="http:"+imgUrl[i],headers=headers)
     responseImg = urllib.request.urlopen(req)
     img = open(str(i)+".jpg","wb")
     img.write(responseImg.read())
     img.close

小甲鱼源码（论坛里复制来的，其实是可以运行的，每个图片地址加上http:就可以了）：

 import urllib.request
 import os
 import random
 # 煎蛋网已经禁用爬虫了，所以此程序无法运行
 def url_open(url):
     req = urllib.request.Request(url)
     req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36')
     # iplist = ['111.197.141.57:9797','116.228.236.219:8080','120.26.51.101:8118','113.222.80.216:3128','117.90.1.88:9000']
     # proxy_support = urllib.request.ProxyHandler({'http':random.choice(iplist)})
     # opener = urllib.request.build_opener(proxy_support)
     # urllib.request.install_opener(opener)
     response = urllib.request.urlopen(url)
     html = response.read()
     return html

 def get_page(url):
     html = url_open(url).decode('utf-8')
     a = html.find('current-comment-page') + 23
     b = html.find(']',a)
     return html[a:b]

 def find_imgs(url):
     html = url_open(url).decode('utf-8')
     img_addrs = []
     a = html.find('img src=')
     while a != -1:
         b = html.find('.jpg', a, a + 100)

         if b != -1:
             img_addrs.append(html[a+9:b+4])
             print('图片地址：'+html[a+9:b+4])
         else:
             b = a + 9
         a = html.find('img src=', b)
     return img_addrs
 def save_imgs(folder, img_addrs):
     for each in img_addrs:
         filename = each.split('/')[-1]
         with open(filename, 'wb') as f:
             img = url_open("http:"+each)
             f.write(img)

 def download_mm(folder = 'Xman', pages = 1):
     os.mkdir(folder)
     os.chdir(folder)
     url = "http://jandan.net/ooxx/"
     page_num = int(get_page(url))
     for i in range(pages):
         page_num -= i
         page_url = url + 'page-' + str(page_num) + '#comments'
         img_addrs = find_imgs(page_url)
         save_imgs(folder, img_addrs)
 if __name__ == '__main__':
     download_mm()