python3爬取动态网站图片

思路：

1、图片放在<image>XXX</image>标签中

2、利用fiddler抓包获取存放图片信息的js文件url

3、利用requests库获取html内容，然后获取其中图片id

4、利用fiddler抓取下载图片地址，结合图片id来下载图片（大文件）

# -*- coding:UTF-8 -*-
import requests, json, time
from contextlib import closing

class get_photos(object):

    def __init__(self):
        self.photos_id = []
        self.download_server = 'https://unsplash.com/photos/xxx/download?force=trues'#下载图片地址，通过fiddler抓包获取
        self.target = 'https://unsplash.com/napi/photos?page=1&per_page=24'#存有图片信息的js文件地址
        self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'}

    """
    函数说明:获取图片ID
    """
    def get_ids(self):
        req = requests.get(url=self.target, headers=self.headers, verify=False)
        html = json.loads(req.text)
        print(type(html), len(html))
        for i in range(len(html)):
            self.photos_id.append(html[i]['id'])#将列表html中获取id放到列表photos_id中
        print(self.photos_id)
        time.sleep(1)

    """
    函数说明:图片下载
    """
    def download(self, photo_id, filename):
        self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'}
        self.target = self.download_server.replace('xxx', photo_id)#将xxx换成图片id
        #closing可以将任意对象转成上下文对象
        with closing(requests.get(url=self.target, stream=True, verify = False, headers = self.headers)) as r:
            with open('%d.jpg' % filename, 'ab+') as f:
                #下载大文件，r.iter_content表示获取响应原始内容
                for chunk in r.iter_content(chunk_size = 1024):#边下载边存硬盘，chunk_size表示以1024比例存数据
                    if chunk:
                        f.write(chunk)
                        f.flush()#强行把缓冲区中的内容放到磁盘中

if __name__ == '__main__':
    gp = get_photos()#类实例化
    print('获取图片连接中:')
    gp.get_ids()#获取图片id
    print('图片下载中:')
    for i in range(len(gp.photos_id)):
        print('正在下载第%d张图片' % (i+1))
        gp.download(gp.photos_id[i], (i+1))