记一次 爬取LOL全皮肤原画保存到本地的实例

 #爬取lol全英雄皮肤
 import re
 import traceback #  异常跟踪
 import requests
 from bs4 import BeautifulSoup
 #获取html
 def get_url(url, hander):
     try:
         r = requests.get(url, headers=hander, timeout=30)
         r.raise_for_status()
         r.encoding = r.apparent_encoding
         return r.text
     except:
         traceback.print_exc() #将异常信息打印出来
         return ""
 #解析html
 def prasing_page(lst,html):
     try:
         soup = BeautifulSoup(html, "html.parser")
         for a in soup.find_all('li', class_=re.compile('boxShadow')):
             tag_a = a('a')
             for i in tag_a:
                 lst.append(i['href'])
         return lst
     except:
         traceback.print_exc()
         return ""

 #解析获取到的单个html并筛选和下载
 def getUrl_prasingpag(lst, hander):
     hero_img_url = []
     hero_skin_name = []
     hero_name = []
     for u in lst:
         try:
             r = requests.get(u, headers=hander, timeout=30)
             r.raise_for_status()
             r.encoding = r.apparent_encoding
 　　　　　　　#二次解析
             soup = BeautifulSoup(r.text, "html.parser")
             pag = soup.find_all('div', class_=re.compile('othersPifuBox'))
             for m in pag:
                 tag_img = m('img')
                 tag_p = m('p')
                 tag_span = m('span')
                 for m in tag_p:
                     hero_skin_name.append(m.string)
                 for m in tag_img:
                     hero_img_url.append(m['src'])
                 for m in tag_span:
                     hero_name.append(m.string)
         except:
             traceback.print_exc()  # 将异常信息打印出来
             continue
 　　　　　　　　　　　　　　#下载到本地
         for i in range(len(hero_name)):
             try:
                 path = 'O:/lol_hero_jpg/' + hero_skin_name[i]+'--' + hero_name[i] + '.jpg'
                 f = open(path, 'wb')
                 r = requests.get(hero_img_url[i], stream=True)
                 f.write(r.content)
                 print("\r当前进度>>>>>>>>>>>>>>>>>>{:.0f}%>>>>>>>>>>>>>>>>>>".format(i * 100 / len(lst)), end="")
                 f.close()
             except:
                 traceback.print_exc()  # 将异常信息打印出来
                 continue

 def main():
     hander = {"User-Agent":"Mozilla/5.0"}
     deep = 43 #定义爬取页数
     list = []
     for i in range(deep):
         try:
             url = "http://********/hero_"+str(1+i)+".shtml"
             html = get_url(url, hander)
             prasing_page(list, html)
             getUrl_prasingpag(list, hander)
         except:
             continue

 main()