python3----练习题(爬取电影天堂资源,大学排名,淘宝商品比价)

 import requests
 import re
 
 url = 'http://www.ygdy8.net/html/gndy/dyzz/list_23_{}.html'
 for n in range(1, 2):
     new_url = url.format(n)
     html_1 = requests.get(new_url)
     html_1.encoding = 'gb2312'
     detil_list = re.findall('<a href="(.*?)" class="ulink">', html_1.text)
 
     for m in detil_list:
         b_url = 'http://www.ygdy8.net' + m
         html_2 = requests.get(b_url)
         html_2.encoding = 'gb2312'
         ftp = re.findall('<a href="(.*?)">.*?</a></td>', html_2.text)
         with open('tddy.txt', 'a', encoding='utf-8') as f:
             f.write(ftp[0] + '\n')

大学排名练习

 import bs4
 import requests
 from bs4 import BeautifulSoup
 
 def get_html_text(url):
     try:
         r = requests.get(url, timeout=20)
         r.raise_for_status()
         r.encoding = r.apparent_encoding
         return r.text
     except:
         return " "
 
 def fill_univ_list(ulist, html):
     soup = BeautifulSoup(html, "html.parser")
     for tr in soup.find('tbody').children:
         if isinstance(tr, bs4.element.Tag):   # 判断类型
             tds = tr('td')
             ulist.append([tds[0].string, tds[1].string, tds[3].string])
 
 def print_univ_list(ulist, num):
     tplt = "{0:^10}\t{1:{3}^10}\t{2:^10}"
     print(tplt.format("排名", "学校名称", "总分", chr(12288)))
     for i in range(num):
         u = ulist[i]
         print(tplt.format(u[0], u[1], u[2], chr(12288)))
 
 def main():
     uinfo = []
     url = 'http://www.zuihaodaxue.com/zuihaodaxuepaiming2016.html'
     html = get_html_text(url)
     fill_univ_list(uinfo, html)
     print_univ_list(uinfo, 20)
 
 main()

淘宝商品比价:

 import requests
 import re
 
 def get_html_text(url):
     try:
         r = requests.get(url, timeout=30)
         r.raise_for_status()
         r.encoding = 'utf-8'
         return r.text
     except:
         return ""
 
 def parse_page(ilt, html):
     try:
         plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"', html)
         tlt = re.findall(r'\"raw_title\"\:\".*?\"', html)
         for i in range(len(plt)):
             price = eval(plt[i].split(':')[1])
             title = eval(tlt[i].split(':')[1])
             ilt.append([price, title])
     except:
         print("")
 
 def print_goods_list(ilt):
     tplt = "{:4}\t{:8}\t{:16}"
     print(tplt.format("序号", "价格", "商品名称"))
     count = 0
     for g in ilt:
         count = count + 1
         print(tplt.format(count, g[0], g[1]))
 
 def main():
     goods = '减肥餐'
     depth = 2
     start_url = 'http://s.taobao.com/search?q=' + goods
     info_list = []
     for i in range(depth):
         try:
             url = start_url + '&s=' + str(44*i)
             html = get_html_text(url)
             parse_page(info_list, html)
         except:
             continue
     print_goods_list(info_list)

股票数据:

 import re
 import traceback
 
 import requests
 import sys
 from bs4 import BeautifulSoup
 
 def get_html_text(url, code='utf-8'):
     headers ={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}
     try:
         r = requests.get(url, timeout=20, headers=headers)
         r.raise_for_status()
         r.encoding = code
         return r.text
     except:
         return ""
 
 def get_stock_list(lst, stock_url):
     html = get_html_text(stock_url, 'GB2312')
     soup = BeautifulSoup(html, 'html.parser')
     a = soup.find_all('a')
     for i in a:
         stock_code = re.findall(r'[s][hz]\d{6}', str(i))
         if len(stock_code) != 0:
             lst.append(stock_code)
 
 def get_stock_info(lst, stock_url, fpath):
     count = 0
     for stock in lst:
         url = stock_url + stock[0] + '.html'
         print(url)
         html = get_html_text(url)
         try:
             if html == "":
                 continue
             info_dict = {}
             soup = BeautifulSoup(html, 'html.parser')
             stock_info = soup.find('div', attrs={'class': 'stock-bets'})
             info_dict.update({'股票名称': stock_info.text.split()[0]})
 
             key_list = stock_info.find_all('dt')
             value_list = stock_info.find_all('dd')
             for i in range(len(key_list)):
                 key = key_list[i].text
                 info_dict[key] = value_list[i].text
 
             with open(fpath, 'a', encoding='utf-8') as f:
                 f.write(str(info_dict) + '\n')
                 count = count + 1
                 print("\r当前进度: {:.2f}%".format(count*100/len(lst), end=""))
         except:
             traceback.print_exc(file=sys.stdout)
             count = count + 1
             print("\r当前进度: {:.2f}%".format(count * 100 / len(lst), end=""))
             continue
 
 def main():
     stock_list_url = 'http://quote.eastmoney.com/stocklist.html'
     stock_info_url = 'http://gupiao.baidu.com/stock/'
     output_file = 'D:/BaiduStockInfo.txt'
     slist = []
     get_stock_list(slist, stock_list_url)
     get_stock_info(slist, stock_info_url, output_file)