爬取淘宝商品数据并保存在excel中

１.re实现

 import requests
 from requests.exceptions import RequestException
 import re,json
 import xlwt,xlrd
 
 # 数据
 DATA = []
 KEYWORD = 'python'
 HEADERS = {'user-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome'\
                         '/63.0.3239.132 Safari/537.36'}
 MAX_PAGE = 10
 
 def get_target(data_list):
     for item in data_list:
          temp = {
         'title': item['title'],
         'price': item['view_price'],
         'sales': item['view_sales'],
         'isTmall': '否' if float(item['view_fee']) else '是',
         'area': item['item_loc'],
         'name': item['nick'],
         'url': item['detail_url']
          }
          DATA.append(temp)
     return True
 
 # 发送http请求，获取网页源码
 def get_html(url,*args):
     try:
         if not args:
             response = requests.get(url,headers=HEADERS)
             global COOKIES
             COOKIES = response.cookies  # 获取cookie
         else:
             response = requests.get(url,headers=HEADERS,cookies=COOKIES)
 
         response.encoding = response.apparent_encoding
         return response.text
     except RequestException:
         print('请求源码出错！')
 
 # 解析源码，得到目标信息
 def parse_html(html,*args):
     if not args:
         pattern = re.compile(r'g_page_config = (.*?)g_srp_loadCss',re.S)
         # 去掉末尾的';'
         result = re.findall(pattern, html)[0].strip()[:-1]
         # 格式化json，可以用json在线解析工具查看结构
         content = json.loads(result)
         data_list = content['mods']['itemlist']['data']['auctions']
     else:
         pattern = re.compile(r'{.*}',re.S)
         result = re.findall(pattern,html)[0]
         content = json.loads(result)
         data_list = content['API.CustomizedApi']['itemlist']['auctions']
 
     get_target(data_list)
 
 def save_to_excel():
     f_name = '淘宝%s数据'%KEYWORD
     book = xlwt.Workbook(encoding='utf-8',style_compression=0)
     sheet = book.add_sheet(f_name)
     sheet.write(0, 0, 'title')
     sheet.write(0, 1, 'price')
     sheet.write(0, 2, 'sales')
     sheet.write(0, 3, 'isTmall')
     sheet.write(0, 4, 'area')
     sheet.write(0, 5, 'name')
     sheet.write(0, 6, 'url')
     for i in range(len(DATA)):
         sheet.write(i+1, 0, DATA[i]['title'])
         sheet.write(i+1, 1, DATA[i]['price'])
         sheet.write(i+1, 2, DATA[i]['sales'])
         sheet.write(i+1, 3, DATA[i]['isTmall'])
         sheet.write(i+1, 4, DATA[i]['area'])
         sheet.write(i+1, 5, DATA[i]['name'])
         sheet.write(i+1, 6, DATA[i]['url'])
     book.save('淘宝%s数据.xls'%KEYWORD)
 
 def main():
     for offset in range(MAX_PAGE):
         #  首页有12条异步加载的数据　api?
         if offset == 0:
             url1 = 'https://s.taobao.com/search?q={}&s={}'.format(KEYWORD,offset*44)
             html = get_html(url1)
             contents = parse_html(html)
 
             url2 = 'https://s.taobao.com/api?_ksTS=1532524504679_226&callback=jsonp227&ajax=true&m=customized&' \
                    'stats_click=search_radio_all:1&q={}'.format(KEYWORD)
             html = get_html(url2,2)
             contents = parse_html(html,2)
         else:
             url = 'https://s.taobao.com/search?q={}&s={}'.format(KEYWORD,offset*44)
             html = get_html(url)
             contents = parse_html(html)
 
     save_to_excel()
     print(len(DATA))
 
 if __name__ == '__main__':
     main()