爬取前尘无忧python职位信息并保存到mongo数据库

１．re实现

 import re,os
 import requests
 from requests.exceptions import RequestException
 
 MAX_PAGE = 10 #最大页数
 KEYWORD = 'python'
 headers = {
     'User-Agent':
         'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
 }
 file_name = 're_job51_python.txt'
 
 # 获取网页源码
 def getHtml(page):
     try:
         url = 'https://search.51job.com/list/040000,000000,0000,00,9,99,{0},2,{1}.html?'.format(KEYWORD,page)
         response = requests.get(url,headers=headers)
         response.encoding = response.apparent_encoding
         return response.text
     except RequestException:
         print('请求出错')
         return None
 
 # 解析网页源码，得到目标信息
 def getTarget(html):
     reg = re.compile(
         r'class="t1 ">.*? <a target="_blank" '
         'title="(.*?)".*? <span class="t2"><a target="_blank" '
         'title="(.*?)".*?<span '
         'class="t3">(.*?)</span>.*?<span '
         'class="t4">(.*?)</span>.*? <span '
         'class="t5">(.*?)</span>',
         re.S)  # 匹配换行符
     target = re.findall(reg,html)
     return target
 
 # 保存到文本中
 def save_to_txt(item):
     with open(file_name,'a',newline='') as f:  # newline参数防止两行之间有空行
         for i in range(len(item)):
             # 最后一个元素换行，非最后则以＇,'隔开
             if i == len(item)-1:
                 f.write(item[i])
                 f.write('\n')
             else:
                 f.write(item[i]+',')
 
 def main():
     # 每次执行前检查文件是否存在，存在则删除
     if os.path.exists(file_name):
         os.remove(file_name)
 
     # 分页爬取
     for page in range(MAX_PAGE+1):
         html = getHtml(page)
         content = getTarget(html)
         for item in content:
             save_to_txt(item)
 
 if __name__ == '__main__':
     main()

２．xpath实现

 import os
 import requests
 from requests.exceptions import RequestException
 from lxml import etree
 import pymongo
 from spiders.前程无忧.mongo_config import *
 
 # mongo数据库设置
 client = pymongo.MongoClient(MONGO_URL)
 db = client[MONGO_DB]
 
 MAX_PAGE = 5
 KEYWORD = 'python'
 headers = {
     'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '\
     'Chrome/63.0.3239.132 Safari/537.36'
 }
 file_name = 'xpath_job51_python.txt'
 
 # 获取网页
 def get_html(page):
     try:
         url = 'https://search.51job.com/list/040000,000000,0000,00,9,99,{},2,{}.html?'.format(KEYWORD,page)
         response = requests.get(url,headers=headers)
         response.encoding = response.apparent_encoding
         return response.text
     except RequestException:
         return None
 
 # 解析网页
 def parse_html(html):
     # 构造xpath解析对象，可自动修整HTML文本
     html = etree.HTML(html)
     # 获取文本 /text()
     # 获取属性　／@href
     # 获取第i个标签　／tar_name[i]  从1开始
     # normalize-space－－＞去空格换行符
     # position_name = html.xpath('normalize-space(//div[@class="el"]/p/span/a/text())')
 
     # 职位名称，
     position_names = []
     for name in html.xpath('//div[@class="el"]/p/span/a/text()'):
         position_name = name.strip()
         position_names.append(position_name)
 
     # 职位地址
     position_urls = html.xpath('//div[@class="el"]/p/span/a/@href')
 
     # 公司名称
     company_names = html.xpath('//div[@class="el"]/span[1]/a/text()')
 
     # 公司地址
     company_urls = html.xpath('//div[@class="el"]/span[1]/a/@href')
 
     # 位置
     locations = html.xpath('//div[@class="el"]/span[@class="t3"]/text()')
 
     # 薪资
     salarys = html.xpath('//div[@class="el"]/span[@class="t4"]/text()')
 
     # 发布时间
     release_dates = html.xpath('//div[@class="el"]/span[4]/text()')
 
     result = zip(position_names,position_urls,company_names,company_urls,locations,salarys,release_dates)
     return result
 
 def save_to_txt(element):
     with open(file_name,'a',newline='') as f:
         for i in range(len(element)):
             # data = ','.join(element[i])
             if i == len(element)-1:
                 f.write(element[i])
                 f.write('\n')
             else:
                 f.write(element[i]+',')
 
 def save_to_mongo(element):
     keys = ['position_name','position_url','company_name',
             'company_url','location','salary','release_date']
     result = dict(zip(keys,list(element)))
     if db[MONGO_TABLE_XPATH].insert(result):
         print('数据成功存储到mongo数据库中')
         return True
     return False
 
     # 遍历字典元素
     # for k,v in result.items():
     #     print(k,':',v)
     for key in result:
         print(key,':',result[key])
 
 def main():
     if os.path.exists(file_name):
         os.remove(file_name)
     for page in range(1,MAX_PAGE+1):
         html = get_html(page)
         elements = parse_html(html)
         if elements:
             for element in elements:
                 save_to_txt(element)
                 save_to_mongo(element)
 
 if __name__ == '__main__':
     main()