Python 爬取 拉钩

...

from urllib import request
from urllib import parse
from urllib.error import URLError
import json
import math
import pymongo

MONGO_URL='localhost'
MONGO_DB='LaGou'
MONGO_TABLE='数据分析'

client = pymongo.MongoClient(MONGO_URL)#连接数据库
db=client[MONGO_DB]#创建数据库的名字

page=0#页码信息
City='深圳'
KeyWord='数据分析师'
base_url='https://www.lagou.com/jobs/positionAjax.json?'
params = {
    'px':'default',
    'city':City,
    'needAddtionalResult':'false'
}
url = base_url+parse.urlencode(params)
headers = {
    'Accept':'application/json, text/javascript, */*; q=0.01',
    'Accept-Encoding':'gzip, deflate, br',
    'Accept-Language':'zh-CN,zh;q=0.9',
    'Connection':'keep-alive',
    'Content-Length':64,
    'Cookie':'_ga=GA1.2.257516341.1522050452; user_trace_token=20180326154732-f0f5196c-30c9-11e8-9f49-525400f775ce; LGUID=20180326154732-f0f520df-30c9-11e8-9f49-525400f775ce; LG_LOGIN_USER_ID=60032b76bd32a5a6a596559eade6d0052f15fc2ab9779cc8; index_location_city=%E6%B7%B1%E5%9C%B3; JSESSIONID=ABAAABAAAIAACBI1A73582AEDC2FDC8335E968502A9F5D2; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1527150390,1528277552,1528789503,1529461642; _gid=GA1.2.1084200828.1529461642; LGSID=20180620153431-5f3689a6-745c-11e8-aa10-525400f775ce; TG-TRACK-CODE=search_code; LGRID=20180620160449-9a8ba121-7460-11e8-aa14-525400f775ce; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1529481888; SEARCH_ID=efa038bbf91d47488b5c710088432205',
    'Host':'www.lagou.com',
    'Origin':'https://www.lagou.com',
    'Referer':'https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90%E5%B8%88?px=default&city=%E6%B7%B1%E5%9C%B3',
    'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8',
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}

#请求和获取信息的函数
def get_postion_info(pn=1):
    global page#全局的页码
    data = {
        'first':'true',
        'pn':pn,
        'kd':KeyWord
    }
    try:
        data = bytes(parse.urlencode(data),encoding='utf-8')
        req = request.Request(url=url,data=data,headers=headers,method='POST')#构建Request对象
        html = request.urlopen(req)#发送请求
        if html.status==200:
            html = html.read().decode('utf-8')#将响应体的信息转化为字符串
            res = json.loads(html)#把类json的格式转化为字典
            #如果是第一页，就获取页码信息
            if pn==1:
                pagesize = res['content']['pageSize']
                totalcount=res['content']['positionResult']['totalCount']
                page = math.ceil(totalcount/pagesize)
            #获取招聘信息，并保存到MongoDB
            result = res['content']['positionResult']['result']
            for i in range(len(result)):
                save_mongodb(result[i])
            return res
        else:
            print('出现错误',html.status)
            return None
    except URLError:
        print('出现异常！')
        return None

#保存到MongoDB
def save_mongodb(info):
    if db[MONGO_TABLE].insert(info):
        print('保存成功！',info)
    else:
        print(info,'保存失败！',info)

#主体函数
def main():
    get_postion_info()#获取第一页的信息和页码信息
    #爬取第二页以后的信息
    for p in range(2,page+1):
        get_postion_info(pn=p)

if __name__=='__main__':
    main()