python爬取大众点评

拖了好久的代码

1.首先进入页面确定自己要抓取的数据（我们要抓取的是左侧分类栏-----包括美食、火锅）
先爬取第一级分类（美食、婚纱摄影、电影），之后根据第一级链接爬取第二层（火锅）。要注意第二级的pid是第一级的classid，这样才能区分出第二级分类哪些是属于第一级的。
2.上一步我们分别把链接存入Redis，名称存入了Mongodb，这一步我们要从Redis取链接，取第二级的链接。因为我们要获取店铺的信息（所以取第二级链接就够），我们首先分析取得第一页的内容，然后找到他下一页的代码（取得下一页代码就获取整个分类的店铺）
3.获取店铺信息（根据取得的店铺链接获取对应的店铺信息）
因为我们发现http协议头和cookie我们几乎每次都用到了，所以我们将它们封装成了一个方法，以便用的时候调用。
至此，大众点评就算结束了，只不过还没取评论信息，会慢慢上。

1.py

# -*- coding: utf-8 -*-
import re
from urllib.request import urlopen
from urllib.request import Request
from bs4 import BeautifulSoup
from lxml import etree

from pymongo import MongoClient
client = MongoClient('localhost',27017)
db=client.dianping
collection=db.classification	#类别表

import redis
r = redis.Redis(host='127.0.0.1',port=6379,db=0)

ii=0

def secClassFind(selector,classid):
	global ii
	ii += 1
	secItems = selector.xpath('//div[@class="sec-items"]/a')
	for secItem in secItems:
		url = secItem.get('href')
		title = secItem.text
		classid = collection.insert({'classname':title,'pid':classid})
		classurl = '%s,%s,%i,%s'%(classid,url,ii,title)
		r.lpush('classurl',classurl)

def findRootNode(url):
	headers = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}
	req_timeout = 5
	req = Request(url=url,headers=headers)
	f = urlopen(req,None,req_timeout)
	s=f.read()
	s=s.decode("utf-8")
	# beautiful 提取数据
	soup=BeautifulSoup(s,'html.parser')
	links=soup.find_all(name='li',class_="first-item")
	for link in links:
		selector = etree.HTML(str(link))
		'''
		indexTitleUrls = selector.xpath('//a[@class="index-title"]/@href')
		#获取一级类别url和title
		for titleurl in indexTitleUrls:
			print(titleurl)
		'''
		indexTitles = selector.xpath('//a[@class="index-title"]/text()')
		for title in indexTitles:
			print(title)
			classid = collection.insert({'classname':title,'pid':None})
			#第二级别url
			secClassFind(selector,classid)
			#print(rs)
			print('-------------')

		print('----------------------------------------------')

findRootNode('http://www.dianping.com/')

　　2.py

# -*- coding: utf-8 -*-
import re
from urllib.request import urlopen
from urllib.request import Request
from slaver3_list import getCurPageList
from bs4 import BeautifulSoup
from lxml import etree

from pymongo import MongoClient
client = MongoClient('localhost',27017)
db=client.dianping

import redis
r = redis.Redis(host='127.0.0.1',port=6379,db=0)

'''
1.从classurl中取得一个链接
2.根据此链接获得一个列表页面
3.分析获得页面上的店铺链接
4.获得下一页链接
5.继续爬取下一页信息,继续解析获得链接(重复2~5)
    直到没有下一页为止
'''
#1.从redis中获取一个链接
#classurls = bytes.decode(r.lindex('classurl',0))
shopflag = int(r.get('shopflag'))

if shopflag==0:
    collection=db.shops0    #类别表
    collection.remove({})
    r.set(')
else:
    collection=db.shops1    #类别表
    collection.remove({})
    r.set(')

r.delete('shopurl')

list = r.lrange('classurl',0,-1)
for item in list:
    classurl = bytes.decode(item)        #二进制转字符串
    arr = classurl.split(',')
    #print(arr[0])        #classid
    #print(arr[1])        #classurl
    getCurPageList(arr[0],arr[1],shopflag)
    break

'''
print(classurls)
arr = classurls.split(',')

if int(arr[2])==16:
    #调用
    getCurPageList(arr[0],arr[1])
'''

3.py

# -*- coding: utf-8 -*-
import re
#from urllib.request import urlopen
#from urllib.request import Request
from common import httpSpider
from bs4 import BeautifulSoup
from lxml import etree
from bson.objectid import ObjectId
from slaver4_shopinfo import getShopInfo

from pymongo import MongoClient
client = MongoClient('localhost',27017)
db=client.dianping
collection=None

import redis
r = redis.Redis(host='127.0.0.1',port=6379,db=0)

ii=0

#id,店名,类别id
def insertShop(classid,shopList):
    global collection
    for div in shopList:
        #print(div.get("href"))
        #print(div.get('title'))
        url = div.get("href")
        shopid = collection.insert({'_id':url,'shopname':div.get('title'),'classid':ObjectId(classid)})
        # shopurl = '%s,%s,%s'%(classid,shopid,url)
        # r.lpush('shopurl',shopurl)
        getShopInfo(shopid,url)

def getCurPageList(classid,url,shopflag):
    global ii
    ii += 1
    html = httpSpider(url)
    #print(html)

    selector = etree.HTML(html)

    global collection
    if shopflag==0:
            collection=db.shops0    #店铺表
    else:
        collection=db.shops1

    divTits = selector.xpath('//div[@class="tit"]/a[@title]')
    insertShop(classid,divTits)
    '''
    for div in divTits:
        print(div.get("href"))
        print(div.get('title'))
    '''
    print('----------%i---------------'%(ii))
    #-----下一页--------------------------
    '''
    nextPage = selector.xpath('//a[@class="next"]/@href')
    if len(nextPage)>0:
        newUrl = nextPage[0]
        #print(nextPage[0])
        getCurPageList(newUrl)
    '''

4.py

# -*- coding: utf-8 -*-
import re
#from urllib.request import urlopen
#from urllib.request import Request
from common import httpSpider
from bs4 import BeautifulSoup
from lxml import etree
from bson.objectid import ObjectId

from pymongo import MongoClient
client = MongoClient('localhost',27017)
db=client.dianping
collection=db.shops    #店铺表

import redis
r = redis.Redis(host='127.0.0.1',port=6379,db=0)

def getShopInfo(shopid,shopurl):
    html = httpSpider(shopurl)
    selector = etree.HTML(html)
    briefInfo = selector.xpath('//div[@class="brief-info"]//span[@class="item"]')
    for item in briefInfo:
        print(item.text)

common.py

# -*- coding: utf-8 -*-
import urllib.request
from urllib.request import urlopen
from urllib.request import Request
import http.cookiejar

from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

head = {
    'Connection': 'Keep-Alive',
    'Accept': 'text/html, application/xhtml+xml, */*',
    'Accept-Language': 'en-US,en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
}

def makeMyOpener(head):
    cj = http.cookiejar.CookieJar()
    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
    header = []
    for key, value in head.items():
        elem = (key, value)
        header.append(elem)
    opener.addheaders = header
    return opener
def httpSpider(url):
    oper = makeMyOpener(head)
    req_timeout = 5
    uop = oper.open(url, timeout = req_timeout)
    data = uop.read()
    html = data.decode()
    return html

def dynamicSpider(url):
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/603.2.4 (KHTML, like Gecko) Version/10.1.1 Safari/603.2.4',
        'Connection': 'keep-alive'
    }
    cap = DesiredCapabilities.PHANTOMJS.copy()    #使用copy()防止修改原代码定义dict
    for key, value in headers.items():
        cap['phantomjs.page.customHeaders.{}'.format(key)] = value
    cap["phantomjs.page.settings.loadImages"] = False
    driver = webdriver.PhantomJS(desired_capabilities=cap,executable_path='D:/phantoms/phantomjs-2.1.1-windows/bin/phantomjs.exe')
    driver.get(url)
    html = driver.page_source
    driver.quit()
    return html