python爬取酒店信息练习

　　爬取酒店信息，首先知道要用到那些库。本次使用request库区获取网页，使用bs4来解析网页，使用selenium来进行模拟浏览。

　　本次要爬取的美团网的蚌埠酒店信息及其评价。爬取的网址为“http://hotel.meituan.com/bengbu/”。首先获取导航页的相关信息，具体代码如下

url = 'http://hotel.meituan.com/bengbu/'

# 获取酒店分页信息，返回最大页码
html = requests.get(url).text
soup = BeautifulSoup(html,'html.parser')
page_info = soup.find_all('li',class_='page-link')  # 获取酒店首页的页面导航条信息
get_page_num = page_info[-1].find('a').get_text()       # 获取酒店页面的总页数
print(get_page_num)                         

　　获取了上面的信息，就可以选择一个具体网页，利用Google浏览器的F12查看具体的元素，利用xpath定位相关元素，把获取的信息保存在文件夹下，具体方法代码如下

# 获取所有酒店详细信息
def get_hotel_info(url):
    dcap = dict(DesiredCapabilities.PHANTOMJS)
    dcap['phantomjs.page.settings.userAgent'] = ('Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36')
    browser = webdriver.PhantomJS("D:/PhantomJS/phantomjs-2.1.1-windows/bin/phantomjs", desired_capabilities=dcap)         #指定phantomjs程序路径
    browser.get(url)
    hotel_info = {}
    page_num = 1

    while(page_num < int(get_page_num)+1):
        # 获取一个页面的所有酒店信息
        for item in browser.find_elements_by_class_name('info-wrapper'):
            hotel_info['name'] = item.find_element_by_class_name('poi-title').text
            hotel_info['star'] = item.find_element_by_class_name('poi-grade').text
            hotel_info['consumers'] = item.find_element_by_class_name('poi-buy-num').text
            hotel_info['link'] = item.find_element_by_class_name('poi-title').get_attribute('href')
            print("酒店名称:{}".format(hotel_info['name']))
            print("酒店评分:{}".format(hotel_info['star']))
            print("酒店销量:{}".format(hotel_info['consumers']))
            print("酒店链接:{}".format(hotel_info['link']))
            f = open("酒店信息.txt", 'a', encoding="utf8")
            f.write(hotel_info['name']+"\n"+hotel_info['star']+"\n"+hotel_info['consumers']+"\n"+hotel_info['link']+"\n")
            u = hotel_info['link'][25:-1]
            # print(u)
            # 获取酒店前10页评论内容(动态加载的静态爬取)
            for i in range(10):
                page = i + 1
                s = i * 10
                print("正在加载第" + str(page) + "页评论")
                html = "http://ihotel.meituan.com/group/v1/poi/comment/" + u + "?sortType=default&noempty=1&withpic=0&filter=all&limit=10&offset=" + str(
                      s)+"&X-FOR-WITH="
                # print(html)
                # 第一次只使用一个header导致爬取信息不全，添加多个可以正常爬取
                my_headers = [
                    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
                    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)"
                ]
                randdom_header = random.choice(my_headers)
                headers = {
                    "User-Agent":randdom_header,
                    "Host":"ihotel.meituan.com"
                    }
                r = requests.get(html,headers=headers)
                print(r.text)
                data = json.loads(r.text,strict=False)
                # print(data)
                comments = data['data']['feedback']
                for n in comments:
                    replytime = n['feedbacktime']
                    content = n['comment']
                    # print("评论时间：", replytime)
                    # print("评论内容：", content)
                    f = open("jieguo-1.txt", 'a',encoding="utf8")
                    f.write(content+"\n")

        browser.find_element_by_class_name('paginator').find_element_by_class_name('next').find_element_by_tag_name('a').click()  # 一个页面写完后，通过点击"下一页"图标至下一页，继续获取
        time.sleep(1)
        page_num += 1

　　实现了上述的方法，就可以把完整的酒店信息抓取下来，所有代码如下：

 # encoding="utf8"
 # 爱学习的兔兔
 import requests
 from bs4 import BeautifulSoup
 from selenium import webdriver
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 import time
 import json
 import random

 url = 'http://hotel.meituan.com/bengbu/'

 # 获取酒店分页信息，返回最大页码
 html = requests.get(url).text
 soup = BeautifulSoup(html,'html.parser')
 page_info = soup.find_all('li',class_='page-link')  # 获取酒店首页的页面导航条信息
 get_page_num = page_info[-1].find('a').get_text()       # 获取酒店页面的总页数
 print(get_page_num)                                     # 返回酒店页面的

 # 获取所有酒店详细信息
 def get_hotel_info(url):
     dcap = dict(DesiredCapabilities.PHANTOMJS)
     dcap['phantomjs.page.settings.userAgent'] = ('Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36')
     browser = webdriver.PhantomJS("D:/PhantomJS/phantomjs-2.1.1-windows/bin/phantomjs", desired_capabilities=dcap)         #指定phantomjs程序路径
     browser.get(url)
     hotel_info = {}
     page_num = 1

     while(page_num < int(get_page_num)+1):
         # 获取一个页面的所有酒店信息
         for item in browser.find_elements_by_class_name('info-wrapper'):
             hotel_info['name'] = item.find_element_by_class_name('poi-title').text
             hotel_info['star'] = item.find_element_by_class_name('poi-grade').text
             hotel_info['consumers'] = item.find_element_by_class_name('poi-buy-num').text
             hotel_info['link'] = item.find_element_by_class_name('poi-title').get_attribute('href')
             print("酒店名称:{}".format(hotel_info['name']))
             print("酒店评分:{}".format(hotel_info['star']))
             print("酒店销量:{}".format(hotel_info['consumers']))
             print("酒店链接:{}".format(hotel_info['link']))
             f = open("酒店信息.txt", 'a', encoding="utf8")
             f.write(hotel_info['name']+"\n"+hotel_info['star']+"\n"+hotel_info['consumers']+"\n"+hotel_info['link']+"\n")
             u = hotel_info['link'][25:-1]
             # print(u)
             # 获取酒店前10页评论内容(动态加载的静态爬取)
             for i in range(10):
                 page = i + 1
                 s = i * 10
                 print("正在加载第" + str(page) + "页评论")
                 html = "http://ihotel.meituan.com/group/v1/poi/comment/" + u + "?sortType=default&noempty=1&withpic=0&filter=all&limit=10&offset=" + str(
                       s)+"&X-FOR-WITH="
                 # print(html)
                 # 第一次只使用一个header导致爬取信息不全，添加多个可以正常爬取
                 my_headers = [
                     "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
                     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
                     "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
                     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
                     "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)"
                 ]
                 randdom_header = random.choice(my_headers)
                 headers = {
                     "User-Agent":randdom_header,
                     "Host":"ihotel.meituan.com"
                     }
                 r = requests.get(html,headers=headers)
                 print(r.text)
                 data = json.loads(r.text,strict=False)
                 # print(data)
                 comments = data['data']['feedback']
                 for n in comments:
                     replytime = n['feedbacktime']
                     content = n['comment']
                     # print("评论时间：", replytime)
                     # print("评论内容：", content)
                     f = open("jieguo-1.txt", 'a',encoding="utf8")
                     f.write(content+"\n")

         browser.find_element_by_class_name('paginator').find_element_by_class_name('next').find_element_by_tag_name('a').click()  # 一个页面写完后，通过点击"下一页"图标至下一页，继续获取
         time.sleep(1)
         page_num += 1

 def main():
     get_hotel_info(url)

 if '__main__' == __name__:
     main()

　　这样就顺利的拿到了酒店信息和评价，为了简单分析下拿到的数据，使用了SnowNLP分词看数据，发现效果一般，又使用了jieba分词和词云来分析，得出一张图片如下图：

这里没有对符号进行过滤，只能给出一个大体的评价关系图。具体代码图下：

# encoding="utf8"
# SnowNLP分词
# 爱学习的兔兔
from snownlp import SnowNLP
f = open("jieguo-1.txt","r",encoding="utf8")
r = f.readlines()   #按行读取
#for line in r:
s = SnowNLP(str(r))
for sentence in s.sentences:
    print(sentence)

# jieba分词与词云
import jieba.posseg as posseg
from collections import Counter
from wordcloud import WordCloud
#for line in r:
words = [w for w,f in posseg.cut(str(r))]
print(words)
c = Counter(words)
print(c.most_common(20))
wc = WordCloud(font_path='c:\\Windows\\Fonts\\simkai.ttf', height=1080, width=1920).generate_from_frequencies(c)
image = wc.to_image()
image.show()
wc.to_file("ex2.png")

为了得到效果更好一点，自行百度了一下，得到新的图片，如下：

具体的实现代码如下：

import matplotlib.pyplot as plt
from wordcloud import WordCloud
import jieba
text_from_file_with_apath = open('jieguo-1.txt',encoding="utf-8").read()

wordlist_after_jieba = jieba.cut(text_from_file_with_apath, cut_all=True)
wl_space_split = " ".join(wordlist_after_jieba)

my_wordcloud = WordCloud(font_path='c:\\Windows\\Fonts\\simkai.ttf', height=1080, width=1920).generate(wl_space_split)

plt.imshow(my_wordcloud)
plt.axis("off")
plt.show()　　

整体走下来，感觉写个简单的爬虫能学到不少有用的信息。