Python爬虫项目--爬取自如网房源信息

本次爬取自如网房源信息所用到的知识点:

1. requests get请求

2. lxml解析html

3. Xpath

4. MongoDB存储

正文

1.分析目标站点

1. url: http://hz.ziroom.com/z/nl/z3.html?p=2 的p参数控制分页

2. get请求

2.获取单页源码

 # -*- coding: utf-8 -*-
 import requests
 import time
 from requests.exceptions import RequestException
 def get_one_page(page):
     try:
         url = "http://hz.ziroom.com/z/nl/z2.html?p=" + str(page)
         headers = {
             'Referer':'http://hz.ziroom.com/',
             'Upgrade-Insecure-Requests':'',
             'User-Agent':'Mozilla/5.0(WindowsNT6.3;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/68.0.3440.106Safari/537.36'
         }
         res = requests.get(url,headers=headers)
         if res.status_code == 200:
             print(res.text)
     except RequestException:
         return None
 def main():
     page = 1
     get_one_page(page)
 if __name__ == '__main__':
     main()
     time.sleep(1)

3.解析单页源码

1. 解析html文档, 目的: 测试XPath表达式

将获取的源码保存到当前文件夹下的"result.html"中, 然后通过XPath对其进行相应内容的提取, 当然你也可以使用某些在线工具.

 from lxml import etree
 #解析html文档
 html = etree.parse("./resul.html",etree.HTMLParser())
 results = html.xpath('//ul[@id="houseList"]/li')
 for result in results[1:]:
     title = result.xpath("./div/h3/a/text()")[0][5:] if len(result.xpath("./div/h3/a/text()")[0]) >5 else ""
     location = result.xpath("./div/h4/a/text()")[0].replace("[","").replace("]",'')
     area = " ".join(result.xpath("./div/div/p[1]/span/text()")).replace(" ","",1) #使用join方法将列表中的内容以" "字符连接
     nearby = result.xpath("./div/div/p[2]/span/text()")[0]
     print(title)
     print(location)
     print(area)
     print(nearby)

2. 解析源代码

 from lxml import etree
 def parse_one_page(sourcehtml):
     '''解析单页源码'''
     contentTree = etree.HTML(sourcehtml)   #解析源代码
     results = contentTree.xpath('//ul[@id="houseList"]/li') #利用XPath提取相应内容
     for result in results[1:]:
         title = result.xpath("./div/h3/a/text()")[0][5:] if len(result.xpath("./div/h3/a/text()")[0]) > 5 else ""
         location = result.xpath("./div/h4/a/text()")[0].replace("[", "").replace("]", '')
         area = " ".join(result.xpath("./div/div/p[1]/span/text()")).replace(" ", "", 1)  # 使用join方法将列表中的内容以" "字符连接
         nearby = result.xpath("./div/div/p[2]/span/text()")[0]
         yield {
         "title": title,
         "location": location,
         "area": area,
         "nearby": nearby
         }
 def main():
     page = 1
     html = get_one_page(page)
     print(type(html))
     parse_one_page(html)
     for item in parse_one_page(html):
       print(item)

 if __name__ == '__main__':
     main()
     time.sleep(1)

4.获取多个页面

 def parse_one_page(sourcehtml):
     '''解析单页源码'''
     contentTree = etree.HTML(sourcehtml)   #解析源代码
     results = contentTree.xpath('//ul[@id="houseList"]/li') #利用XPath提取相应内容
     for result in results[1:]:
         title = result.xpath("./div/h3/a/text()")[0][5:] if len(result.xpath("./div/h3/a/text()")[0]) > 5 else ""
         location = result.xpath("./div/h4/a/text()")[0].replace("[", "").replace("]", '')
         area = " ".join(result.xpath("./div/div/p[1]/span/text()")).replace(" ", "", 1)  # 使用join方法将列表中的内容以" "字符连接
         #nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip()这里需要加判断, 改写为下句
         nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip() if len(result.xpath("./div/div/p[2]/span/text()"))>0 else ""
         yield {
         "title": title,
         "location": location,
         "area": area,
         "nearby": nearby
         }
         print(nearby)
     #yield {"pages":pages}
 def get_pages():
     """得到总页数"""
     page = 1
     html = get_one_page(page)
     contentTree = etree.HTML(html)
     pages = int(contentTree.xpath('//div[@class="pages"]/span[2]/text()')[0].strip("共页"))
     return pages
 def main():
     pages = get_pages()
     print(pages)
     for page in range(1,pages+1):
         html = get_one_page(page)
         for item in parse_one_page(html):
             print(item)

 if __name__ == '__main__':
     main()
     time.sleep(1)

5. 存储到MongoDB中

需确保MongoDB已启动服务, 否则必然会存储失败

 def save_to_mongodb(result):
     """存储到MongoDB中"""
     # 创建数据库连接对象, 即连接到本地
     client = pymongo.MongoClient(host="localhost")
     # 指定数据库,这里指定ziroom
     db = client.iroomz
     # 指定表的名称, 这里指定roominfo
     db_table = db.roominfo
     try:
         #存储到数据库
         if db_table.insert(result):
             print("---存储到数据库成功---",result)
     except Exception:
         print("---存储到数据库失败---",result)

6.完整代码

 # -*- coding: utf-8 -*-

 import requests
 import time
 import pymongo
 from lxml import etree
 from requests.exceptions import RequestException
 def get_one_page(page):
     '''获取单页源码'''
     try:
         url = "http://hz.ziroom.com/z/nl/z2.html?p=" + str(page)
         headers = {
             'Referer':'http://hz.ziroom.com/',
             'Upgrade-Insecure-Requests':'',
             'User-Agent':'Mozilla/5.0(WindowsNT6.3;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/68.0.3440.106Safari/537.36'
         }
         res = requests.get(url,headers=headers)
         if res.status_code == 200:
             return res.text
         return None
     except RequestException:
         return None
 def parse_one_page(sourcehtml):
     '''解析单页源码'''
     contentTree = etree.HTML(sourcehtml)   #解析源代码
     results = contentTree.xpath('//ul[@id="houseList"]/li') #利用XPath提取相应内容
     for result in results[1:]:
         title = result.xpath("./div/h3/a/text()")[0][5:] if len(result.xpath("./div/h3/a/text()")[0]) > 5 else ""
         location = result.xpath("./div/h4/a/text()")[0].replace("[", "").replace("]", '')
         area = " ".join(result.xpath("./div/div/p[1]/span/text()")).replace(" ", "", 1)  # 使用join方法将列表中的内容以" "字符连接
         #nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip()这里需要加判断, 改写为下句
         nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip() if len(result.xpath("./div/div/p[2]/span/text()"))>0 else ""
         data = {
         "title": title,
         "location": location,
         "area": area,
         "nearby": nearby
         }
         save_to_mongodb(data)
     #yield {"pages":pages}
 def get_pages():
     """得到总页数"""
     page = 1
     html = get_one_page(page)
     contentTree = etree.HTML(html)
     pages = int(contentTree.xpath('//div[@class="pages"]/span[2]/text()')[0].strip("共页"))
     return pages
 def save_to_mongodb(result):
     """存储到MongoDB中"""
     # 创建数据库连接对象, 即连接到本地
     client = pymongo.MongoClient(host="localhost")
     # 指定数据库,这里指定ziroom
     db = client.iroomz
     # 指定表的名称, 这里指定roominfo
     db_table = db.roominfo
     try:
         #存储到数据库
         if db_table.insert(result):
             print("---存储到数据库成功---",result)
     except Exception:
         print("---存储到数据库失败---",result)

 def main():
     pages = get_pages()
     print(pages)
     for page in range(1,pages+1):
         html = get_one_page(page)
         parse_one_page(html)

 if __name__ == '__main__':
     main()
     time.sleep(1)

点击查看

7.最终结果

总结

1. 在第三步中XPath使用注意事项

title = result.xpath("./div/h3/a/text()")
此处的点'.'不能忘记, 它表示当前节点, 如果不加'.', '/'就表示从根节点开始选取

2. 在第四步获取多个页面时出现索引超出范围错误

nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip()

IndexError: list index out of range

造成这种错误原因有两种:

1) [index] index超出list范围

2) [index] index索引内容为空

因为这里的nearby的index是0, 排除第一种情况, 那么这里就是空行了, 加句if判断就可以解决

nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip()
#改写以后:
nearby = result.xpath("./div/div/p[2]/span/text()")[0].strip() if len(result.xpath("./div/div/p[2]/span/text()"))>0 else ""

以上主要是对爬虫过程学习的总结, 若有不对的地方, 还请指正, 谢谢!