[Python] 快速爬取当前城市所有租房网站房源及配置，一目了然

Python爬取当前城市房源信息，以徐州为例

代码效果图请看下方，其他部分请查看附件，一起学习，谢谢

1. # -*- coding: utf-8 -*-
2. """
3. @Time ： 2020/3/18 22:23
4. @Auth ： Suk
5. @File ： 5.小猪短租相关信息.py
6. @IDE ： PyCharm
7. @Motto： Knowing your ignorance is the best way to succeed.
8. @Tips ： 版权所有，转载，转发请注明，如有侵权请联系，谢谢.
9. """
10. # 小猪短租相关信息，包含出租房屋名称、地址、价格、房东、详细链接等信息
11. # 爬取搜索页面信息，爬取5页相关内容,通过获得的详细链接页面，爬取详细页面内容
12. import bs4
13. import requests
14. from bs4 import BeautifulSoup
15. kv = {
16. 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36 Edg/83.0.478.45'
17. }
18. # format='{0:<10}\t{1:{6}<40}\t{2:{6}<10}\t{3:{6}^10}\t{4:{6}<10}\t{5:{6}<10}'
19. format = '{0:<10}\t{1:{5}<40}\t{2:{5}<10}\t{3:{5}^30}\t{4:{5}<10}'
20. def main():
21. global a
22. print('为您找到了{}条信息，您可以根据需要检索相关信息'.format(len(allInitMessage)))
23. print('-------------------------------------------------------------------------')
24. print(
25. '序号 房屋名称 价格 地址 房东 ')
26. for i in allInitMessage:
27. # print(format.format(i[0],i[1],i[2],i[3],i[4],i[5],chr(12288)))
28. print(format.format(i[0], i[1], i[2], i[3], i[4], chr(12288)))
29. a = eval(input('请输入你想了解的房屋序号：'))
30. return a
31. def houseDetails(num):
32. deurl = allInitMessage[num - 1][-1]
33. detailRe = requests.get(deurl, headers=kv)
34. sp = BeautifulSoup(detailRe.text, 'lxml')
35. print('标题：' + sp.title.string.strip())
36. print("价格：" + sp.find('div', class_='fl').text.strip())
37. print("地址：" + sp.find('span', class_='pr5').text.strip())
38. print("房东：" + sp.find('a', class_='lorder_name').text.strip())
39. print('详细信息：')
40. for i in sp.find('ul', class_='house_info clearfix').text.split():
41. if '：' in i:
42. print("\t" + i)
43. elif "宜住" in i:
44. print("\t" + i)
45. print("个性描述：")
46. for i in sp.find('div', class_='box_white clearfix detail_intro_item').text.split():
47. if "个性描述" in i or "查看全部" in i or "收起" in i:
48. continue
49. else:
50. print("\t" + i)
51. print("内部情况：")
52. for i in sp.find('div', class_='box_gray clearfix detail_intro_item').text.split():
53. if "内部情况" in i or "查看全部" in i or "收起" in i:
54. continue
55. else:
56. print("\t" + i)
57. print("交通情况：")
58. for i in sp.findAll('div', class_='info_r')[2].text.split():
59. if "交通信息" in i or "交通情况" in i or "查看全部" in i or "收起" in i:
60. continue
61. else:
62. print("\t" + i)
63. print("周边情况：")
64. for i in sp.findAll('div', class_='info_r')[3].text.split():
65. if "交通信息" in i or "查看全部" in i or "收起" in i:
66. continue
67. else:
68. print("\t" + i)
69. print("配套条件：")
70. print('\t', end="")
71. for i in sp.findAll('div', class_='info_r')[4].children:
72. if type(i) == bs4.element.Tag:
73. if i.ul != None:
74. for j in i.ul.children:
75. if type(j) == bs4.element.Tag:
76. if 'no' in j.get('class')[0]:
77. continue
78. else:
79. print(j.text.strip() + ',', end="")
80. print('\n' + "入住须知：")
81. for i in sp.findAll('div', class_='info_r')[5].text.split():
82. print('\t' + i)
83. print("押金及其他费用")
84. for i in sp.find('div', class_='clause_box').text.split():
85. if "押金及其他费用" in i or "查看全部" in i or "收起" in i or ">" in i:
86. continue
87. elif i[-1] == "：":
88. print("\t" + i, end="")
89. else:
90. print("\t" + i)
91. try:
92. select = input('是否返回主页面(y/n)?')
93. if select in 'yY':
94. main()
95. return True
96. if select in 'nN':
97. print("退出成功！")
98. return False
99. except:
100. print("ERROR！输入错误！")
101. url = 'http://xuzhou.xiaozhu.com/search-duanzufang-p'
102. allhref = []
103. allInitMessage = []
104. for page in range(1, eval(input('当前城市：徐州\t请输入你想检索的页数，共13页：')) + 1):
105. lis = []
106. hrefs = []
107. print('\t正在检索第' + str(page) + '页')
108. re = requests.get(url + str(page) + '-0/')
109. soup = BeautifulSoup(re.text, 'lxml')
110. for i in soup.find('ul', class_='pic_list clearfix list_code').children:
111. if type(i) == bs4.element.Tag:
112. lis.append(i.find("a"))
113. for i in lis:
114. if i != None:
115. hrefs.append(i.get('href'))
116. for i in hrefs:
117. message = []
118. innitMessageget = requests.get(i)
119. innitMessage = BeautifulSoup(innitMessageget.text, 'lxml')
120. message.append(str(len(allInitMessage) + 1) + '.')
121. message.append(innitMessage.title.string)
122. message.append(innitMessage.find('div', class_='fl').text.strip())
123. message.append(innitMessage.find('span', class_='pr5').text.strip())
124. message.append(innitMessage.find('a', class_='lorder_name').text.strip())
125. message.append(i)
126. allInitMessage.append(message)
127. allhref.append(hrefs)
128. print("\n\n\tMission Success!\n\n\n")
129. main()
130. while (True):
131. bolean = houseDetails(a)
132. if bolean == False:
133. exit(0)
134. else:
135. continue