IMDB TOP 250爬虫
这个小学期Python大作业搞了个获取IMDB TOP 250电影全部信息的爬虫。第二次写爬虫,比在暑假集训时写的熟练多了。欢迎大家评论。
'''
************************************************
*Time:2017.9.11
*Target:All movies' information of IMDB TOP_250
*Resources:http://www.imdb.cn/IMDB250/
************************************************
''' import re
import requests
import numpy as np
import matplotlib.pyplot as plt
from bs4 import BeautifulSoup num = 1 #电影计数
All_txt = [] #全部电影的信息
headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'}#浏览器代理
def getHTMLText(url):
try:
#print(url)
r = requests.get( url,headers = headers )
#print(r)
r.encoding = 'utf-8'
return r.text
except:
return "错误" #从每一部电影的页面中获取全部信息
def get_all_information(url,page):
global num,All_txt
txt = getHTMLText(url)
if txt != "错误":
print('page'+str(page)+' NO.'+str(num)+' Get it!')
if num == 247:
print('Finished!!!')
soup = BeautifulSoup(txt,"html.parser")
Cname,Ename,Score,title,Actor,Starring,Infor = '','','','','','','' #TOP250-film_Chinese_name&Score
infor_1 = soup.find_all('div',class_ = 'hdd')
rel = '<h3>'+'[\s\S]*?'+'</h3>'
pattern = re.compile(rel)
Cname = ''.join(pattern.findall(str(infor_1[0])))
Cname = Cname.replace('<h3>','').replace('</h3>','')
#print(Cname)
#find_the_year & save
rel = '('+'[\s\S]*?'+')'
pattern = re.compile(rel)
time_ = ''.join(pattern.findall(Cname))
#print(time_)
with open('time.txt','a',encoding='utf-8') as t:
t.write( time_.replace('(','').replace(')','') + '\n' )
#find_Score
rel = '<i>'+'[\s\S]*?'+'</i>'
pattern = re.compile(rel)
Score = ''.join(pattern.findall(str(infor_1[0])))
Score = Score.replace('<i>','').replace('</i>','')
#print(Cname,Score) #TOP250-film_many_infor
now = soup.find_all('div',class_ = 'bdd clear')
#print(now[0])
a = BeautifulSoup(str(now[0]), "html.parser")
many_infor = a.find_all('li') #TOP250-film_Ename
Ename = str(many_infor[0]).replace('<li>','').replace('<i>','').replace('</i>','').replace('</li>','').replace('<a>','').replace('</a>','')
#TOP250-film_Actor
Actor_temp = BeautifulSoup(str(many_infor[2]), "html.parser").find_all('a')
Actor = Actor_temp[0].get_text().replace('导演:','')
#TOP250-film_Starring
Starring_temp = BeautifulSoup(str(many_infor[3]), "html.parser").find_all('a')
for i in Starring_temp:
Starring += i.get_text().replace(' ','') + ' '
#print(Starring) #Top-film_Infor
for j in range(4,7):
Infor_temp = BeautifulSoup(str(many_infor[j]), "html.parser")
for i in Infor_temp.children:
Infor += i.get_text().replace(' ','') + ' '
Infor += '\n'
#print(Infor) #TOP250-film_Synopsis
content = soup.find_all('div',class_ = 'fk-4 clear')
#print(content)
soup_con = BeautifulSoup(str(content[0]), "html.parser")
title = soup_con.find_all('div',class_ = 'hdd')
title = str(title[0]).replace('<div class="hdd">','').replace('</div>','\n')
#print(title)
content_1 = soup_con.find_all('div',class_ = 'bdd clear')
content_1 = str(content_1[0]).replace('<div class="bdd clear" style="font-size:15px">','').replace('</div>','')
content_1 = content_1.replace('<!-- <p><a href="#">更多剧情 >></a></p> -->','').replace('<br/>','\n') #Save_all_information
All_txt.append('第'+str(num)+'部'+'\n')
All_txt.append( Cname+'\n' )
All_txt.append( '【英文名】'+Ename+'\n' )
All_txt.append( '【评分】'+Score+'\n' )
All_txt.append( '【导演】'+Actor+'\n' )
All_txt.append( '【主演】'+Starring+'\n' )
All_txt.append( Infor+'\n' )
All_txt.append( title+'\n'+content_1+'\n' )
All_txt.append('\n')
num += 1 #在每一页中得到当前页的全部电影的url
def getin_one(url,page):
txt = getHTMLText(url)
soup = BeautifulSoup(txt, "html.parser")
#print(soup)
temp = soup.find_all('div',class_="ss-3 clear")
rel = '<a href="' + '[\s\S]*?' + '">'
pattern = re.compile(rel)
All_url = pattern.findall( str(temp[0]) )
for i in range(len(All_url)):
temp_url = 'http://www.imdb.cn'+All_url[i].replace('<a href="','').replace('">','')
get_all_information(temp_url,page)
#print(All_url) #将所有电影的年份统计并生成条形图
def Analyze_some_infor():
plt.rc('font', family='SimHei', size=13)#字体及大小
#Analyze_time
file = open('time.txt')
a,b,c,d,e,f = 0,0,0,0,0,0
for line in file:
line = eval(line)
if line == 0:
f += 1
elif line < 1940 and line >= 1920:
a += 1
elif line < 1960 and line >= 1940:
b += 1
elif line < 1980 and line >= 1960:
c += 1
elif line < 2000 and line >= 1980:
d += 1
else:
e += 1
times = [a,b,c,d,e,f]
range_time = ['1920-1940','1940-1960','1960-1980','1980-2000','2000-现在','无信息']
idx = np.arange(len(range_time))
width = 0.5
plt.bar(idx,times,width,color='green')
plt.xticks(idx+width/2, range_time, rotation=40)
plt.xlabel('电影年代')
plt.ylabel('数目')
plt.savefig('time_pic.jpg')
plt.show() def main():
global All_txt
getin_one('http://www.imdb.cn/IMDB250/',1)
for i in range(2,10):
getin_one( 'http://www.imdb.cn/imdb250/'+str(i) , i )
#将已有内容清空
with open('All_infor.txt','w',encoding='utf-8') as x:
pass
with open('All_infor.txt','a',encoding='utf-8') as x:
for i in All_txt:
x.write(i)
Analyze_some_infor() main()
作者: LB919
出处:http://www.cnblogs.com/L1B0/
该文章为LB919投入了时间和精力的原创;
如有转载,荣幸之至!请随手标明出处;
IMDB TOP 250爬虫的更多相关文章
- 吐血推荐250部必看电影下载 IMDB TOP 250 download
中文名: IMDB Top 250合辑 TLF-MiniSD收藏版英文名: IMDB Top 250 TLF-MiniSD Collection版本: (更新至TOP119)[MiniSD]发行日期: ...
- Douban Top 250爬虫
# Ref: https://fishc.com.cn/forum.php?mod=viewthread&tid=101887&extra=page%3D1%26filter%3Dty ...
- 爬取豆瓣电影TOP 250的电影存储到mongodb中
爬取豆瓣电影TOP 250的电影存储到mongodb中 1.创建项目sp1 PS D:\scrapy> scrapy.exe startproject douban 2.创建一个爬虫 PS D: ...
- 用python爬取豆瓣电影Top 250
首先,打开豆瓣电影Top 250,然后进行网页分析.找到它的Host和User-agent,并保存下来. 然后,我们通过翻页,查看各页面的url,发现规律: 第一页:https://movie.dou ...
- Python开发爬虫之静态网页抓取篇:爬取“豆瓣电影 Top 250”电影数据
所谓静态页面是指纯粹的HTML格式的页面,这样的页面在浏览器中展示的内容都在HTML源码中. 目标:爬取豆瓣电影TOP250的所有电影名称,网址为:https://movie.douban.com/t ...
- Python爬虫-豆瓣电影 Top 250
爬取的网页地址为:https://movie.douban.com/top250 打开网页后,可观察到:TOP250的电影被分成了10个页面来展示,每个页面有25个电影. 那么要爬取所有电影的信息,就 ...
- 豆瓣电影 Top 250
import refrom urllib.request import urlopen def getPage(url): # 获取网页的字符串 response = urlopen(url) ret ...
- IMDB-TOP_250-爬虫
这个小学期Python大作业搞了个获取IMDB TOP 250电影全部信息的爬虫.第二次写爬虫,比在暑假集训时写的熟练多了.欢迎大家评论. ''' ************************** ...
- Livid : 在 26 岁时写给 18 岁的自己
转载自: https://livid.v2ex.com/essays/2012/01/24/a-letter-from-26-to-18.html 在 26 岁时写给 18 岁的自己 Jan 24, ...
随机推荐
- Hadoop2.7.3集群搭建
hadoop2.0已经发布了稳定版本了,增加了很多特性,比如HDFS HA.YARN等.最新的hadoop-2.4.1又增加了YARN HA 注意:apache提供的hadoop-2.4.1的安装 ...
- win10 uwp 异步进度条
本文主要讲我设计的几个进度条,还有如何使用异步控制进度条,如何使用动画做进度. 进度条可以参见:http://edi.wang/post/2016/2/25/windows-10-uwp-modal- ...
- 系统装机硬盘格式 >> GPT或者UEFI
预装Win8系统的电脑,硬盘都是采用这种分区格式,因为出厂安装时,是以Uefi方式启动安装的. 简单的办法,仍安装Win8系统:或者是转换磁盘分区格式为MBR,不用任何软件就可实现,需要重建分区表,会 ...
- mysql单独可连接,php连接mysql失败之 Can't connect to local MySQL server through socket '/var/lib/mysql/mysql.sock' (2)
此种解决方案使用场景: 1,mysql单独可以启动而且远程工具也可以连接 2,php无法连接. 3,find / -name mysql.sock 可以找到文件路径 4,报错 Can't connec ...
- javascrip实现:若选中TreeView的父节点checkbox,则其子节点全部选中;子节点全部没选中,则父节点也会没选中。
<script type="text/javascript"> function public_GetParentByTagName(element, tagName) ...
- Fedora 19 搭建Qt环境
1.搭建桌面环境fedora的源里包含的需要的套件包,用下面命令安装sudo yum intall qt qt-devel qt-x11 qt-doc qt-demos qt-examples qt- ...
- iOS开发从申请账号到上线APP Store步骤
1.developer.apple.com 申请开发者账号 2.根据API Cloud创建证书: http://docs.apicloud.com/Dev-Guide/iOS-License-Appl ...
- linux root修改密码失败
问题: 当使用root修改密码时,报错passwd:Authentication token manipulation error 解决: 1.查看是否权限问题, /etc/passwd /etc/s ...
- php语法标记风格
1.xml风格(标准风格推荐使用) 复制代码 代码如下: <?php echo"这是xml风格的标记"; ?> xml风格的标记是常用的标记,也是推荐使用的标记,服务器 ...
- 读书笔记-你不知道的JS中-promise(2)
继续填坑 模式 考虑下面的代码: function fn(x) { //do something return new Promise(function(resolve, reject) { //调用 ...