Python3.5爬取cbooo.cn数据并且同步到mysql中
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
# Python: 3.5
# Author: wucl(),zhenghai.zhang
# Program: 爬取CBO网站上所有电影的名称并写入数据库。
# Version: 0.1
# History: 2017.10.25 import requests,time, pymysql, re, datetime
from exchangelib import DELEGATE, Account, Credentials, Message, Mailbox, HTMLBody host = 'xxx'
user = 'xxx'
passwd = 'xxx'
dbme = 'crawl'
dbtarget = 'back_brace'
table = 'movie_hotwords'
tabledelta = 'movie_hotwords_delta'
tablesync = 'slot_value'
port = 3306
tolist = ['xxx@xxx.com'] def get_info():
try:
url = 'http://www.cbooo.cn/Mdata/getMdata_movie?area=50&type=0&year=0&initial=%E5%85%A8%E9%83%A8&pIndex=1'
pData = requests.get(url).json()
return pData['tPage'], pData['tCount']
except:
print("获取总页数和总电影数失败") def get_movies(page):
try:
url = 'http://www.cbooo.cn/Mdata/getMdata_movie?area=50&type=0&year=0&initial=%E5%85%A8%E9%83%A8&pIndex=' + str(page)
pData = requests.get(url).json()
movies_list = pData['pData']
return movies_list
except:
print('获取第%s页电影列表失败' % page) def Movie_insert(host, user, passwd, dbme, port, table, movies_list):
conn=pymysql.connect(host=host, user=user, passwd=passwd, db=dbme, port=port, charset="utf8")
cur=conn.cursor()
new_movies = []
punc = "!?。"#$%&'()*+,-/:;<=>@[\]^_`{|}~⦅⦆「」、、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.()::。·"
punctuation = punc
for movie in movies_list:
try:
movie['MovieName'] = re.sub(r"[%s]+" % punctuation, "", movie["MovieName"])
cmd = 'insert into %s(movie_id, movie_name) values("%s", "%s")' % (table, movie['ID'], movie['MovieName'])
cur.execute(cmd)
new_movies.append(movie)
except pymysql.Error:
print(" "*20, movie['MovieName'], "already exists, skip……")
cur.close()
conn.commit()
conn.close()
return new_movies def Movie_new_and_sync(host, user, passwd, dbme, dbtarget, port, tabledelta, movies_list, tablesync):
conn = pymysql.connect(host=host, user=user, passwd=passwd, db=dbme, port=port, charset="utf8")
cur = conn.cursor()
cur.execute("delete from %s " % dbme+"."+tabledelta) for movie in movies_list:
try:
cmd = 'insert into %s(movie_id, movie_name) values("%s", "%s")' % (tabledelta, movie['ID'], movie['MovieName'])
cmdsync = 'insert into %s(slot_type_id, slot_value, create_by, modify_by, gmt_create, gmt_modify, out_value) values("%s", "%s", "%s", "%s", "%s", "%s", "%s")' % (dbtarget+"."+tablesync, "xxxxxx", movie['MovieName'], "system", "system", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),"")
cur.execute(cmd)
cur.execute(cmdsync)
except pymysql.Error:
print(" " * 20, movie['MovieName'], "already exists, skip……") try:
cmdbacktoskill = 'insert into back_brace.release_task(app_type,app_status,type,ref_id,status,register_id,create_by,modify_by,gmt_create,gmt_modify) values("BACKBRACE","testpass","SLOT","xxxxxx","init","SLOT_BACKBRACE_TESTPASS" ,"zhenghai.zhang","zhenghai.zhang","%s","%s")' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
cmdskilltoskillpro = 'insert into back_brace.release_task(app_type,app_status,type,ref_id,status,register_id,create_by,modify_by,gmt_create,gmt_modify) values("SKILL","deploy","SLOT","xxxxxx","init","SLOT_SKILL_DEPLOY" ,"zhenghai.zhang","zhenghai.zhang","%s","%s")' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) print(cmdbacktoskill)
cur.execute(cmdbacktoskill)
print(cmdskilltoskillpro)
cur.execute(cmdskilltoskillpro)
except pymysql.Error:
print("write into back_brace.release_task error!!!")
cur.close()
conn.commit()
conn.close() def Email(to, subject, body):
creds = Credentials(
username='xxxxxx',
password='xxxxxx')
account = Account(
primary_smtp_address='xxx@xxx.com',
credentials=creds,
autodiscover=True,
access_type=DELEGATE)
m = Message(
account=account,
subject=subject,
body=HTMLBody(body),
to_recipients=[Mailbox(email_address=to)])
m.send_and_save() if __name__ == '__main__':
update_movies = []
pages, counts = get_info()
pages = 1
for i in range(1,pages + 1):
print("*"*30,i,"*"*30)
movies_list = get_movies(i)
new_movies = Movie_insert(host, user, passwd, dbme, port, table, movies_list)
for new_movie in new_movies:
print(new_movie['MovieName'],"Added")
onemovie = {}
onemovie["ID"] = new_movie["ID"]
onemovie["MovieName"] = new_movie["MovieName"]
update_movies.append(onemovie)
time.sleep(1)
print(update_movies) try:
Movie_new_and_sync(host, user, passwd, dbme, dbtarget, port, tabledelta, update_movies, tablesync) # 将增加的电影写入movie_hotwords_delta表中
except:
print("Movie update and sync Error!") subject = '本次新增电影名称'
body = "本次新增的电影名称为:<hr>"
for movie in update_movies:
body += movie["MovieName"] + "<br>"
for to in tolist:
Email(to, subject, body)
欢迎大侠指点
Python3.5爬取cbooo.cn数据并且同步到mysql中的更多相关文章
- Python3.5爬取豆瓣电视剧数据并且同步到mysql中
#!/usr/local/bin/python # -*- coding: utf-8 -*- # Python: 3.5 # Author: zhenghai.zhang@xxx.com # Pro ...
- node 爬虫 --- 将爬取到的数据,保存到 mysql 数据库中
步骤一:安装必要模块 (1)cheerio模块 ,一个类似jQuery的选择器模块,分析HTML利器. (2)request模块,让http请求变的更加简单 (3)mysql模块,node连接mysq ...
- Python爬取新浪微博评论数据,写入csv文件中
因为新浪微博网页版爬虫比较困难,故采取用手机网页端爬取的方式 操作步骤如下: 1. 网页版登陆新浪微博 2.打开m.weibo.cn 3.查找自己感兴趣的话题,获取对应的数据接口链接 4.获取cook ...
- 一个简易的Python爬虫,将爬取到的数据写入txt文档中
代码如下: import requests import re import os #url url = "http://wiki.akbfun48.com/index.php?title= ...
- Python3从零开始爬取今日头条的新闻【三、滚动到底自动加载】
Python3从零开始爬取今日头条的新闻[一.开发环境搭建] Python3从零开始爬取今日头条的新闻[二.首页热点新闻抓取] Python3从零开始爬取今日头条的新闻[三.滚动到底自动加载] Pyt ...
- Python3:爬取新浪、网易、今日头条、UC四大网站新闻标题及内容
Python3:爬取新浪.网易.今日头条.UC四大网站新闻标题及内容 以爬取相应网站的社会新闻内容为例: 一.新浪: 新浪网的新闻比较好爬取,我是用BeautifulSoup直接解析的,它并没有使用J ...
- 吴裕雄--天生自然python数据清洗与数据可视化:MYSQL、MongoDB数据库连接与查询、爬取天猫连衣裙数据保存到MongoDB
本博文使用的数据库是MySQL和MongoDB数据库.安装MySQL可以参照我的这篇博文:https://www.cnblogs.com/tszr/p/12112777.html 其中操作Mysql使 ...
- Python爬取招聘网站数据,给学习、求职一点参考
1.项目背景 随着科技的飞速发展,数据呈现爆发式的增长,任何人都摆脱不了与数据打交道,社会对于“数据”方面的人才需求也在不断增大.因此了解当下企业究竟需要招聘什么样的人才?需要什么样的技能?不管是对于 ...
- Java爬取同花顺股票数据(附源码)
最近有小伙伴问我能不能抓取同花顺的数据,最近股票行情还不错,想把数据抓下来自己分析分析.我大A股,大家都知道的,一个概念火了,相应的股票就都大涨. 如果能及时获取股票涨跌信息,那就能在刚开始火起来的时 ...
随机推荐
- 虚拟机virtualbox中挂载新硬盘
在virtualbox中装好Ubuntu后,发现硬盘空间太小,怎样才能增加硬盘容量?那就是再建一个硬盘: 1. 添加新硬盘 设置 -> Storage -> SATA控制器->右击, ...
- C#项目中引入app.manifest管理员权限运行
原文地址:https://blog.csdn.net/qq395537505/article/details/51010962 参考UAC:User Account Control 参考IE保护模式: ...
- Easyui combobox onChange事件
Easyui combobox onChange事件: 注册事件: $(function () { $('#cc_id').combobox({ onChange: function (newValu ...
- session的取代者:Json Web Tokens----在客户端存储登陆状态
https://auth0.com/blog/2014/01/07/angularjs-authentication-with-cookies-vs-token/
- 一个酷绚的linux 桌面程序 GLX-DOCK (cario-dock)
记录一个酷绚的linux 桌面程序 GLX-DOCK (cario-dock),支持多种风格的桌面主题. http://glx-dock.org/ 优势: 多个workspaces 方便自由切换 ...
- JAVA Socket编程和C++ Socket编程有什么不同
原文链接: http://zhidao.baidu.com/link?url=16TEzhom2Nr8x1_2uTRp-e2pgZRgS5nW5ywtRX2XLHbtLOG8btif5DTyP85jf ...
- SharePoint 2013怎样创建Wiki库
们使用Wiki页面来分享知识,增进团队合作.下面我将向大家展示SharePoint 2013 Wiki的使用方法.教程我都将以这张Wiki页面(即当前页)为示例. 编辑页面 如要编辑页面,单击顶部Ed ...
- 《自己动手写框架2》:用200行的DBF解析器来展示良好架构设计
因为工作关系.须要工作其中,须要读取DBF文件.找了一些DBF读取开源软件,要么是太过庞大,动不动就上万行.要么是功能有问题,编码,长度,总之是没有找到一个很爽的. 在万般无奈之下,我老人家怒从心头起 ...
- mysql存储过程批量向表插入数据
业务需要,往某个表中批量插入数据,使用存储过程插入 首先,要建立一张mysql表,表明为phone_number, 三个字段,id 自增,number 就是要插入的表格,is_used 表示十分已经使 ...
- pycharm + selenium + python 提示 Unresolved reference 'webdriver' 解决办法
尝试使用python + selenium + pycharm 做自动化测试, 命令行pip install selenium 安装了selenium.但是使用pycharm 新建一个测试项目后并新建 ...