增量式爬虫 Scrapy-Rredis 详解及案例
1、创建scrapy项目命令
scrapy startproject myproject
2、在项目中创建一个新的spider文件命令：
scrapy genspider mydomain mydomain.com #mydomain为spider文件名，mydomain.com为爬取网站域名
3、运行项目命令
scrapy crawl <spider>
scrapy runspider <spider_file.py> #运行spider第二种方法
4、检查spider文件有无语法错误
scrapy check
5、其他的语法
scrapy crawl <spider> --nolog #运行spider文件 不显示日志
scrapy list #列出spider路径下的spider文件
scrapy fetch <url> #将网页内容下载下来，然后在终端打印当前返回的内容，相当于 request 和 urllib 方法
scrapy view <url> #将网页内容保存下来，并在浏览器中打开当前网页内容，直观呈现要爬取网页的内容
scrapy shell [url] #打开 scrapy 显示台，类似ipython，可以用来做测试
scrapy parse <url> [options] #输出格式化内容
scrapy settings [options] #返回系统设置信息
scrapy bench #测试电脑当前爬取速度性能
1、以当当网为例爬取信息如下，首先文件的结构目录如下所示
E:.
│ dangdang_content.sql
│ scrapy.cfg
│
├─.idea
│ │ misc.xml
│ │ modules.xml
│ │ ScrapyRedisPro.iml
│ │ workspace.xml
│ │
│ └─libraries
│ R_User_Library.xml
│
└─ScrapyRedisPro
│ items.py
│ middlewares.py
│ pipelines.py
│ settings.py
│ test.py
│ __init__.py
│
├─.idea
│ │ misc.xml
│ │ modules.xml
│ │ ScrapyRedisPro.iml
│ │ workspace.xml
│ │
│ └─libraries
│ R_User_Library.xml
│
├─spiders
│ │ test.py
│ │ __init__.py
│ │
│ └─__pycache__
│ test.cpython-36.pyc
│ __init__.cpython-36.pyc
│
└─__pycache__
items.cpython-36.pyc
middlewares.cpython-36.pyc
pipelines.cpython-36.pyc
settings.cpython-36.pyc
__init__.cpython-36.pyc
2、mysql数据库数据结构
SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;
-- ----------------------------
-- Table structure for dangdang_content
-- ----------------------------
DROP TABLE IF EXISTS `dangdang_content`;
CREATE TABLE `dangdang_content` (
`id` int NOT NULL AUTO_INCREMENT,
`b_cate` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`m_cate` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`s_href` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`s_cate` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_img` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_desc` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_price` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_author` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_publish_date` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`book_press` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL,
`insert_data` timestamp(0) NULL DEFAULT CURRENT_TIMESTAMP(0) ON UPDATE CURRENT_TIMESTAMP(0),
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic;
SET FOREIGN_KEY_CHECKS = 1;
1、spider 爬虫项目文件test.py 如下所示
# -*- coding: utf-8 -*-
import scrapy
from scrapy_redis.spiders import RedisCrawlSpider
from ScrapyRedisPro.items import ScrapyredisproItem
from copy import deepcopy
import urllib
class TestSpider(RedisCrawlSpider):
name = 'test'
# allowed_domains = ['www.baidu.com']
# start_urls = ['http://www.baidu.com/']
#调度器队列名称
redis_key = 'dangdang'
def parse(self, response):
"""
逻辑分析
1.通过抓取下一页的链接，交给scrapy实现自动翻页,如果没有下一页则爬取完成
2.将本页面的所有文章url爬下，并交给scrapy进行深入详情页的爬取
"""
# 大分类分组
div_list = response.xpath("//div[@class='con flq_body']/div")
for div in div_list:
item = {}
item["b_cate"] = div.xpath("./dl/dt//text()").extract()
item["b_cate"] = [i.strip() for i in item["b_cate"] if len(i.strip()) > 0]
# 中间分类分组
dl_list = div.xpath("./div//dl[@class='inner_dl']")
for dl in dl_list:
item["m_cate"] = dl.xpath("./dt//text()").extract()
item["m_cate"] = [i.strip() for i in item["m_cate"] if len(i.strip()) > 0][0]
# 小分类分组
a_list = dl.xpath("./dd/a")
for a in a_list:
item["s_href"] = a.xpath("./@href").extract_first()
item["s_cate"] = a.xpath("./text()").extract_first()
if item["s_href"] is not None:
# print(item)
yield scrapy.Request(
item["s_href"],
callback=self.parse_book_list,
meta={"item": deepcopy(item)}
)
def parse_book_list(self, response):
"""
将爬虫爬取的数据送到item中进行序列化
这里通过ItemLoader加载item
"""
item = response.meta["item"]
li_list = response.xpath("//ul[@class='bigimg']/li")
for li in li_list:
item["book_img"] = li.xpath("./a[@class='pic']/img/@src").extract_first()
if item["book_img"] == "images/model/guan/url_none.png":
item["book_img"] = li.xpath("./a[@class='pic']/img/@data-original").extract_first()
item["book_name"] = li.xpath("./p[@class='name']/a/@title").extract_first()
item["book_desc"] = li.xpath("./p[@class='detail']/text()").extract_first()
item["book_price"] = li.xpath(".//span[@class='search_now_price']/text()").extract_first()
item["book_author"] = li.xpath("./p[@class='search_book_author']/span[1]/a/text()").extract()
item["book_publish_date"] = li.xpath("./p[@class='search_book_author']/span[2]/text()").extract_first()
item["book_press"] = li.xpath("./p[@class='search_book_author']/span[3]/a/text()").extract_first()
# print(item)
yield item
# 下一页
next_url = response.xpath("//li[@class='next']/a/@href").extract_first()
if next_url is not None:
next_url = urllib.parse.urljoin(response.url, next_url)
yield scrapy.Request(
next_url,
callback=self.parse_book_list,
meta={"item": item}
)
Middlewware.py 中间件文件
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
from scrapy.http import HtmlResponse
import time
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
import random
class ScrapyredisproSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class ScrapyredisproDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
# 拦截到响应对象（下载器传递给Spider的响应对象）
# request：响应对象对应的请求对象
# response：拦截到的响应对象
# spider：爬虫文件中对应的爬虫类的实例
def process_response(self, request, response, spider):
# 响应对象中存储页面数据的篡改
if request.url in ['http://news.163.com/domestic/', 'http://news.163.com/world/', 'http://news.163.com/air/',
'http://war.163.com/']:
spider.bro.get(url=request.url)
js = 'window.scrollTo(0,document.body.scrollHeight)'
spider.bro.execute_script(js)
time.sleep(3) # 一定要给与浏览器一定的缓冲加载数据的时间
# 页面数据就是包含了动态加载出来的新闻数据对应的页面数据
page_text = spider.bro.page_source
# 篡改响应对象
return HtmlResponse(url=spider.bro.current_url, body=page_text, encoding='utf-8', request=request)
else:
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
#UA池代码的编写（单独给UA池封装一个下载中间件的一个类）
#1，导包UserAgentMiddlware类
class RandomUserAgent(UserAgentMiddleware):
def process_request(self, request, spider):
#从列表中随机抽选出一个ua值
ua = random.choice(user_agent_list)
#ua值进行当前拦截到请求的ua的写入操作
request.headers.setdefault('User-Agent',ua)
#批量对拦截到的请求进行ip更换
class Proxy(object):
def process_request(self, request, spider):
#对拦截到请求的url进行判断（协议头到底是http还是https）
#request.url返回值：http://www.xxx.com
h = request.url.split(':')[0] #请求的协议头
if h == 'https':
ip = random.choice(PROXY_https)
request.meta['proxy'] = 'https://'+ip
else:
ip = random.choice(PROXY_http)
request.meta['proxy'] = 'http://' + ip
PROXY_http = [
'153.180.102.104:80',
'195.208.131.189:56055',
]
PROXY_https = [
'120.83.49.90:9000',
'95.189.112.214:35508',
]
user_agent_list = [
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 "
"(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 "
"(KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 "
"(KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 "
"(KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 "
"(KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 "
"(KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 "
"(KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 "
"(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 "
"(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
]
3、items.py 文件
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class ScrapyredisproItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
b_cate = scrapy.Field()
m_cate = scrapy.Field()
s_href = scrapy.Field()
s_cate = scrapy.Field()
book_img = scrapy.Field()
book_name = scrapy.Field()
book_desc = scrapy.Field()
book_price = scrapy.Field()
book_author = scrapy.Field()
book_publish_date = scrapy.Field()
book_press = scrapy.Field()
4、pipelines.py 管道文件
注意 一般下载方式有两种，异步与同步下载方式（下列两种方式都有）
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymysql.cursors
from twisted.enterprise import adbapi
class ScrapyredisproPipeline(object):
# def process_item(self, item, spider):
# return item
# class MysqlTwistedPipeline(object):
'''
异步机制将数据写入到mysql数据库中
'''
# 创建初始化函数，当通过此类创建对象时首先被调用的方法
def __init__(self, dbpool):
self.dbpool = dbpool
# 创建一个静态方法,静态方法的加载内存优先级高于init方法，
# 在创建这个类的对之前就已将加载到了内存中，所以init这个方法可以调用这个方法产生的对象
@classmethod
# 名称固定的
def from_settings(cls, settings):
# 先将setting中连接数据库所需内容取出，构造一个地点
dbparms = dict(
host=settings["MYSQL_HOST"],
port=settings["MYSQL_PORT"],
db=settings["MYSQL_DBNAME"],
user=settings["MYSQL_USER"],
password=settings["MYSQL_PASSWORD"],
charset='utf8',
# 游标设置
cursorclass=pymysql.cursors.DictCursor,
# 设置编码是否使用Unicode
use_unicode=True
)
# 通过Twisted框架提供的容器连接数据库,pymysql是数据库模块名
dbpool = adbapi.ConnectionPool("pymysql",**dbparms)
print("连接成功！")
# 无需直接导入 dbmodule. 只需要告诉 adbapi.ConnectionPool 构造器你用的数据库模块的名称比如pymysql.
return cls(dbpool)
def process_item(self, item, spider):
# 使用Twisted异步的将Item数据插入数据库
query = self.dbpool.runInteraction(self.do_insert, item)
query.addErrback(self.handle_error, item, spider) # 这里不往下传入item,spider，handle_error则不需接受,item,spider)
def do_insert(self, cursor, item):
# 执行具体的插入语句,不需要commit操作,Twisted会自动进行
insert_sql = """
insert into dangdang_content(b_cate, m_cate, s_href, s_cate, book_img, book_name, book_desc, book_price, book_author, book_publish_date, book_press) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
"""
cursor.execute(insert_sql, (item["b_cate"][0], item["m_cate"], item["book_img"], item["s_href"], item["s_cate"], item["book_name"], item["book_desc"], item["book_price"], item["book_author"][0], item["book_publish_date"], item["book_press"]))
print("------------------------------数据插入成功！")
def handle_error(self, failure, item, spider):
# 异步插入异常
if failure:
print(failure)
# class MysqlTwistedPipeline(object):
# '''
# 同步步机制将数据写入到mysql数据库中
# '''
#
# # 创建初始化函数，当通过此类创建对象时首先被调用的方法
# def __init__(self, conn, cursor):
# self.conn = conn
# self.cursor = cursor
#
# # 创建一个静态方法,静态方法的加载内存优先级高于init方法，
# # 在创建这个类的对之前就已将加载到了内存中，所以init这个方法可以调用这个方法产生的对象
# @classmethod
# # 名称固定的
# def from_settings(cls, settings):
# # 先将setting中连接数据库所需内容取出，构造一个地点
# dbparms = dict(
# host=settings["MYSQL_HOST"],
# port=settings["MYSQL_PORT"],
# db=settings["MYSQL_DBNAME"],
# user=settings["MYSQL_USER"],
# password=settings["MYSQL_PASSWORD"],
# charset='utf8',
# # 游标设置
# cursorclass=pymysql.cursors.DictCursor,
# # 设置编码是否使用Unicode
# use_unicode=True
# )
# conn = pymysql.connect(**dbparms)
# cursor = conn.cursor()
# # 无需直接导入 dbmodule. 只需要告诉 adbapi.ConnectionPool 构造器你用的数据库模块的名称比如pymysql.
# return cls(conn, cursor)
#
# def process_item(self, item, spider):
# insert_sql = 'insert into dangdang_content(b_cate[0], m_cate, s_href, s_cate, book_img, book_name, book_desc, book_price, book_author, book_publish_date, book_press) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'.format(item["b_cate"], item["m_cate"], item["book_img"], item["s_href"], item["s_cate"], item["book_name"], item["book_desc"], item["book_price"], item["book_author"][0], item["book_publish_date"], item["book_press"])
# print(insert_sql)
# self.cursor.execute(insert_sql)
# self.conn.commit()
# print("------------------------------数据插入成功！")
#
# def handle_error(self, failure, item, spider):
# # 异步插入异常
# if failure:
# print(failure)
#
# def close_spider(self, spider):
# """
# 清理
# :param spider:
# :return:
# """
# self.cursor.close()
# self.conn.close()
5、settings.py 配置文件
# -*- coding: utf-8 -*-
# Scrapy settings for ScrapyRedisPro project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'ScrapyRedisPro'
SPIDER_MODULES = ['ScrapyRedisPro.spiders']
NEWSPIDER_MODULE = 'ScrapyRedisPro.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'ScrapyRedisPro (+http://www.yourdomain.com)'
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36' # 伪装请求载体身份
# Obey robots.txt rules
ROBOTSTXT_OBEY = False #可以忽略或者不遵守robots协议
#只显示指定类型的日志信息
LOG_LEVEL='ERROR'
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'ScrapyRedisPro.middlewares.ScrapyredisproSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
'ScrapyRedisPro.middlewares.ScrapyredisproDownloaderMiddleware': 543,
'ScrapyRedisPro.middlewares.RandomUserAgent': 542,
}
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'ScrapyRedisPro.pipelines.ScrapyredisproPipeline': 300,
'scrapy_redis.pipelines.RedisPipeline': 400 ,
# 'ScrapyRedisPro.pipelines.MysqlTwistedPipeline': 301,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
""" scrapy-redis配置 """
# Enables scheduling storing requests queue in redis.
# 使用scrapy-redis组件自己的调度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 增加了一个去重容器类的配置, 作用使用Redis的set集合来存储请求的指纹数据, 从而实现请求去重的持久化
# Ensure all spiders share same duplicates filter through redis.
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 配置调度器是否要持久化, 也就是当爬虫结束了, 要不要清空Redis中请求队列和去重指纹的set。如果是True, 就表示要持久化存储, 就不清空数据, 否则清空数据
SCHEDULER_PERSIST = True # 为false Redis关闭了 Redis数据也会被清空
# redis的配置
REDIS_HOST = '127.0.0.1'
REDIS_PORT = 6379
REDIS_ENCODING ='utf8'
REDIS_PARAMS = {'password':'xhw123'}
MYSQL_HOST = '127.0.0.1'
MYSQL_PORT = 3306
MYSQL_DBNAME = 'spidertest'
MYSQL_USER = 'root'
MYSQL_PASSWORD = 'xhw123'
# 增加了一个去重容器类的配置, 作用使用Redis的set集合来存储请求的指纹数据, 从而实现请求去重的持久化
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 使用scrapy-redis组件自己的调度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 配置调度器是否要持久化, 也就是当爬虫结束了, 要不要清空Redis中请求队列和去重指纹的set。如果是True, 就表示要持久化存储, 就不清空数据, 否则清空数据
SCHEDULER_PERSIST = True
注意 启动项目后需要在redis的客户端执行如下命令
127.0.0.1:6379> lpush dangdang http://book.dangdang.com/
增量式爬虫 Scrapy-Rredis 详解及案例的更多相关文章
- python爬虫scrapy项目详解（关注、持续更新）
python爬虫scrapy项目(一) 爬取目标:腾讯招聘网站(起始url:https://hr.tencent.com/position.php?keywords=&tid=0&st ...
- 第三百五十五节，Python分布式爬虫打造搜索引擎Scrapy精讲—scrapy信号详解
第三百五十五节,Python分布式爬虫打造搜索引擎Scrapy精讲—scrapy信号详解 信号一般使用信号分发器dispatcher.connect(),来设置信号,和信号触发函数,当捕获到信号时执行 ...
- Scrapy 增量式爬虫
Scrapy 增量式爬虫 https://blog.csdn.net/mygodit/article/details/83931009 https://blog.csdn.net/mygodit/ar ...
- Scala 深入浅出实战经典 第64讲：Scala中隐式对象代码实战详解
王家林亲授<DT大数据梦工厂>大数据实战视频 Scala 深入浅出实战经典(1-87讲)完整视频.PPT.代码下载:百度云盘:http://pan.baidu.com/s/1c0noOt6 ...
- Scala 深入浅出实战经典 第63讲：Scala中隐式类代码实战详解
王家林亲授<DT大数据梦工厂>大数据实战视频 Scala 深入浅出实战经典(1-87讲)完整视频.PPT.代码下载:百度云盘:http://pan.baidu.com/s/1c0noOt6 ...
- python 3.x 爬虫基础---Urllib详解
python 3.x 爬虫基础 python 3.x 爬虫基础---http headers详解 python 3.x 爬虫基础---Urllib详解 前言 爬虫也了解了一段时间了希望在半个月的时间内 ...
- SPFA + 链式前向星(详解)
求最短路是图论中最基础的算法,最短路算法挺多,本文介绍SPFA算法. 关于其他最短路算法,请看我另一篇博客最短路算法详解 链式前向星概念 简单的说,就是存储图的一个数据结构.它是按照边来存图,而邻接矩 ...
- SQL Server 表的管理_关于事务操作的详解（案例代码）
SQL Server 表的管理_关于事务操作的详解(案例代码) 1.概念 事务(transaction): 是将多个修改语句组合在一起的方法,这个方法中的所有语句只有全部执行才能正确完成功能.即要么全 ...
- Springboot 整合 Dubbo/ZooKeeper 详解 SOA 案例
摘要: 原创出处:www.bysocket.com 泥瓦匠BYSocket 希望转载,保留摘要,谢谢! “看看星空,会觉得自己很渺小,可能我们在宇宙中从来就是一个偶然.所以,无论什么事情,仔细想一 ...
- SQL Server 表的管理_关于事务的处理的详解（案例代码）
SQL Server 表的管理_关于事务的处理的详解(案例代码) 一.SQL 事务 1.1SQL 事务 ●事务是在数据库上按照一定的逻辑顺序执行的任务序列,既可以由用户手动执行,也可以由某种数据库程序 ...
随机推荐
- git的一些操作命令
一,如何修改一个commit的注释? root@kubuntu:/data/git/clog# git commit --amend 说明:架构森林是一个专注架构的博客,地址:https://www. ...
- centos8下启用rc-local服务
一,centos8不建议写rc.local,默认启动时执行的命令放到何处? 以前我们会把linux开机执行的命令写入到/etc/rc.local 在centos8上系统不再建议我们写入到rc.loca ...
- XSS攻击 js 脚本注入
原文地址:http://www.cnblogs.com/robot/archive/2009/04/15/1436107.html 1.不要相信Request.QueryString: 相信在asp时 ...
- 第七章 TCP和UDP原理
一.引入 1.TCP/IP协议族的传输层协议主要包括TCP和UDP 2.TCP是面向连接的可靠的传输层协议.它支持在并不可靠的网络上实现面向连接的可靠的数据传输 3.UDP是无连接的传输协议,主要用于 ...
- 【应用程序见解 Application Insights】使用Azure Monitor Application Insights Agent获取Azure VM中监控数据及IIS请求指标等信息
问题情形 为了使用Application Insights也可以监控Azure VM中的相关性能数据,如CPU, Memory,IIS Reuqest等信息,可以在VM中开始一个一个扩展插件: Azu ...
- MySQL安装版本介绍
MySQL安装 源码安装 二进制格式的程序包 程序包管理器安装 yum dnf MySQL在CentOS上的情况 6上只有mySQL 7上只有mariadb 8上既有mysql也有mariadb Ce ...
- c++ qsort的使用
c++ qsort的使用 qsort函数定义在头文件algorithm中,使用时需要include该头文件 void qsort (void* base, size_t num, size_t siz ...
- Oracle一些常用操作语句
--创建oracle登录用户 create user CHECKDATAUSER identified by "bsoft" default tablespace PBPG ...
- C语言之 Switch和?:运算符的反汇编
Switch条件语句 通过上面一篇了解了条件语句的使用,接下来就直接进行反汇编学习 #include <stdio.h> void print() { int b = 1; switch ...
- Vue.js 学习笔记之六：构建更复杂的组件
在掌握了如何构建与编译 Vue 组件的基础知识之后,接下来就可以试着来构建一些更具有实际用处的复杂组件了.为了赋予组件更具实用性的后面,首先要做的就是让这些组件具备监听用户自定义事件的能力,并且允许用 ...