Scrapy框架: middlewares.py设置
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
class DownloadtestSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class DownloadtestDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
import random
#设置请求头
class RandomUA():
def __init__(self):
self.user_agent=[
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0.1 Safari/605.1.16',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)'
]
def process_request(self, request, spider):
request.headers['User-Agent'] = random.choice(self.user_agent)
def process_response(self, request, response, spider):
response.status=201
return response
#设置代理
class ProxyMiddleware:
proxy_list=[
"http://127.0.0.1:8080"
# "http://183.91.33.41:80",
# "http://111.29.3.190:80",
# "http://124.156.108.71:82"
]
def process_request(self, request, spider):
ip=random.choice(self.proxy_list)
request.meta['proxy']=ip
Scrapy框架: middlewares.py设置的更多相关文章
- Scrapy框架: settings.py设置
# -*- coding: utf-8 -*- # Scrapy settings for maitian project # # For simplicity, this file contains ...
- Scrapy框架: pipelines.py设置
保存数据到json文件 # -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your p ...
- Scrapy框架之高级 转
一.CrawlSpider模板 创建项目 scrapy startproject 项目名称 查看模板 scrapy genspider -l 创建crawl模板 scrapy genspider -t ...
- 基于scrapy框架的爬虫
Scrapy是一个为了爬取网站数据,提取结构性数据而编写的应用框架. 可以应用在包括数据挖掘,信息处理或存储历史数据等一系列的程序中. scrapy 框架 高性能的网络请求 高性能的数据解析 高性能的 ...
- scrapy框架的中间件
中间件的使用 作用:拦截所有的请求和响应 拦截请求:process_request拦截正常的请求,process_exception拦截异常的请求 篡改请求的头信息 def process_reque ...
- scrapy框架设置代理
网易音乐在单ip请求下经常会遇到网页返回码503的情况经查询,503为单个ip请求流量超限,猜测是网易音乐的一种反扒方式因原音乐下载程序采用scrapy框架,所以需要在scrapy中通过代理的方式去解 ...
- scrapy框架设置代理ip,headers头和cookies
[设置代理ip] 根据最新的scrapy官方文档,scrapy爬虫框架的代理配置有以下两种方法: 一.使用中间件DownloaderMiddleware进行配置使用Scrapy默认方法scrapy s ...
- 网络爬虫之scrapy框架设置代理
前戏 os.environ()简介 os.environ()可以获取到当前进程的环境变量,注意,是当前进程. 如果我们在一个程序中设置了环境变量,另一个程序是无法获取设置的那个变量的. 环境变量是以一 ...
- Python爬虫Scrapy框架入门(2)
本文是跟着大神博客,尝试从网站上爬一堆东西,一堆你懂得的东西 附上原创链接: http://www.cnblogs.com/qiyeboy/p/5428240.html 基本思路是,查看网页元素,填写 ...
随机推荐
- Ubuntu18.10 编译libevent出现错误: creating symbolic link XXXXXX : Operation not supported
今天在VirtualBox虚拟机下的Ubuntu18.10编译libevent源代码时,按照github中使用cmake方式: $ mkdir build && cd build $ ...
- 转 cpu高 问题分析定位
文章来源: http://www.blogjava.net/hankchen/archive/2012/08/09/377735.html 一个应用占用CPU很高,除了确实是计算密集型应用之外,通常原 ...
- java反射(一)--认识反射机制
一.认识java反射机制 在java语言中,之所以会有如此众多的开源技术支撑,很大的一部分来源于java最大特征--反射机制.能够灵活的去使用反射机制进行项目的开发与设计,才能够真正接触到java的精 ...
- poj3252 Round Numbers(数位dp)
题目传送门 Round Numbers Time Limit: 2000MS Memory Limit: 65536K Total Submissions: 16439 Accepted: 6 ...
- SpringBoot-技术专区-异步编程
最近在实现一个聚合搜索的需求时,由于需要从五个索引中查询数据,然后再将搜索结果组合返回给前端app展现,显然这个地方不能再用同步的方式来操作了,如果有一个索引查询出现耗时较长,那么其余的请求都会排同步 ...
- canvas---从基础到实战
canvas是H5新增的一个元素,可以用来描绘各种你想描绘的东西. canvas本身没有绘制能力,你可以把它当做一个容器,需要我们用脚本,也就是js来给他灌满水. 兼容性 1. IE9版本以上.Fir ...
- 【记录】@Configuration注解作用 mybatis @Param作用
参考地址: 1:https://www.cnblogs.com/duanxz/p/7493276.html 2:https://www.wandouip.com/t5i91156/ 3:https:/ ...
- Dev常用控件
GridControl TreeView DEV GridControl小结.. https://blog.csdn.net/happy09li/article/details/7186829 Dev ...
- 前端学习(二十九)nodejs(笔记)
后台语言 java php .Net python Node.js------------------------------------------------- ...
- 杂谈、 素材资源,没有美工不会ps一样可以美观
免费素材网站 阿里巴巴矢量图,大部分图标都有颜色像素可选,格式可选3种, http://www.iconfont.cn/plus/home/index?spm=a313x.7781069.199891 ...