# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals class DownloadtestSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects. @classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider. # Should return None or raise an exception.
return None def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response. # Must return an iterable of Request, dict or Item objects.
for i in result:
yield i def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception. # Should return either None or an iterable of Response, dict
# or Item objects.
pass def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated. # Must return only requests (not items).
for r in start_requests:
yield r def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name) class DownloadtestDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects. @classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware. # Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None def process_response(self, request, response, spider):
# Called with the response returned from the downloader. # Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception. # Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name) import random #设置请求头
class RandomUA():
def __init__(self):
self.user_agent=[
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0.1 Safari/605.1.16',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)'
] def process_request(self, request, spider):
request.headers['User-Agent'] = random.choice(self.user_agent) def process_response(self, request, response, spider):
response.status=201
return response #设置代理
class ProxyMiddleware:
proxy_list=[
"http://127.0.0.1:8080"
# "http://183.91.33.41:80",
# "http://111.29.3.190:80",
# "http://124.156.108.71:82"
] def process_request(self, request, spider):
ip=random.choice(self.proxy_list)
request.meta['proxy']=ip

Scrapy框架: middlewares.py设置的更多相关文章

  1. Scrapy框架: settings.py设置

    # -*- coding: utf-8 -*- # Scrapy settings for maitian project # # For simplicity, this file contains ...

  2. Scrapy框架: pipelines.py设置

    保存数据到json文件 # -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your p ...

  3. Scrapy框架之高级 转

    一.CrawlSpider模板 创建项目 scrapy startproject 项目名称 查看模板 scrapy genspider -l 创建crawl模板 scrapy genspider -t ...

  4. 基于scrapy框架的爬虫

    Scrapy是一个为了爬取网站数据,提取结构性数据而编写的应用框架. 可以应用在包括数据挖掘,信息处理或存储历史数据等一系列的程序中. scrapy 框架 高性能的网络请求 高性能的数据解析 高性能的 ...

  5. scrapy框架的中间件

    中间件的使用 作用:拦截所有的请求和响应 拦截请求:process_request拦截正常的请求,process_exception拦截异常的请求 篡改请求的头信息 def process_reque ...

  6. scrapy框架设置代理

    网易音乐在单ip请求下经常会遇到网页返回码503的情况经查询,503为单个ip请求流量超限,猜测是网易音乐的一种反扒方式因原音乐下载程序采用scrapy框架,所以需要在scrapy中通过代理的方式去解 ...

  7. scrapy框架设置代理ip,headers头和cookies

    [设置代理ip] 根据最新的scrapy官方文档,scrapy爬虫框架的代理配置有以下两种方法: 一.使用中间件DownloaderMiddleware进行配置使用Scrapy默认方法scrapy s ...

  8. 网络爬虫之scrapy框架设置代理

    前戏 os.environ()简介 os.environ()可以获取到当前进程的环境变量,注意,是当前进程. 如果我们在一个程序中设置了环境变量,另一个程序是无法获取设置的那个变量的. 环境变量是以一 ...

  9. Python爬虫Scrapy框架入门(2)

    本文是跟着大神博客,尝试从网站上爬一堆东西,一堆你懂得的东西 附上原创链接: http://www.cnblogs.com/qiyeboy/p/5428240.html 基本思路是,查看网页元素,填写 ...

随机推荐

  1. Ubuntu18.10 编译libevent出现错误: creating symbolic link XXXXXX : Operation not supported

    今天在VirtualBox虚拟机下的Ubuntu18.10编译libevent源代码时,按照github中使用cmake方式: $ mkdir build && cd build $ ...

  2. 转 cpu高 问题分析定位

    文章来源: http://www.blogjava.net/hankchen/archive/2012/08/09/377735.html 一个应用占用CPU很高,除了确实是计算密集型应用之外,通常原 ...

  3. java反射(一)--认识反射机制

    一.认识java反射机制 在java语言中,之所以会有如此众多的开源技术支撑,很大的一部分来源于java最大特征--反射机制.能够灵活的去使用反射机制进行项目的开发与设计,才能够真正接触到java的精 ...

  4. poj3252 Round Numbers(数位dp)

    题目传送门 Round Numbers Time Limit: 2000MS   Memory Limit: 65536K Total Submissions: 16439   Accepted: 6 ...

  5. SpringBoot-技术专区-异步编程

    最近在实现一个聚合搜索的需求时,由于需要从五个索引中查询数据,然后再将搜索结果组合返回给前端app展现,显然这个地方不能再用同步的方式来操作了,如果有一个索引查询出现耗时较长,那么其余的请求都会排同步 ...

  6. canvas---从基础到实战

    canvas是H5新增的一个元素,可以用来描绘各种你想描绘的东西. canvas本身没有绘制能力,你可以把它当做一个容器,需要我们用脚本,也就是js来给他灌满水. 兼容性 1. IE9版本以上.Fir ...

  7. 【记录】@Configuration注解作用 mybatis @Param作用

    参考地址: 1:https://www.cnblogs.com/duanxz/p/7493276.html 2:https://www.wandouip.com/t5i91156/ 3:https:/ ...

  8. Dev常用控件

    GridControl TreeView DEV GridControl小结.. https://blog.csdn.net/happy09li/article/details/7186829 Dev ...

  9. 前端学习(二十九)nodejs(笔记)

    后台语言    java     php     .Net     python    Node.js------------------------------------------------- ...

  10. 杂谈、 素材资源,没有美工不会ps一样可以美观

    免费素材网站 阿里巴巴矢量图,大部分图标都有颜色像素可选,格式可选3种, http://www.iconfont.cn/plus/home/index?spm=a313x.7781069.199891 ...