Scrapy框架: middlewares.py设置
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
class DownloadtestSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class DownloadtestDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
import random
#设置请求头
class RandomUA():
def __init__(self):
self.user_agent=[
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0.1 Safari/605.1.16',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)'
]
def process_request(self, request, spider):
request.headers['User-Agent'] = random.choice(self.user_agent)
def process_response(self, request, response, spider):
response.status=201
return response
#设置代理
class ProxyMiddleware:
proxy_list=[
"http://127.0.0.1:8080"
# "http://183.91.33.41:80",
# "http://111.29.3.190:80",
# "http://124.156.108.71:82"
]
def process_request(self, request, spider):
ip=random.choice(self.proxy_list)
request.meta['proxy']=ip
Scrapy框架: middlewares.py设置的更多相关文章
- Scrapy框架: settings.py设置
# -*- coding: utf-8 -*- # Scrapy settings for maitian project # # For simplicity, this file contains ...
- Scrapy框架: pipelines.py设置
保存数据到json文件 # -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your p ...
- Scrapy框架之高级 转
一.CrawlSpider模板 创建项目 scrapy startproject 项目名称 查看模板 scrapy genspider -l 创建crawl模板 scrapy genspider -t ...
- 基于scrapy框架的爬虫
Scrapy是一个为了爬取网站数据,提取结构性数据而编写的应用框架. 可以应用在包括数据挖掘,信息处理或存储历史数据等一系列的程序中. scrapy 框架 高性能的网络请求 高性能的数据解析 高性能的 ...
- scrapy框架的中间件
中间件的使用 作用:拦截所有的请求和响应 拦截请求:process_request拦截正常的请求,process_exception拦截异常的请求 篡改请求的头信息 def process_reque ...
- scrapy框架设置代理
网易音乐在单ip请求下经常会遇到网页返回码503的情况经查询,503为单个ip请求流量超限,猜测是网易音乐的一种反扒方式因原音乐下载程序采用scrapy框架,所以需要在scrapy中通过代理的方式去解 ...
- scrapy框架设置代理ip,headers头和cookies
[设置代理ip] 根据最新的scrapy官方文档,scrapy爬虫框架的代理配置有以下两种方法: 一.使用中间件DownloaderMiddleware进行配置使用Scrapy默认方法scrapy s ...
- 网络爬虫之scrapy框架设置代理
前戏 os.environ()简介 os.environ()可以获取到当前进程的环境变量,注意,是当前进程. 如果我们在一个程序中设置了环境变量,另一个程序是无法获取设置的那个变量的. 环境变量是以一 ...
- Python爬虫Scrapy框架入门(2)
本文是跟着大神博客,尝试从网站上爬一堆东西,一堆你懂得的东西 附上原创链接: http://www.cnblogs.com/qiyeboy/p/5428240.html 基本思路是,查看网页元素,填写 ...
随机推荐
- 通过URL传参数,然后第二个页面需要获取参数
/** * 方法说明:通过url参数键值名称获取参数的值 * @method getQueryString * @param name 要获取的参数键值 * @return * @remark */ ...
- SPI、IIC、IIS、UART、JTAG的应用场合级区别
SPI SPI接口的全称是"Serial Peripheral Interface",意为串行外围接口,是Motorola首先在其MC68HCXX系列处理器上定义的. SPI接口 ...
- 一、spring的基本认识
Spring的认识 Spring是一个开放源代码的设计层面框架,它解决的是业务逻辑层和其他各层的松耦合问题,因此它将面向接口的编程思想贯穿整个系统应用 Spring是于2003 年兴起的一个轻量级的J ...
- db2 连接数据库与断开数据库
连接数据库: connect to db_name user db_user using db_pass 断开连接: connect resetdisconnect current quit是退出交 ...
- 2018-2-13-win10-uwp-HttpClient-post错误
title author date CreateTime categories win10 uwp HttpClient post错误 lindexi 2018-2-13 17:23:3 +0800 ...
- matplotlib不显示图片
import matplotlib.pyplot as plt 绘制好图像之后,需要使用plt.show()才会显示出图片.
- Eur yml
server: port: 6001 # 服务端口 eureka: instance: hostname: localhost # eureka服务端的实例名称 client: registerWit ...
- UNP学习 路由套接口
一.概述 在路由套接口中支持三种类型的操作: 1.进程能通过写路由套接口想内核发消息.举例:路径就是这样增加和删除的. 2.进程能在路由套接口上从内核读消息. 3.进程可以用sysctl函数得到路由表 ...
- POJ 3414 Pots (dfs,这个代码好长啊QAQ)
Description You are given two pots, having the volume of A and B liters respectively. The following ...
- Cisco基础(一):Vlan的划分、配置trunk中继链路、以太通道配置、DHCP服务配置
一.Vlan的划分 目标: VLAN(虚拟局域网)是对连接到的第二层交换机端口的网络用户的逻辑分段,不受网络用户的物理位置限制而根据用户需求进行网络分段.一个VLAN可以在 一个交换机或者跨交换机实现 ...