爬虫入门之scrapy模拟登陆(十四)

注意：模拟登陆时，必须保证settings.py里的COOKIES_ENABLED(Cookies中间件) 处于开启状态

COOKIES_ENABLED = True或# COOKIES_ENABLED = False

策略一：直接POST数据（比如需要登陆的账户信息)

只要是需要提供post数据的，就可以用这种方法。下面示例里post的数据是账户密码：

• 可以使用yield scrapy.FormRequest(url, formdata, callback)方法发送POST请求。
• 如果希望程序执行一开始就发送POST请求，可以重写Spider类的start_requests(self)方法，并且不再调用start_urls里的url。

class mySpider(scrapy.Spider):
    # start_urls = ["http://www.example.com/"]
    def start_requests(self):
        url = 'http://www.renren.com/PLogin.do'  #从源码中form表单提取的action网址

        # FormRequest 是Scrapy发送POST请求的方法
        yield scrapy.FormRequest(
            url = url,
            formdata = {"email" : "mr_mao_hacker@163.com", "password" : "axxxxxxxe"},
            callback = self.parse_page
        )
    def parse_page(self, response):
        # do something
        # 业务逻辑

策略二：标准的模拟登陆步骤

正统模拟登录方法：

1. 首先发送登录页面的get请求，获取到页面里的登录必须的参数（比如说zhihu登陆界面的 _xsrf）
2. 然后和账户密码一起post到服务器，登录成功
3. 使用FormRequest.from_response()方法[模拟用户登录]

import scrapy

class LoginSpider(scrapy.Spider):
    name = 'example.com'
    start_urls = ['http://www.example.com/users/login.php']

    def parse(self, response):
        return scrapy.FormRequest.from_response(
        response,
        formdata={'username': 'john', 'password': 'secret'},
        callback=self.after_login
        )

    def after_login(self, response):
        # check login succeed before going on
        if "authentication failed" in response.body:
            self.log("Login failed", level=log.ERROR)
            return

# continue scraping with authenticated session...

模拟浏览器登录

start_requests()方法，可以返回一个请求给爬虫的起始网站，这个返回的请求相当于start_urls，start_requests()返回的请求会替代start_urls里的请求

Request()get请求，可以设置，url、cookie、回调函数

FormRequest.from_response()表单post提交，第一个必须参数，上一次响应cookie的response对象，其他参数，cookie、url、表单内容等

正统模拟登录方法

import scrapy

# 正统模拟登录方法：
# 首先发送登录页面的get请求，获取到页面里的登录必须的参数，比如说zhihu的 _xsrf
# 然后和账户密码一起post到服务器，登录成功

# 第二种标准
def parse(self, response):
    print(response.body.decode('utf-8'), "@@" * 40)
    yield scrapy.FormRequest.from_response(response,formdata={
                                                   "email": "18588403840",
                						"origURL":"http://www.renren.com/422167102/profile",
                                                   "domain": "renren.com",
                                                   "key_id": "1",
                                                   "captcha_type": "web_login",
                                                   "password": "97bfc03b0eec4df7c76eaec10cd08ea57b01eefd0c0ffd4c0e5061ebd66460d9",
                                                   "rkey": "26615a8e93fee56fc1fb3d679afa3cc4",
                                                   "f": ""
                                               },
                                               dont_filter=True,
                                               headers=self.headers,
                                               callback=self.get_page)
    def get_page(self, response):
        print("===================", response.url)
        print(response.body.decode('utf-8'))
        url = "http://www.renren.com/353111356/profile"
        yield scrapy.Request(url, callback=self.get_info)

    def get_info(self, response):
        print('*******' * 30)
        print(response.body.decode('utf-8'))

yield Request()可以将一个新的请求返回给爬虫执行

在发送请求时cookie的操作， meta={'cookiejar':1}表示开启cookie记录，首次请求时写在Request()里 meta={'cookiejar':response.meta['cookiejar']}表示使用上一次response的cookie，写在FormRequest.from_response()里post授权 meta={'cookiejar':True}表示使用授权后的cookie访问需要登录查看的页面

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

class MyrenSpider(CrawlSpider):
    name = 'myren'
    allowed_domains = ['renren.com']
    start_urls = ["http://www.renren.com/353111356/profile"]

    rules = [Rule(LinkExtractor(allow=('(\d+)/profile')), callback='get_info', follow=True)]
    headers = {
        "Accept": "*/*",
        "Connection": "keep-alive",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36",
    }

    def start_requests(self):
        yield scrapy.Request(url="http://www.renren.com/", meta={'cookiejar': 1}, callback=self.post_login)

    # 第二种标准
    def post_login(self, response):
        yield scrapy.FormRequest.from_response(response,
                                               url="http://www.renren.com/PLogin.do",
                                               meta={'cookiejar': response.meta['cookiejar']},
                                               # 在之前需要打开 meta = {'cookiejar' : 1}
                                               headers=self.headers,
                                               formdata={
                                                   "email": "18588403840",
                                                   "password": "Changeme_123"
                                               },
                                               dont_filter=True,

                                               callback=self.after_login)

    def after_login(self, response):
        for url in self.start_urls:
            # yield self.make_requests_from_url(url)
            yield scrapy.Request(url, meta={'cookiejar': response.meta['cookiejar']})

    def get_info(self, response):
        print('*******' * 30)
        print(response.body.decode('utf-8'))

    def _requests_to_follow(self, response):
    """重写加入cookiejar的更新"""
    if not isinstance(response, HtmlResponse):
        return
    seen = set()
    for n, rule in enumerate(self._rules):
        links = [l for l in rule.link_extractor.extract_links(response) if l not in seen]
        if links and rule.process_links:
            links = rule.process_links(links)
        for link in links:
            seen.add(link)
            r = Request(url=link.url, callback=self._response_downloaded)
            # 下面这句是我重写的
            r.meta.update(rule=n, link_text=link.text, cookiejar=response.meta['cookiejar'])
            yield rule.process_request(r)

策略三：直接使用保存登陆状态的Cookie模拟登陆

如果实在没办法了，可以用这种方法模拟登录，虽然麻烦一点，但是成功率100%

ChangeCookies 将cookie解析成字典形式

class transCookie:

    def __init__(self, cookie):
        self.cookie = cookie

    def stringToDict(self):
        '''
        将从浏览器上Copy来的cookie字符串转化为Scrapy能使用的Dict
        :return:
        '''
        itemDict = {}
        items = self.cookie.split(';')
        for item in items:
            key = item.split('=')[0].strip()
            value = item.split('=')[1]
            itemDict[key] = value
        return itemDict

if __name__ == "__main__":
    cookie = "你的cookie"
    trans = transCookie(cookie)
    print(trans.stringToDict())

将解析好的cookie格式放入请求

# -*- coding: utf-8 -*-
import scrapy

class RenrenSpider(scrapy.Spider):
    name = "renren"
    allowed_domains = ["renren.com"]
    start_urls = [
        'http://www.renren.com/111111',
        'http://www.renren.com/222222',
        'http://www.renren.com/333333',
        ]  #开始请求url列表
    cookies = {
    "anonymid" : "ixrna3fysufnwv",
    "_r01_" : "1",
    "ap" : "327550029",
    "JSESSIONID" : "abciwg61A_RvtaRS3GjOv",
    "depovince" : "GW",
    "springskin" : "set",
    "jebe_key" : "f6fb270b-d06d-42e6-8b53-e67c3156aa7e%7Cc13c37f53bca9e1e7132d4b58ce00fa3%7C1484060607478%7C1%7C1486198628950",
    "t" : "691808127750a83d33704a565d8340ae9",
    "societyguester" : "691808127750a83d33704a565d8340ae9",
    "id" : "327550029",
    "xnsid" : "f42b25cf",
    "loginfrom" : "syshome"
    }

    # 可以重写Spider类的start_requests方法，附带Cookie值，发送POST请求
    def start_requests(self):
        return [scrapy.FormRequest(url, cookies = self.cookies, callback = self.parse)]

    # 处理响应内容
    def parse(self, response):
        print "===========" + response.url
        with open("deng.html", "w") as filename:
            filename.write(response.body)

策略四 : 使用selenium插件(全能)

1 spider.browser.page_source 获取响应的源代码

2 session.get(request.url).text 获取响应的源代码

3 requests采用session管理cookie

4 urllib 采用cookieJar管理cookie

模拟登录淘宝

class TaobaoSpider(scrapy.Spider):
    name = 'mytaobao'
    allowed_domains = ['taobao.com']
    start_urls = ['https://login.m.taobao.com/login.htm',
                  "http://h5.m.taobao.com/mlapp/olist.html?spm=a2141.7756461.2.6"]

    def __init__(self):  # 初始化
        self.browser = None
        self.cookies = None
        super(TaobaoSpider, self).__init__()  # 传递给父类

    def parse(self, response):
        # 打印链接，打印网页源代码
        print(response.url)
        print(response.body.decode("utf-8", "ignore"))

#中间件middleware 自定义LoginMiddleware登录

from scrapy import signals
from selenium import webdriver
from scrapy.http import HtmlResponse  # 网页响应
import requests
import time

class LoginMiddleware(object):
	'''
	找到password username输入框并send_keys
	点击登录并抓取cookie,spider.browser.get_cookies()
	返回页面信息,HtmlResponse
	'''
    def process_request(self, request, spider):
        if spider.name == "mytaobao":  # 指定仅仅处理这个名称的爬虫
            if request.url.find("login") != -1:  # 判断是否登陆页面
                mobilesetting = {"deviceName": "iPhone 6 Plus"}
                options = webdriver.ChromeOptions()  # 浏览器选项
                options.add_experimental_option("mobileEmulation", mobilesetting)  # 模拟手机
                spider.browser = webdriver.Chrome(chrome_options=options)  # 创建一个浏览器对象
                spider.browser.set_window_size(400, 800)  # 配置手机大小

                spider.browser.get(request.url)  # 爬虫访问链接
                time.sleep(3) #必须要睡下因为考虑到输入:用户名密码 要时间
                print("login访问", request.url)
                username = spider.browser.find_element_by_id("username")
                password = spider.browser.find_element_by_id("password")
                time.sleep(1)
                username.send_keys("2403239393@qq.com")  # 账户
                time.sleep(2)
                password.send_keys("bama100")  # 密码
                time.sleep(2)
                spider.browser.find_element_by_id("btn-submit").click()

                time.sleep(4)
                spider.cookies = spider.browser.get_cookies()  # 抓取全部的cookie
                # spider.browser.close()
                return HtmlResponse(url=spider.browser.current_url,  # 当前连接
                                    body=spider.browser.page_source,  # 源代码
                                    encoding="utf-8")  # 返回页面信息
            else:#登录后则执行
                '''
                1 采用requests.session保存cookie
                2 设置cookie session.cookie.set(name,value)
                3 清空headers  session.headers.clear()
                4 发起get请求   session.get(url)
                '''
                print("request  访问")
                session = requests.session()  # 会话
                for cookie in spider.cookies:
                    session.cookies.set(cookie['name'], cookie["value"])
                session.headers.clear()  # 清空头
                newpage = session.get(request.url)
                print("---------------------")
                print(request.url)
                print("---------------------")
                print(newpage.text)
                print("---------------------")
                # 页面
                time.sleep(3)
                return HtmlResponse(url=request.url,  # 当前连接
                                    body=newpage.text,  # 源代码
                                    encoding="utf-8")  # 返回页面信息