scrapy全栈抓xpc练习

#  spider文件
# -*- coding: utf-8 -*-
import scrapy
import re
from scrapy import Request
import json
import string
import random
from xpc.items import PostItem, CommentItem, CopyItem  # 多个item

def strip(s):
    # s存在就去空，不存在就返回空
    if s:
        return s.strip()
    return ""

# 使用scrapy.Request和scrapy.FormRequest发送请求的时候，默认会把cookies保存下来
# 模拟登录的时候不用scrapy框架，直接使用request模块
cookies = dict(
    Authorization='4F635191B0602B5D3B06024483B0602AAF8B06023C2F6259656D'
)
# 上面的cookies是网站返回的，需要先登陆的一下把这个cookies找到

# 生成26个字母+数字
def gen_sessionid():
    return "".join(random.choices(string.ascii_lowercase + string.digits, k=26))

class XinpianchangSpider(scrapy.Spider):
    name = 'XinPianChang'
    allowed_domains = ['xinpianchang.com', 'openapi-vtom.vmovier.com']
    start_urls = ['https://www.xinpianchang.com/channel/index/sort-like?from=tabArticle']
    # 假设从21页开始访问，这里就需要带上cookies，这时候最开始设置的cookies就不能用了，网站会返回4个cookies。需要从写start_requests函数
    # start_urls = ['https://www.xinpianchang.com/channel/index/type-/sort-like/duration_type-0/resolution_type-/page-21']
    page_count = 0

    # 重写父类中的 start_requests方法，该方法默认对start_urls中的url发get请求
    # def start_requests(self):
    #     for url in self.start_urls:
    #         # data = {
    #         #     "kw": "cat"
    #         # }
    #         # post请求发送,使用FormRequest
    #         # yield scrapy.FormRequest(url=url, callback=self.parse, formdata=data)
    #
    #         c = cookies.copy()
    #         c.update(PHPSESSID=gen_sessionid(),
    #                  SERVER_ID='b52601c8-285bdd26',
    #                  channel_page='apU%3D')
    #         yield Request(url, cookies=c, dont_filter=True)

    def parse(self, response):
        # from scrapy.shell import inspect_response
        # inspect_response(response, self)
        self.page_count += 1
        if self.page_count >= 100:
            cookies.update(PHPSESSID=gen_sessionid())
            self.page_count = 0

        url_list = response.xpath('//ul[@class="video-list"]/li/@data-articleid').extract()
        for pid in url_list:
            detail_url = 'https://www.xinpianchang.com/a{}?from=ArticleList'.format(pid)
            # print(detail_url)
            request = response.follow(detail_url, callback=self.parse_post)
            request.meta['pid'] = pid
            yield request  # 进入作品的详情页请求

        pages = response.xpath('//div[@class="page"]/a/@href').extract()
        for page_url in pages:
            # print("列表页翻页url", page_url)  # page_url是一个相对路径，不完整的
            yield response.follow(page_url, self.parse, cookies=cookies)

    def parse_post(self, response):
        pid = response.meta['pid']
        post = PostItem()
        post['pid'] = pid
        post['title'] = response.xpath('//div[@class="title-wrap"]/h3/text()').get()
        # video_url = 'https://openapi-vtom.vmovier.com/v3/video/5E34203E92450?expand=resource&usage=xpc_web'
        # response.text拿到网页返回的源码
        vid = re.findall('vid: "(.*?)",', response.text)[0]
        # print(vid)
        video_url = 'https://openapi-vtom.vmovier.com/v3/video/{}?expand=resource&usage=xpc_web'.format(vid)
        cates = response.xpath('//span[@class="cate v-center"]/a/text()').extract()
        post['category'] = ''.join([cate.strip() for cate in cates])
        post['create_time'] = response.xpath('//span[contains(@class,"update-time")]/i/text()').get()
        post['play_count'] = response.xpath('//i[contains(@class,"play-counts")]/text()').get()
        desc_lst = response.xpath('//p[contains(@class,"desc")]//text()').extract()
        post['desc'] = ' '.join([i.strip() for i in desc_lst])

        # 请求这个video_url, 多了一步这个注意一下
        request = Request(video_url, callback=self.parse_video)
        # 把之前获取到的post通过meta传到下一个函数中. 这个post是请求传参
        request.meta['post'] = post
        yield request

        # 获取评论链接‘https://app.xinpianchang.com/comments?resource_id=10664352&type=article&page=1&per_page=24’
        comment_url = "https://app.xinpianchang.com/comments?resource_id={}&type=article&page=1&per_page=24".format(
            pid)
        request = Request(comment_url, callback=self.parse_comment)
        # 把之前获取到的post通过meta传到下一个函数中
        request.meta['pid'] = pid
        yield request

        # 获取作者页链接
        creator_list = response.xpath('//div[@class="filmplay-creator right-section"]/ul[@class="creator-list"]/li')
        composer_url = 'https://www.xinpianchang.com/u{}?from=articleList'
        # cid = response.xpath('//div[@class="filmplay-creator right-section"]/ul[@class="creator-list"]/li/a/@data-userid')
        for creator in creator_list:
            cid = creator.xpath('./a/@data-userid').get()
            composer_url = 'https://www.xinpianchang.com/u{}?from=articleList'.format(cid)
            request = response.follow(composer_url, self.parse_composer)
            request.meta['cid'] = cid
            # 避免在cookies更新之后，不断的添加到请求头里面，避免请求头里带有一串cookies
            request.meta['dont_merge_cookies'] = True
            yield request

            # 作者和视频的对应关系
            cr = CopyItem()
            cr['pid'] = pid
            cr['cid'] = cid
            cr['pcid'] = pid + cid
            cr['role'] = creator.xpath('./div[@class="creator-info"]/span/text()').get()
            # print("cr", cr)
            yield cr

    def parse_video(self, response):  # 这个response是json格式
        post = response.meta['post']
        # 先把返回的json转化一下， 注意一下
        result = json.loads(response.text)
        post['video_url'] = result['data']['resource']['default']['url']
        # 直接返回给管道了
        yield post

    def parse_comment(self, response):
        result = json.loads(response.text)
        for c in result['data']['list']:
            comment = CommentItem()
            comment['uname'] = c['userInfo']['username']
            comment['user_id'] = c['userInfo']['id']
            # comment['user_page'] = c['userInfo']['web_url']
            comment['content'] = c['content']
            comment['content_id'] = c['id']
            print(comment)
            yield comment

        # 如果有下一页
        if result['data']['next_page_url']:
            next_page = 'https://app.xinpianchang.com' + result['data']['next_page_url']
            # print("next_page", next_page)
            yield response.follow(next_page, self.parse_comment)

    def parse_composer(self, response):
        pass

# settings文件
# -*- coding: utf-8 -*-

# Scrapy settings for xpc project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'xpc'

SPIDER_MODULES = ['xpc.spiders']
NEWSPIDER_MODULE = 'xpc.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = 'xpc (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
# 如果使用自定义cookie就把COOKIES_ENABLED设置为True
# 如果使用settings的cookie就把COOKIES_ENABLED设置为False
COOKIES_ENABLED = True
COOKIES_DEBUG = True  # 可以打印出来详细的cookies信息

# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)',
}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
# SPIDER_MIDDLEWARES = {
#    'xpc.middlewares.XpcSpiderMiddleware': 543,
# }

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
#    'xpc.middlewares.XpcDownloaderMiddleware': 543,
# }

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
# EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
# }

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'xpc.pipelines.XpcPipeline': 300,  # 优先级高
   # 'xpc.pipelines.MysqlPipeline': 301,
   # 'xpc.pipelines.RedisPipeline': 302,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
# AUTOTHROTTLE_ENABLED = True
# The initial download delay
# AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
# AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
# AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
HTTPCACHE_ENABLED = False   # True缓存访问过的网页，不会真实的发请求
# HTTPCACHE_ENABLED = True

# HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

# 日志类型： INFO DEBUG ERROR
LOG_LEVEL = 'DEBUG'

# item文件
# -*- coding: utf-8 -*-
import scrapy

class PostItem(scrapy.Item):
    # 保存视频信息
    # 自定义字段，有多个表的时候需要写个table_name
    table_name = 'posts'

    # 下面的是数据字段
    pid = scrapy.Field()
    title = scrapy.Field()
    category = scrapy.Field()
    create_time = scrapy.Field()
    play_count = scrapy.Field()
    desc = scrapy.Field()
    video_url = scrapy.Field()

class CommentItem(scrapy.Item):
    # 保存评论信息
    table_name = 'comments'
    content_id = scrapy.Field()
    pid = scrapy.Field()
    cid = scrapy.Field()
    uname = scrapy.Field()
    user_id = scrapy.Field()
    content = scrapy.Field()
    user_page = scrapy.Field()

class CopyItem(scrapy.Item):
    table_name = 'copyrights'
    pcid = scrapy.Field()  # 表的主键
    pid = scrapy.Field()
    cid = scrapy.Field()
    role = scrapy.Field()

# pipeline文件
# -*- coding: utf-8 -*-

import csv
from xpc.items import PostItem, CommentItem, CopyItem
import pymysql
from redis import Redis
import os

class XpcPipeline(object):
    def __init__(self):
        # 当前文件的上一级
        store_file = os.path.dirname(__file__) + '/xpc.csv'
        # 打开文件
        self.file = open(store_file, 'w', newline="")
        # csv 写法
        self.writer = csv.writer(self.file)

    def open_spider(self, spider):
        print("pipeline 开始爬虫......")
　　

　　# 执行多个不同的item时
    def process_item(self, item, spider):
        if isinstance(item, PostItem):
            print("这是发布信息：", item)
        elif isinstance(item, CommentItem):
            print("这是评论信息：", item)
        elif isinstance(item, CopyItem):
            print("这是版权信息：", item)
        return item  # 返回给下一个要执行的管道类

    def close_spider(self, spider):
        print("pipeline 结束爬虫......")

# 连接数据库
class MysqlPipeline(object):
    conn = None
    cursor = None

    def open_spider(self, spider):
        self.conn = pymysql.Connect(
            host='127.0.0.1',
            port=3306,
            user='root',
            password='',
            db='test_db',
            charset='utf8'
        )
        print("数据库连接成功")

    def process_item(self, item, spider):
        self.cursor = self.conn.cursor()
        try:
            self.cursor.execute('insert into test_db values("%s", "%s")' % (item['author'], item['content']))
            self.conn.commit()
        except Exception as e:
            print("数据库插入异常：", e)
            print("数据库执行回滚")
            self.conn.rollback()
        return item

    def close_spider(self, spider):
        print("断开数据库连接")
        self.cursor.close()
        self.conn.close()

# 连接数据库
class RedisPipeline(object):
    conn = None
    cursor = None

    def open_spider(self, spider):
        self.conn = Redis(
            host='127.0.0.1',
            port=6379
        )
        print("数据库连接成功")

    def process_item(self, item, spider):
        dic = {
            "author": item["author"],
            "content": item["content"]
        }
        self.conn.lpush("队列名字", dic)

    def close_spider(self, spider):
        print("断开数据库连接")
        self.cursor.close()
        self.conn.close()