requests获取所有状态码

requests获取所有状态码

requests默认是不会获取301/302的状态码的。可以设置allow_redirects=False，这样就可以获取所有的状态码了

import requests

# url
# url = 'http://www.freebuf.com/news/157100.html'  # 请求200，返回200
url = 'http://www.freebuf.com/fevents/133225.html'  # 请求302，返回200。要想不跳转，获取302，用参数：allow_redirects=False
# url = 'http://www.freebuf.com/articles/database/151839.html'  # 请求403，返回403
# url = 'http://www.freebuf.com/articles/database/1518391.html'  # 请求存在的域名中不存在的页面，请求404，返回404
# url = 'http://www.freebudfsf.com/articles/database/1518391.html'  # 请求不存在的域名。程序崩溃
# url = 'https://www.douban.com/group/topic/49606658/'  # 请求存在的域名，公司限制访问，返回抛出异常，程序崩溃。效果和网络中断一样。
# url = 'http://10.1.75.241'  # 请求ip，（一定要加协议HTTP，否则崩溃）
# headers
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}
try:
    # 发请求，得响应
    response = requests.get(url, headers=headers, allow_redirects=False)
    # 解析
    print('    give url:', url)
    print(' request.url:', response.request.url)
    print('response.url:', response.url)
    print(response.content)
    print(response.status_code)
except Exception as e:
    print(e)

　　

封装一个获取所有状态码的函数，同时实现验证返回值的方法

import requests

def get_statecode_or_errinfo(url=''):
    '''
    获取响应状态码，或者未响应的错误信息
    :param url: 请求的url
    :return: 状态码，或者未响应的错误信息
    '''
    if url == '':
        return '请输入一个url作为get_statecode_or_errinfo的参数'
    # headers
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
    }
    try:
        # 发请求，得响应
        response = requests.get(url, headers=headers, allow_redirects=False)
        # 返回状态码
        return response.status_code
    except Exception as e:
        # 返回异常信息
        return e

if __name__ == '__main__':
    # url
    # url = 'http://www.freebuf.com/news/157100.html'  # 请求200，返回200
    url = 'http://www.freebuf.com/fevents/133225.html'  # 请求302，返回200。要想不跳转，获取302，用参数：allow_redirects=False
    # url = 'http://www.freebuf.com/articles/database/151839.html'  # 请求403，返回403
    # url = 'http://www.freebuf.com/articles/database/1518391.html'  # 请求存在的域名中不存在的页面，请求404，返回404
    # url = 'http://www.freebudfsf.com/articles/database/1518391.html'  # 请求不存在的域名。程序崩溃。如果有Nginx，返回200
    # url = 'http://dsfs'  # 请求不存在的域名，设置了参数：allow_redirects=False，在有Nginx处理的情况下，有304，返回200。
    # url = 'https://www.douban.com/group/topic/49606658/'  # 请求存在的域名，公司限制访问，返回抛出异常，程序崩溃。效果和网络中断一样。
    # url = 'http://10.1.75.241'  # 请求ip，请求200，返回200（一定要加协议HTTP，否则崩溃）

    # url = 'http://www.freebuf.com/fevents/133225.html'  # 请求302，返回200。要想不跳转，获取302，用参数：allow_redirects=False
    url = 'http://www.freebuf.com/news/171238.html'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
    }
    # response=requests.get(url,headers=headers,allow_redirects=False)
    response=requests.get(url,headers=headers)
    # 检查状态码
    print(response.status_code)

    # # 检查url
    print(url)
    print(response.url)
    #
    # # 检查请求头
    print(response.request.headers)
    #
    # # 检查响应头
    print(response.headers)
    #
    # # 检查源码
    # print(response.content)
    # print(response.content.decode())
    # print(response.text)
    #
    # response.encoding = 'utf-8'
    # print(response.text)
    print(response.encoding)
    #
    # # 检查源码字符串长度
    print(len(response.content))

　　

说明：

反扒：
总结多种验证返回值的方式。requests
比如：检查状态码、检查url（有可能发送了跳转）、检查请求头、检查响应头、检查源码、检查源码字符串长度。
检查状态码
print (response.status_code)
检查url
print (response.url)
检查请求头
print (response.request.headers)
检查响应头
print (response.headers)
检查源码字符串长度
print (len(response.content))
检查源码
print (response.content)
print (response.content.decode())
response.encoding='utf-8'
print (response.text)
print (response.encoding)

scrapy爬虫的响应规则：

# 1、被过滤掉，不发出请求：不在允许的域名范围内
# temp['title_url'] = "https://www.baidu.com/"  # 跨域。请求发出前，url直接被过滤掉。
# temp['title_url'] = "http://open.freebuf.com/live?id=1021"  # 跨域。请求发出前，url直接被过滤掉。
# temp['title_url'] = "http://10.1.75.241"  # 请求ip地址，请求发出前，url直接过来掉。如果设置为允许ip网站，没有被过滤，就返回200

# 2、禁止访问
# temp['title_url'] = "http://www.freebuf.com/articles/database/151839.html"#禁止访问403，资源存在，不让访问。Ignoring non-200 response
# temp['title_url'] = "http://www.freebuf.com/articles/database/1518391.html"#禁止访问404，资源本身不存在。Ignoring non-200 response

# 3、重定向后的作为新请求
# temp['title_url'] = "http://www.freebuf.com/news/156654.html"  # 重定向301、302。会返回重定向后200的状态码

# 4、断网
# temp['title_url'] = "https://www.douban.com/group/topic/49606658/"  # 公司限制访问。[<twisted.python.failure.Failure twisted.internet.error.ConnectionLost: Connection to the other side was lost in a non-clean fashion: Connection lost.>]

# 5、没有的网站
# temp['title_url'] = "https://www.badfsdsdfsdfsdfsdddd.com/"  # 直接被过滤掉，如果没有被过滤，就返回域名解析错误：DNS lookup failed: no results for hostname lookup: www.badfsdsdfsdfsdfsdddd.com.
pass

　　

scrapy爬虫举例

freebuf2.py

# -*- coding: utf-8 -*-
import scrapy
from scrapy_FB.items import ScrapyFb2Item

# from util.logger import Logger

# logger_freebuf2 = Logger(logname=__name__, logpath='collection_log', logformat=1, loglevel=10).getlog()
# logger_freebuf2.debug('i am debug3')
# logger_freebuf2.info('i am info3')
# logger_freebuf2.warning('i am warning3')

class Freebuf2Spider(scrapy.Spider):
    # freebuf2爬虫
    name = 'freebuf2'
    allowed_domains = ['freebuf.com','douban.com']
    start_urls = ['http://www.freebuf.com/page/708']

    def parse(self, response):
        cur_url = response.url  # 当前列表页url
        cur_page_num = int(cur_url.rpartition('/')[-1])  # 当前page num

        print('cur_url：%s' % cur_url)
        print('cur_page_num：%s' % cur_page_num)

        # 获取列表节点
        node_list = response.xpath('//*[@id="timeline"]/div/div[2]/dl/dt/a[1]')
        print('len(node_list)：%s' % len(node_list))

        page_num = int(cur_url.rpartition('/')[-1])  # 当前页码
        count_node = len(node_list)  # 当前列表页，一共有的详细页条数

        # 遍历节点
        for i, node in enumerate(node_list):
            # temp = {}
            temp = ScrapyFb2Item()
            temp['title'] = node.xpath('./text()').extract()[0].strip()
            if i == 0:
                # 1、被过滤掉，不发出请求：不在允许的域名范围内
                # temp['title_url'] = "https://www.baidu.com/"  # 跨域。请求发出前，url直接被过滤掉。
                # temp['title_url'] = "http://open.freebuf.com/live?id=1021"  # 跨域。请求发出前，url直接被过滤掉。
                # temp['title_url'] = "http://10.1.75.241"  # 请求ip地址，请求发出前，url直接过来掉。如果设置为允许ip网站，没有被过滤，就返回200

                # 2、禁止访问
                # temp['title_url'] = "http://www.freebuf.com/articles/database/151839.html"#禁止访问403，资源存在，不让访问。Ignoring non-200 response
                # temp['title_url'] = "http://www.freebuf.com/articles/database/1518391.html"#禁止访问404，资源本身不存在。Ignoring non-200 response

                # 3、重定向后的作为新请求
                # temp['title_url'] = "http://www.freebuf.com/news/156654.html"  # 重定向301、302。会返回重定向后200的状态码

                # 4、断网
                # temp['title_url'] = "https://www.douban.com/group/topic/49606658/"  # 公司限制访问。[<twisted.python.failure.Failure twisted.internet.error.ConnectionLost: Connection to the other side was lost in a non-clean fashion: Connection lost.>]

                # 5、没有的网站
                # temp['title_url'] = "https://www.badfsdsdfsdfsdfsdddd.com/"  # 直接被过滤掉，如果没有被过滤，就返回域名解析错误：DNS lookup failed: no results for hostname lookup: www.badfsdsdfsdfsdfsdddd.com.
                pass

            else:
                temp['title_url'] = node.xpath('./@href').extract()[0]
            temp['page_num'] = str(page_num)
            temp['line_num'] = i + 1
            temp['line_total'] = str(count_node)
            # print(temp['line_num'])
            yield scrapy.Request(temp['title_url'], callback=self.parse_detail, meta={"meta_1": temp}, errback=self.err)

        if len(node_list) != 0:  # 爬虫不终止的条件
            # 下一页
            next_url = 'http://www.freebuf.com/page/{}'.format(cur_page_num + 1)
            # print('next_url：%s' % next_url)
            yield scrapy.Request(next_url, callback=self.parse)  # 访问下一页

    def parse_detail(self, response):
        item = response.meta['meta_1']

        print(item['line_num'], item['title_url'])
        # print(response.status)
        print(item['line_num'], response.request.url)

    def err(self, response):
        print('err:',response.request.url)
        # print('err:',response.status)
        # print(dir(response))
        print('err:',response.getErrorMessage())
        print(dir(response))
        # print(type(response.getErrorMessage()))