Flask开发系列之Flask+redis实现IP代理池

Flask开发系列之Flask+redis实现IP代理池

代理池的要求

• 多站抓取，异步检测：多站抓取：指的是我们需要从各大免费的ip代理网站，把他们公开的一些免费代理抓取下来；一步检测指的是：把这些代理通过异步请求的方式，利用这些代理请求网站：如果能正常请求就证明代理可用，如果不能正常请求就证明代理不行，这时就可以把这个代理剔除掉，异步指的是：我们不需要一直等待代理请求网站，到得到response之后在执行相应的操作就可以了，异步可以提高检测效率。

• 定时筛选，持续更新：我们维护一个代理池，我们需要做的是需要定时从里面拿出一部分来检测，剔除掉不可用的代理。这可以保证代理是可用的

• 提供接口，易于提取：代理实际上是维护在一个队列中，队列可以使用数据库存储，也可以使用一些数据结构来存储，但是如果要获取代理的话，要提供一个简单的接口，最简单的是web形式的接口：本文主要演示一个利用python flask包来提供接口：之后使用python请求网址，从网页中拿到代理的信息了

代理池的架构

• 获取器：从各大网站平台抓取代理：ip和端口

• 过滤器：剔除掉不可用的代理

• 将可用代理放到代理队列

• 定时检测器：剔除不可用的代理

• API：通过接口形式拿到代理对象，方便使用

测试实现版

import requests
import re
import time
import redis
from bloom_filter import BloomFilter
import ast

pool = redis.ConnectionPool(host='localhost',password='xxx', port=6379, decode_responses=True)
r = redis.Redis(connection_pool=pool)
bloombloom = BloomFilter(max_elements=10000, error_rate=0.1)
bloombloom.add(str({'http': '117.91.232.53:9999'}))

def get_ip(i):
    ip_list=[]
    url = 'https://www.kuaidaili.com/free/inha/'
    url = url + str(i + 1)
    html = requests.get(url=url, ).text
    regip = '<td.*?>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>.*?<td.*?>(\d{1,5})</td>'
    matcher = re.compile(regip, re.S)
    ipstr = re.findall(matcher, html)
    time.sleep(1)
    for j in ipstr:
        ip_list.append(j[0] + ':' + j[1])
    print('共收集到%d个代理ip' % len(ip_list))
    print(ip_list)
    return ip_list

def valVer(proxys):
    global badNum,goodNum,good_list
    good = []
    for proxy in proxys:
        try:
            proxy_host = proxy
            protocol = 'https' if 'https' in proxy_host else 'http'
            proxies = {protocol: proxy_host}
            print('现在正在测试的IP：', proxies)
            response = requests.get('http://www.baidu.com', proxies=proxies, timeout=2)
            if response.status_code != 200:
                badNum += 1
                print(proxy_host, 'bad proxy')
            else:
                goodNum += 1
                good.append(proxies)
                good_list.append(proxies)
                print(proxy_host, 'success proxy')
        except Exception as e:
            print(e)
            # print proxy_host, 'bad proxy'
            badNum += 1
            continue
    print('success proxy num : ', goodNum)
    print('bad proxy num : ', badNum)
    print("这次：",good)
    print("此时全部：",good_list)
    return good

def time_valVer(proxys):
    good = []
    for proxy in proxys:
        try:
            print('现在正在定时测试的IP：',proxy)
            proxy = ast.literal_eval(proxy)
            response = requests.get('http://www.baidu.com', proxies=proxy, timeout=2)
            if response.status_code != 200:
                r.lrem("ip_list", proxy, 1)
                print(proxy, 'bad proxy')
            else:
                good.append(proxy)
                good_list.append(proxy)
                print(proxy, 'success proxy')
        except Exception as e:
            print(e)
            continue

def stone(good):
    for IP in good:
        if str(IP) in bloombloom:
            print("%s不能存储，有相同的IP",IP)
            continue
        else:
            print("存储的IP：", IP)
            bloombloom.add(str(IP))
            r.rpush("ip_list", str(IP))

if __name__ == '__main__':

    badNum = 0
    goodNum = 0
    good_list = []
    for i in range(0,10):
        if i%10 == 0 and i!=0:
            proxy_list = []
            for i in range(0, r.llen("ip_list")):
                proxy_list.append(r.lindex("ip_list", i))
            time_valVer(proxy_list)
        else:
            ip_list = get_ip(i)
            good = valVer(ip_list)
            stone(good)

from flask import Flask
import redis   # 导入redis模块，通过python操作redis 也可以直接在redis主机的服务端操作缓存数据库

r = redis.Redis(host='localhost', port=6379,password='xxx',decode_responses=True)
app = Flask(__name__)
@app.route('/ip/<int:index>')
def reponse(index):
    print(index)
    print(r.lindex("ip_list", index))
    return r.lindex("ip_list", index)
if __name__ == '__main__':
    app.run(debug=True)

获取ip：

改进版

import requests
import re
import time
import redis
from bloom_filter import BloomFilter
import ast

pool = redis.ConnectionPool(host='localhost',password='XXX', port=6379, decode_responses=True)
r = redis.Redis(connection_pool=pool)
bloombloom = BloomFilter(max_elements=10000, error_rate=0.1)

def get_ip(i):
    ip_list=[]
    url = 'https://www.kuaidaili.com/free/inha/'
    url = url + str(i + 1)
    html = requests.get(url=url, ).text
    regip = '<td.*?>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>.*?<td.*?>(\d{1,5})</td>'
    matcher = re.compile(regip, re.S)
    ipstr = re.findall(matcher, html)
    time.sleep(1)
    for j in ipstr:
        ip_list.append(j[0] + ':' + j[1])
    print('共收集到%d个代理ip' % len(ip_list))
    print(ip_list)
    return ip_list

def valVer(proxys):
    global badNum,goodNum,good_list
    good = []
    for proxy in proxys:
        try:
            proxy_host = proxy
            protocol = 'https' if 'https' in proxy_host else 'http'
            proxies = {protocol: proxy_host}
            response = requests.get('http://www.baidu.com', proxies=proxies, timeout=2)
            if response.status_code != 200:
                badNum += 1
            else:
                goodNum += 1
                good.append(proxies)
                good_list.append(proxies)
        except Exception as e:
            print(e)
            badNum += 1
            continue
    print('success proxy num : ', goodNum)
    print('bad proxy num : ', badNum)
    print("这次：",good)
    print("此时全部：",good_list)
    return good

def time_valVer(proxys):
    for proxy in proxys:
        try:
            print('现在正在定时测试的IP：',proxy)
            proxy = ast.literal_eval(proxy)
            response = requests.get('http://www.baidu.com', proxies=proxy, timeout=2)
            if response.status_code != 200:
                r.lrem("ip_list", proxy, 1)
        except Exception as e:
            print(e)
            continue

def stone_redis(good):
    for IP in good:
        if str(IP) in bloombloom:
            print("%s不能存储，有相同的IP",IP)
            continue
        else:
            print("存储的IP：", IP)
            bloombloom.add(str(IP))
            r.rpush("ip_list", str(IP))

def init():
    for i in range(0, r.llen("ip_list")):
        print(r.lindex("ip_list", i))
        bloombloom.add(r.lindex("ip_list", i))

if __name__ == '__main__':
    badNum = 0
    goodNum = 0
    good_list = []
    init()
    for i in range(0,10):
        if i%2 == 0 and i!=0:
            proxy_list = []
            for i in range(0, r.llen("ip_list")):
                proxy_list.append(r.lindex("ip_list", i))
            time_valVer(proxy_list)
        else:
            ip_list = get_ip(i)
            good = valVer(ip_list)
            stone_redis(good)

from flask import Flask, abort, request, jsonify
import redis   # 导入redis模块，通过python操作redis 也可以直接在redis主机的服务端操作缓存数据库

r = redis.Redis(host='localhost', port=6379,password='XXX',decode_responses=True)
app = Flask(__name__)
@app.route('/ip/<int:index>', methods=['GET'])
def reponse(index):
    print(index)
    ip = {"ip":r.lindex("ip_list", index)}
    print(r.lindex("ip_list", index))
    return jsonify(ip)
if __name__ == '__main__':
    app.run(debug=True)

获取ip：