RandomUserAgentMiddleware练习

# 请求头添加随机user-agent
class RandomUserAgentMiddleware(object):

    def __init__(self, agents):
        self.agent = agents

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            agents=crawler.settings.get('CUSTOM_USER_AGENT')
        )

    def process_request(self, request, spider):
        request.headers['User-Agent']=random.choice(self.agent)

php

yum list --enablerepo=remi --enablerepo=remi-php56 | grep php;