基于python第三方requests 模块的HTTP请求类

使用requests模块构造的下载器,首先安装第三方库requests

pip install requests

 1 class StrongDownload(object):
 
     def __init__(self):
         #拿到代理iplist
         self.iplist = ['自己想办法搞'] #
         self.UserAgent = ['自己想办法搞']  
 
     def get(self,url,timeout,proxy=False,num_retries=3):
         '''url timeout,proxy,num_retries
                 返回response对象
         '''
         #伪造UserAgent
         UA = self.UserAgent[random.randint(0,len(self.UserAgent)-1)]
         #UA = random.choice(self.UserAgent)
         headers = {'User-Agent':UA}
         if proxy == False:
             try:
                 return requests.get(url,timeout=timeout,headers=headers)
             except Exception as e:
                 print e
                 #判断retries
                 if num_retries>0:
                     print u'爬取网页错误,10s后继续尝试.....'
                     time.sleep(1)
                     return self.get(url, timeout, False, num_retries-1)
                 else:#重试次数用尽,使用代理
                     time.sleep(1)
                     IP = str(random.choice(self.iplist))
                     proxy = {'http',IP}
                     return self.get(url, timeout, True,5)
 
         else:#使用代理
 
             try:
                 print '开始使用代理'
                 IP = random.choice(self.iplist)
                 proxy = {'http':IP}
                 print proxy['http']
                 return requests.get(url,headers=headers, timeout=timeout, proxies=proxy)
             except:
                 if num_retries>0:
                     print u'代理爬取网页错误,10s后继续尝试.....'
                     time.sleep(1)
                     return self.get(url, timeout,True, num_retries-1)
                 else:#代理也不行
                     time.sleep(30)
                     print u'代理也没用...'
                     return None