aiohttp的模板

 import aiohttp
 import asyncio
 import async_timeout
 from urllib.parse import urljoin,urldefrag
 
 root_url = 'http://python/org/'  # 开始的url
 crawled_urls,url_hub = [], [root_url]
 headers = {'user-agent': 'Opera/9.80 (X11; Linux x86_64; U; en) Presto/2.2.15 Version/10.10'}
 
 async def get_body(url):
     async with aiohttp.ClientSession() as session:
         try:
             with async_timeout.timeout(10): #超时时间的设定
                 async with session.get(url,headers=headers) as response:
                     if response.status == 200:
                         html = await response.text()
                         return {'error':'','html':html,'url':url}
                     else:
                         return {'error':response.status,'html':'','url':url}
         except Exception as err:
             return {'error':response.status,'html':'','url':url}
 
 async def handle_task(task_id,work_queue):
     while not work_queue.empty():
         queue_url = await work_queue.get()
         if not queue_url in crawled_urls:
 
             body = await get_body(queue_url)
             if not body['error']:
                 crawled_urls.append(queue_url)
                 parse(body)
             else:
                 print('{}爬取失败'.format(queue_url))
 
 #解析返回的数据
 def parse(body):
     pass
 
 def remove_fragment(url):
     pure_url, frag = urldefrag(url)
     return pure_url
 
 #解析html，拼接新的url
 def get_urls(html):
     new_urls = [url.split('"')[0] for url in str(html).replace("'", '"').split('href="')[1:]]
     return [urljoin(root_url, remove_fragment(new_url)) for new_url in new_urls]
 
 if __name__ == '__main__':
     q = asyncio.Queue()  #初始化一个异步的队列
     [q.put_nowait(url) for url in url_hub]  #从初始的url队列中遍历，把url放入到队列中
     loop = asyncio.get_event_loop()
     tasks = [handle_task(task_id, q) for task_id in range(3)]  #3个并发
     loop.run_until_complete(asyncio.wait(tasks))
     loop.close()
     for u in crawled_urls:
         print(u)
     print('-' * 30)
     print(len(crawled_urls))