beautifulsoup爬取糗事百科

 # _*_ coding:utf-8 _*_
 import urllib2
 from bs4 import BeautifulSoup
 
 user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0"
 headers = {'User-Agent':user_agent}
 url = "https://www.qiushibaike.com"
 # 爬取article链接的content内容
 def getContent(article_url, headers):
     request = urllib2.Request(article_url,data=None,headers=headers)
     response = urllib2.urlopen(request,timeout=60)
     html = response.read().decode('utf-8')
     soup = BeautifulSoup(html,'html.parser')
     contents= soup.select('.content')[0].strings
     print u"内容："
     for content in contents:
         print u"%s" % content.strip()
     print '\n'
 # ----------------------------
 # 获取"https://www.qiushibaike.com/hot/page/1/"页面的作者、好笑、评论，文章地址信息
 def getData(url, headers,pages=1):
     for page in range(1,pages+1):
         page_url = url + "/hot/page/" + str(page)
         print "正在爬取第 %s 页+++" % page
         request = urllib2.Request(page_url,data=None,headers=headers)
         response = urllib2.urlopen(request,timeout=60)
         html = response.read().decode('utf-8')
         # print html
         soup = BeautifulSoup(html,'html.parser')
 
         authors = soup.select('h2')
         smile_nums = soup.select('.stats-vote > .number')
         comment_nums = soup.select('.stats-comments > .qiushi_comments > .number')
         article_urls = soup.select('.contentHerf')
         for i in range(25):
             print "正在爬取第 %s 页的第 %s 条数据---" % (page,i+1)
             author = authors[i].string.strip()
             print u"作者： %s" % author
             funny_num = smile_nums[i].string
             comment_num = comment_nums[i].string
             print u"好笑： %s" % funny_num
             print u"评论： %s"% comment_num
             article_url = article_urls[i]['href']
 
             article_url = url + article_url
             # print article_url
             getContent(article_url, headers)
 # ---------------------------------
 getData(url,headers,pages=10)