简单的百度贴吧爬虫实现（urllib）

环境：ubuntu 16.04 LTS   （X86-64），pycharm

python版本 ：3.5.1+

#生成的文件默认会保存到代码所在根目录

 1 import urllib.request,urllib.error,re

 class Tool:
     removeImg=re.compile('<img.*?| {7}|')
     removeAddr=re.compile('<a.*?|</a>')
     replaceLine=re.compile('<tr>|<div>|</div>|</p>')
     replaceTD=re.compile('<td>')
     replaceBR=re.compile('<br></br>|br')
     replaceExtra=re.compile('<.*?>')
     def replace(self,x):
         x=re.sub(self.removeImg,"",x)
         x=re.sub(self.removeAddr,"",x)
         x=re.sub(self.replaceLine,"\n",x)
         x=re.sub(self.replaceTD,"\t",x)
         x=re.sub(self.replaceBR,"\n",x)
         x=re.sub(self.replaceExtra,"",x)
         return x.strip()

 class BDTB:
     def __init__(self,baseUrl,see_lz):
         self.tool=Tool()
         self.baseurl=baseUrl+'?see_lz='+str(see_lz)+'&pn='
         self.defaultTitle=u'百度贴吧'

     def getPage(self,pagenum):
         try:
             url=self.baseurl+str(pagenum)
             request=urllib.request.Request(url)
             response=urllib.request.urlopen(request)
             content = response.read().decode('utf-8')
             return content
         except urllib.error.URLError as e:
             if hasattr(e,"reason"):
                 print(u'connect error reason:'+e.reason)
             if hasattr(e,'code'):
                 print(u'connect error,reason:'+e.code)

     def getPns(self,content):
         pattern = re.compile('<li class="l_reply_num".*?<span class="red">(.*?)</span>', re.S)
         pns = int((re.findall(pattern, content))[0])
         return pns

     def getTitle(self,content):
         pattern=re.compile('<h3 class="core_title_txt pull-left text-overflow  ".*?>(.*?)</h3>',re.S)
         return str((re.findall(pattern,content))[0])

     def getContent(self,content):
         pattern=re.compile('<ul class="p_author".*?<li class="d_name".*?target="_blank">(.*?)</a>.*?<div id="post_content_.*?>(.*?)</div>',re.S)
         items=re.findall(pattern,content)
         contents=[]
         for item in items:
             content='Username:   '+item[0]+'   content:   '+self.tool.replace(item[1])+'\n'
             contents.append(content)
         return contents

     def setFileTitle(self,Title):
         if Title is not None:
             self.file=open(Title+'.txt','w+')
         else:
             self.file=open(self.defaultTitle+'.txt','w+')

     def WriteData(self,contents):
         for content in contents:
             self.file.write(content)

     def start(self):
         Pns=self.getPns(self.getPage(1))
         self.setFileTitle(self.getTitle(self.getPage(1)))
         for i in range(Pns):
             print('Page Sum:'+str(Pns)+'\n')
             print('Now is Write page:'+str(i)+'\n')
             self.WriteData(self.getContent(self.getPage(i)))
         self.file.close()

 print('please enter discussion num:')
 url='http://tieba.baidu.com/p/'+str(input())
 see_lz=input('Whether just see lz(enter 0 or 1)')
 bdtb=BDTB(url,see_lz)
 bdtb.start()

运行结果：