Python爬虫框架

本文章的源代码来源于https://github.com/Holit/Web-Crawler-Framwork

一、爬虫框架的代码

 import urllib.request
 from bs4 import BeautifulSoup
 import re
 import time
 import _thread

 # Input your Url here####################################
 BaseURL = '127.0.0.1/'
 #########################################################
 TaxURL = ".html"

 #Input your data-saving path ############################
 SavePath = ""
 #########################################################

 #Input your threads count ###############################
 thread_count = 1
 #########################################################

 #Set each spider will spy how many pages ################
 thread_spy_count_ench = 5
 #########################################################
 def mkdir(path):
     # Create the directory
     import os
     path=path.strip()
     path=path.rstrip("\\")
     isExists=os.path.exists(path)
     if not isExists:
         os.makedirs(path)
         return True
     else:
         return False

 def download(start, count):
     #Spider main
     for i in range(start,start + count):
         try:
             #DEBUG##################################################
             #print("[INFO] Connecting to page #" + str(i) + "...")
             ########################################################

             #Used to record time
             time_start=time.time()

             #Construct url
             #This only work like
             # https://127.0.0.1/articles/00001.html
             # https://127.0.0.1/articles/00002.html
             # https://127.0.0.1/articles/00003.html
             TargetURL = BaseURL + str(i) + TaxURL

             #create Request object
             req = urllib.request.Request(TargetURL)
             #create headers using general header, you could find this by Fiddler(R) or by Chrome(R)
             req.add_header('Host','')    #Your Host, usally set as url-base
             req.add_header('Referer',TargetURL)        #Your Referer, usally set as url
             req.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166  Safari/535.19')
             #finishing create Request object

             #get information
             res = urllib.request.urlopen(req)
             #decode the html
             soup = BeautifulSoup(res,"html.parser")
             ##############################################################
             #add your functions  here....

             #operate_data(data)

             #soup find div to get inforamtion.
             #soup is able to opreate html tag very easily, by using soup.find(...)
             ##############################################################

             #Change saving path here.
             savetarget = SavePath

             #trying to saving files
             try:
                 #create directory if it doesn't existed
                 mkdir(SavePath+"\\"+str(zone)+"\\")
                 #using open...
                 f = open(savetarget,'w')

                 #edit this
                 f.write("data")

             except Exception as e:
                 time_end=time.time()
                 print("  [Failed] - #" + str(i) + " Error : " + str(e))
             else:

                 time_end=time.time()
                 print("  [Succeed] - #" + str(i) + " has saved to path.("+str(time_end-time_start)+"s)")

             pass
         except Exception as e:
             print("  [Global Failure] - #" + str(i) + " Error : " + str(e))
             pass

 #if __name__ == __main__:
 try:
     #Multithreading
     print("Spidering webiste...")
     print("Current configuration :")
     print("--Will create " + str(thread_count) + "threads to access.")
     print("--Will save to " + SavePath)
     print("-------------START---------------------------")
     # press any key to continue
     # this won't work under linux
     import os
     os.system('pause')
     try:
         for i in range(0,thread_count):
             print("[Thread #"+ str (i) +"] started successfully")
             _thread.start_new_thread(download, (thread_spy_count_ench * i,thread_spy_count_ench))
     except Exception as e:
         print("[Threading@" + str(i) +"] Error:"+ str(e))
 except Exception as e:
    print("[Global Failure] Error:"+ str(e))
 while 1:
    pass

二、对其中功能的实例化操作

　　1.文本获取功能

　　　　文本获取是指对页面的<div class='content'>...</div>中的内容进行获取，这是前提。如果不同需要更改。

　　　　（1）思路

　　　　　　使用BeautifulSoup对html分析之后得到解码的文件，例如

             <div class="content" style="text-align: left">
             基础内容
             </div>

　　　　　　现在对该段落进行选取，即使用soup.find功能

　　　　（2）基本代码

 passages_div = soup.find('div')
 passages_set = passages_div.findAll(attrs={"class":"content"})
 for passages in passages_set:
     article = str(passages)
     #文字处理
     article = article.replace('<div class="content" style="text-align: left">', '')
     article = article.replace(u'\ue505', u' ')#对Unicode的空格进行处理，如果不处理gbk无法编码
     article = article.replace(u'\ue4c6', u' ')
     article = article.replace(u'\xa0', u' ')
     article = article.replace('<br/>', '\n')
     article = article.replace('</div>', '')
     savetarget = 'D:\test\test.txt'
     try:
         mkdir('D:\test\')
         f = open(savetarget,'w')
         f.write(article)
     except Exception as e:
         print("  [Failed] - "+ str(e))
     else:
         time_end=time.time()
         print("  [Succeed] - saved to path.")

 pass

　　2.图片获取操作

　　　　图片获取一般是通过对网页上的<img src="127.0.0.1/png.png">Hello</img>中src上的内容进行下载操作

　　　　目前可以使用多种操作方式，例如urlretrieve，不再赘述