多线程网页爬虫 python 实现（二）

#!/usr/bin/env python
#coding=utf-8
import threading
import urllib
import re
import time
 
cur=0
last=0
totalcount=0
depth=0
t_mutex=threading.Condition() 
 
class Mycrawler:
    def __init__(self,crawlername,seeds,threadnum):
        self.crawlername=crawlername
        self.seeds=seeds
        self.crawqueue=CrawQueue()
        self.initQueue(self.seeds)
        self.threadnum=threadnum
        self.threadpools=[]
        self.logfile=file('log2.txt','w')
    def initQueue(self,seeds):
        if isinstance(seeds,str):
            self.crawqueue.push(seeds)
        elif isinstance(seeds,list):
            for seed in seeds:
                self.crawqueue.push(seed)
        global last
        global totalcount
        totalcount=self.crawqueue.getQueueCount()
        last=totalcount
    def crawling(self):
        global cur
        global depth
        global last
        global totalcount
        self.log(">>>Depth "+str(depth)+":\n")
        while self.crawqueue.getQueueCount()!=0:
            url=self.crawqueue.pop()
            self.log(url)
            if url==None:
                continue
            self.crawqueue.addToVisited(url)
            links=self.getLinks(url)
            if links==None:
                print 'None'
                self.crawqueue.failed.append(url)
                continue
            beforenum = self.crawqueue.getQueueCount()
            self.crawqueue.addLinks(links)
            afternum  = self.crawqueue.getQueueCount()
            totalcount+=afternum-beforenum
            cur+=1
            if cur==last:
                depth+=1
                self.log(">>>Depth "+str(depth)+":\n")
                last=totalcount
    def crawling2(self):
        global last
        global totalcount
        global depth
        self.log(">>>Depth "+str(depth)+":\n")
        totalcount=self.crawqueue.getQueueCount()
        last=totalcount
        while self.crawqueue.getQueueCount()!=0:
            for i in range(self.threadnum):
                url=self.crawqueue.pop()
                if url==None:
                    break
                crawthread=crawlerThread(url,i,self)
                self.threadpools.append(crawthread)
                crawthread.start()
            for i in range(len(self.threadpools)):
                crawthread=self.threadpools[i]
                crawthread.join(30)
    def log(self,content):
        self.logfile.write(content+"\n")
class crawlerThread(threading.Thread):
    def __init__(self,url,tid,mycrawler):
        threading.Thread.__init__(self)
        self.url=url
        self.tid=tid
        self.mycrawler=mycrawler
    def run(self):
        global t_mutex
        global cur
        global last
        global totalcount
        global depth
        t_mutex.acquire()
        self.mycrawler.log(self.url)
        t_mutex.release()
        links=self.getLinks(self.url)
        if links==None:
            t_mutex.acquire()
            self.mycrawler.crawqueue.addToVisited(self.url)
            self.mycrawler.crawqueue.addToFailed(self.url)
            t_mutex.release()
        else:
            t_mutex.acquire()
            self.mycrawler.crawqueue.addToVisited(self.url)
            beforenum=self.mycrawler.crawqueue.getQueueCount()
            self.mycrawler.crawqueue.addLinks(links)
            afternum =self.mycrawler.crawqueue.getQueueCount()
            totalcount+=afternum-beforenum
            t_mutex.release()
        t_mutex.acquire()
        cur+=1
        if cur==last:
            depth+=1
            self.mycrawler.log(">>>Depth "+str(depth)+":\n")
            last=totalcount
        t_mutex.release()
    def getLinks(self,url):
        try:
            page=urllib.urlopen(url)
            html=page.read()
            reg=r'"(http://.+?)"'
            regob=re.compile(reg,re.DOTALL)
            links=regob.findall(html)
            return links
        except:
            print 'Failed downloading and saving',url
            return None
class CrawQueue:
    def __init__(self):
        self.queue=[]
        self.visited=[]
        self.failed=[]
    def getQueue(self):
        return self.queue
    def getVisited(self):
        return self.visited
    def getFailed(self):
        return self.failed
    def push(self,url):
        if url!="" and url not in self.queue and url not in self.visited:
            self.queue.insert(0,url)
    def pop(self):
        if len(self.queue)==0:
            #print 'failed to pop: queue is empty'
            return None
        else:
            return self.queue.pop()
    def isEmpty(self):
        if len(self.queue)==0:
            return 1
        else:
            return 0
    def addToVisited(self,url):
        self.visited.append(url)
    def addToFailed(self,url):
        self.failed.append(url)
    def remove(self,url):
        self.queue.remove(url)
    def getVisitedCount(self):
        return len(self.visited)
    def getQueueCount(self):
        return len(self.queue)
    def addLinks(self,links):
        for link in links:
            self.push(link)
 
if __name__=="__main__":
    seeds="http://www.douban.com/"
    threadnum=int(raw_input("设置线程数:"))
    crawlername="小小爬虫"
    mycrawler=Mycrawler(crawlername,seeds,threadnum)
    mycrawler.crawling2()