python查找并删除相同文件-UNIQ File-wxPython版本

今天用wxPython做了一个GUI程序，我称之为UNIQ File，实现查找指定目录内的相同文件，主要原理是计算文件的md5值（计算前先找出文件大小相同的文件，然后计算这些文件的md5值，而不是所有文件都计算，大大减少了md5的计算量），加入了多线程功能。

运行该程序需要安装wxPython。

界面图

源代码：

UNIQFile-wxPython.py

 # -*- coding: gbk -*-

 '''
 Author:@DoNotSpyOnMe
 Blog: http://www.cnblogs.com/aaronhoo
 '''

 import wx
 import hashlib
 import os
 import threading

 class WorkerThread(threading.Thread):
     def __init__(self, frame,dir,operation,msg):
         """初始化工作线程: 把主窗口传进来"""
         threading.Thread.__init__(self)
         self.frame = frame
         self.dir=dir
         self.operation=operation
         self.msg=msg
         self.setDaemon(True)#设置子线程随UI主线程结束而结束
         self.start() 

     #----------------------------------------------------------------------
     def run(self):
         """执行工作线程"""
         self.frame.SetButtons('operating')
         try:
             if self.operation=='list':
                 self.listSameFile(self.dir)
                 self.frame.btnList.Enable()
             elif self.operation=='remove':
                 self.removeSameFile(self.dir)
                 self.frame.btnRemove.Enable()
         except:
             pass
         finally:
             self.frame.SetButtons('completed')
 #
 #     def stop(self):
 #         self.keepRunning=False
     def appendMsg(self,msg):
         if self.frame:
             #以下方式可以实现终端式的刷新：自动滚动到最新行
             self.frame.txtContent.AppendText(msg+'\n')
             #废弃的方式
 #             currentMsg=self.frame.txtContent.GetValue()
 #             currentMsg=currentMsg+'\n'+msg
 #             self.frame.txtContent.SetValue(currentMsg)

     def getFileSize(self,filePath):
         return os.path.getsize(filePath)        

     ''' 一般文件的md5计算方法,一次读取文件的全部内容'''
     def CalcMD5(self,filepath):
         with open(filepath,'rb') as f:
             md5obj = hashlib.md5()
             md5obj.update(f.read())
             hash = md5obj.hexdigest()
             return hash
     '''大文件计算md5的方法，分批读取文件内容，防止内存爆掉'''
     def GetFileMd5(self,filename):
         if not os.path.isfile(filename):
             return
         myhash = hashlib.md5()
         f = open(filename,'rb')
         while True:
             b = f.read(8*1024)
             if not b :
                 break
             myhash.update(b)
         f.close()
         return myhash.hexdigest()

     def GetAllFiles(self,directory):
         files=[]
         for dirpath, dirnames,filenames in os.walk(directory):
             if filenames!=[]:
                 for file in filenames:
                     files.append(dirpath+'\\'+file)
　　　　　　　files.sort(key=len)#按照文件名的长度排序　
         return files

     def findSameSizeFiles(self,files):
         dicSize={}
         for f in files:
             size=self.getFileSize(f)
             if not dicSize.has_key(size):
                 dicSize[size]=f
             else:
                 dicSize[size]=dicSize[size]+';'+f
         dicCopy=dicSize.copy()
         for k in dicSize.iterkeys():
             if dicSize[k].find(';')==-1:
                dicCopy.pop(k)
         del dicSize
         return dicCopy

     def findSameMD5Files(self,files):
         dicMD5={}
         for f in files:
             self.appendMsg('calculating the md5 value of file %s'%f)
             md5=self.GetFileMd5(f)
             if not dicMD5.has_key(md5):
                 dicMD5[md5]=f
             else:
                 dicMD5[md5]=dicMD5[md5]+';'+f
         dicCopy=dicMD5.copy()
         for k in dicMD5.iterkeys():
             if dicMD5[k].find(';')==-1:
                dicCopy.pop(k)
         del dicMD5
         return dicCopy

     def removeSameFile(self,mydir):
         msg=''
         msgUniq='Congratulations,no file is removed since they are all uniq.'
         try:
             existsFlag=False
             files=self.GetAllFiles(mydir)
             self.appendMsg('%s files found in directory %s\n'%(len(files),mydir))
             dicFileOfSameSize=self.findSameSizeFiles(files)
             if dicFileOfSameSize=={}:
                 self.appendMsg(msgUniq)
                 return
             else:
                 #list the duplicated files first:
                 dicFiltered={}
                 for k in dicFileOfSameSize.iterkeys():
                     filesOfSameSize=dicFileOfSameSize[k].split(';')
                     dicSameMD5file=self.findSameMD5Files(filesOfSameSize)
                     if dicSameMD5file!={}:
                         existsFlag=True
                         for k in dicSameMD5file.iterkeys():
                             msg=msg+'md5 %s: %s'%(k,dicSameMD5file[k])+'\n'
                             dicFiltered[k]=dicSameMD5file[k]
                 if not existsFlag:
                     msg=msgUniq
                     return
                 else:
                     msg='Duplicated files:\n'+msg+'\n'
                     #then remove the duplicated files:
                     removeCount=0
                     for k in dicFiltered.iterkeys():
                         sameFiles=dicFiltered[k].split(';')
                         flagRemove=False
                         for f in sameFiles:
                             if not flagRemove:
                                 flagRemove=True
                             else:
                                 msg=msg+'Removing file: %s'%f+'\n'
                                 os.remove(f)
                                 removeCount=removeCount+1
                     msg=msg+'%s files are removed.\n'%removeCount
         except Exception,e:
 #             print e
             msg='Exception occured.'
         finally:
             self.appendMsg(msg+'\n'+'Operation finished.')

     def listSameFile(self,mydir):
         msg=''
         msgUniq='Congratulations,all files are uniq.'
         try:
             existsFlag=False
             files=self.GetAllFiles(mydir)
             self.appendMsg('%s files found in directory %s\n'%(len(files),mydir))
             dicFileOfSameSize=self.findSameSizeFiles(files)
             if dicFileOfSameSize=={}:
                 self.appendMsg(msgUniq)
                 return
             else:
                 for k in dicFileOfSameSize.iterkeys():
                     filesOfSameSize=dicFileOfSameSize[k].split(';')
                     dicSameMD5file=self.findSameMD5Files(filesOfSameSize)
                     if dicSameMD5file!={}:
                         existsFlag=True
                         for k in dicSameMD5file.iterkeys():
                             msg=msg+'md5 %s: %s'%(k,dicSameMD5file[k])+'\n'
                 if not existsFlag:
                     msg=msgUniq
                 else:
                     msg='Duplicated files:\n'+msg
         except Exception,e:
 #             print e
             msg='Exception occured.'
         finally:
             self.appendMsg(msg+'\n'+'Operation finished.')

 class MyFrame(wx.Frame):
     def __init__(self):
         super(MyFrame,self).__init__(None,title='UNIQ File-wxPython',size=(780,450))
         pan=wx.Panel(self)
         self.lblDir=wx.StaticText(pan,-1,'Dir:',style=wx.ALIGN_LEFT)
         self.txtFile=wx.TextCtrl(pan,size=(380,30))
 #         self.txtFile.Disable()
         self.btnOpen=wx.Button(pan,label='Pick Directory')
         self.btnOpen.Bind(wx.EVT_BUTTON, self.BtnOpenHandler)
         self.btnList=wx.Button(pan,label='Find Same')
         self.btnList.Bind(wx.EVT_BUTTON, self.BtnListHandler)
         self.btnRemove=wx.Button(pan,label='Remove duplicated')
         self.btnRemove.Bind(wx.EVT_BUTTON, self.BtnRemoveHandler)
 #         self.btnStop=wx.Button(pan,label='Stop')
 #         self.btnStop.Bind(wx.EVT_BUTTON, self.BtnStopHandler)

         hbox=wx.BoxSizer()
         hbox.Add(self.lblDir,proportion=0,flag=wx.LEFT,border=5)
         hbox.Add(self.txtFile,proportion=0,flag=wx.LEFT,border=5)
         hbox.Add(self.btnOpen,proportion=0,flag=wx.LEFT,border=5)
         hbox.Add(self.btnList,proportion=0,flag=wx.LEFT,border=5)
         hbox.Add(self.btnRemove,proportion=0,flag=wx.LEFT,border=5)
 #         hbox.Add(self.btnStop,proportion=0,flag=wx.LEFT,border=5)

         self.txtContent=wx.TextCtrl(pan,style=wx.TE_MULTILINE|wx.HSCROLL)
         vbox=wx.BoxSizer(wx.VERTICAL)
         vbox.Add(hbox,proportion=0,flag=wx.EXPAND|wx.ALL,border=5)
         vbox.Add(self.txtContent,proportion=1,flag=wx.EXPAND,border=5)
         pan.SetSizer(vbox)
 #         self.SetButtons('init')

     def BtnOpenHandler(self,event):
         dlg = wx.DirDialog(None,u"选择文件夹",style=wx.DD_DEFAULT_STYLE)
         if dlg.ShowModal() == wx.ID_OK:
             dlg.Destroy()
             if dlg.GetPath():
                 self.dirSelected=dlg.GetPath() #文件夹路径
                 self.txtFile.SetValue(self.dirSelected)      

                 self.SetButtons('selected')
                 self.txtContent.SetValue('Selected dirctory: %s\n'%self.dirSelected)

     def BtnListHandler(self,event):
         if not self.txtFile.GetValue() or not os.path.isdir(self.txtFile.GetValue()):
             wx.MessageBox('please select a valid directory first.','Tip Message',wx.YES_DEFAULT|wx.ICON_INFORMATION)
             return
         self.dirSelected=self.txtFile.GetValue()
         self.txtContent.SetValue('')
         msg='Listing same files in %s\n'%self.dirSelected
         self.txtContent.SetValue(msg)
         workThread=WorkerThread(self,self.dirSelected,'list',msg)

     def BtnRemoveHandler(self,event):
         if not self.txtFile.GetValue() or not os.path.isdir(self.txtFile.GetValue()):
             wx.MessageBox('please select a valid directory first.','Tip Message',wx.YES_DEFAULT|wx.ICON_INFORMATION)
             return
         self.dirSelected=self.txtFile.GetValue()
         self.txtContent.SetValue('')
         msg='Removing duplicated files in %s\n'%self.dirSelected
         self.txtContent.SetValue(msg)
         WorkerThread(self,self.dirSelected,'remove',msg)

     def BtnStopHandler(self,event):
         pass

     def SetButtons(self,status):
         if status=='init':
             self.btnOpen.Enable()
             self.btnList.Disable()
             self.btnRemove.Disable()
 #             self.btnStop.Disable()
         elif status=='operating':
             self.btnOpen.Disable()
             self.btnList.Disable()
             self.btnRemove.Disable()
 #             self.btnStop.Enable()
         elif status=='completed':
             self.btnOpen.Enable()
             self.btnList.Enable()
             self.btnRemove.Enable()
 #             self.btnStop.Disable()
         elif status=='selected':
             self.btnOpen.Enable()
             self.btnList.Enable()
             self.btnRemove.Enable()
 #             self.btnStop.Disable() 

 if __name__=="__main__":
     app=wx.App()
     MyFrame().Show()
     app.MainLoop()