无比强大！Python抓取cssmoban网站的模版并下载

Python实现抓取http://www.cssmoban.com/cssthemes网站的模版并下载

实现代码

1. # -*- coding: utf-8 -*-
2. import urlparse
3. import urllib2
4. import re
5. import os
6. import os.path
7. URL='http://www.cssmoban.com/cssthemes'
8. #全局超时设置
9. urllib2.socket.setdefaulttimeout(500)
10. #根据url获取内容
11. def getUrlContent(url):
12. response = urllib2.urlopen(url)
13. html = response.read();
14. return html
15. #获取html中的a标签，且格式是<a target="_blank" href="/showcase/*">的
16. def getAllUrl(html):
17. return re.findall('<a[\\s]+href="/cssthemes/\d+\.shtml">.*?\/a>',html)
18. #获取下载文件的标题
19. def getDownTitle(html):
20. return re.findall('\<h1>(.*?)\</h1>',html)
21. #获取文件下载的url
22. def getDownUrl(html):
23. return re.findall('<a.*?class="button btn-down".*?\/a>',html)
24. #获取下一页的url
25. def getNextUrl(html):
26. return re.findall('<a.*?下一页</a>',html)
27. #下载文件
28. def download(title,url):
29. result = urllib2.urlopen(url).read()
30. if os.path.exists("template/")==False:
31. os.makedirs("template/")
32. newname=("template/"+title.decode('utf-8'))
33. newname=newname+'.'+url[url.rfind('.')+1:len(url)]
34. open(newname, "wb").write(result)
35. #记录日志
36. def i(msg):
37. fileobj=open('info.log','a')
38. fileobj.write(msg+'\n')
39. fileobj.close();
40. print msg
41. #记录错误日志
42. def e(msg):
43. fileobj=open('error.log','a')
44. fileobj.write(msg+'\n')
45. fileobj.close();
46. print msg
47. if __name__ == '__main__':
48. #print getDownUrl('<a href="http://down.cssmoban.com/cssthemes1/cctp_17_jeans.zip" target="_blank" class="button btn-down" title="免费下载"><i class="icon-down icon-white"></i><i class="icon-white icon-down-transiton"></i>免费下载</a>')
49. html= getUrlContent(URL)
50. i('开始下载：%s' %(URL))
51. while True:
52. lista= getAllUrl(html);
53. #print lista;
54. nextPage=getNextUrl(html)
55. #print nextPage[0]
56. nextUrl=''
57. #i('下一页%s'%(nextPage))
58. if len(nextPage)<=0:
59. e('地址：%s，未找到下一页，程序退出' %(nextPage))
60. break;
61. nextUrl=nextPage[0]
62. nextUrl=URL+'/'+nextUrl[nextUrl.index('href="')+6:nextUrl.index('" target')]
63. #print nextPage
64. for a in lista:
65. downGotoUrl=''
66. try:
67. #print a.decode('utf-8')
68. downGotoUrl=(URL+''+a[a.index('href="')+6:a.index('">')])
69. downGotoUrl=downGotoUrl.replace(URL,'http://www.cssmoban.com')
70. #print downGotoUrl
71. downHtml=getUrlContent(downGotoUrl)
72. #print downHtml
73. downTitleList= getDownTitle(downHtml)
74. downTitle=''
75. if len(downTitleList)>0:
76. downTitle=downTitleList[0]
77. #print downTitle
78. downUrlList= getDownUrl(downHtml)
79. downUrl=''
80. if len(downUrlList)>0:
81. downUrl=downUrlList[0]
82. downUrl= downUrl[downUrl.index('href="')+6:downUrl.index('" target')]
83. #print downUrl
84. i('开始下载：%s,文件名：%s' %(downUrl,downTitle))
85. download(downTitle,downUrl)
86. i('%s下载完成，保存文件名：%s' %(downUrl,downTitle))
87. except Exception,e:
88. e('地址：%s下载失败，失败信息：' %(downGotoUrl))
89. e(str(e))
90. i('-----------------------------------------')
91. i('执行下一页：%s' %(nextUrl))
92. html= getUrlContent(nextUrl)

# -*- coding: utf-8 -*-
import urlparse
import urllib2
import re
import os
import os.path

URL='http://www.cssmoban.com/cssthemes'
全局超时设置

urllib2.socket.setdefaulttimeout(500)
根据url获取内容

def getUrlContent(url):

response = urllib2.urlopen(url)

html = response.read();

return html
获取html中的a标签，且格式是<a target="_blank" href="/showcase/*">的

def getAllUrl(html):

return re.findall('<a[\s]+href="/cssthemes/\d+.shtml">.*?/a>',html)
获取下载文件的标题

def getDownTitle(html):

return re.findall('&lt;h1>(.*?)&lt;/h1>',html)
获取文件下载的url

def getDownUrl(html):

return re.findall('<a.?class="button btn-down".?/a>',html)
获取下一页的url

def getNextUrl(html):

return re.findall('<a.*?下一页</a>',html)
下载文件

def download(title,url):

result = urllib2.urlopen(url).read()

if os.path.exists("template/")==False:

os.makedirs("template/")

newname=("template/"+title.decode('utf-8'))

newname=newname+'.'+url[url.rfind('.')+1:len(url)]

open(newname, "wb").write(result)
记录日志

def i(msg):

fileobj=open('info.log','a')

fileobj.write(msg+'\n')

fileobj.close();

print msg
记录错误日志

def e(msg):

fileobj=open('error.log','a')

fileobj.write(msg+'\n')

fileobj.close();

print msg

if name == 'main':
#print getDownUrl('&lt;a href="http://down.cssmoban.com/cssthemes1/cctp_17_jeans.zip" target="_blank" class="button btn-down" title="免费下载"&gt;&lt;i class="icon-down icon-white"&gt;&lt;/i&gt;&lt;i class="icon-white icon-down-transiton"&gt;&lt;/i&gt;免费下载&lt;/a&gt;')

html= getUrlContent(URL)
i('开始下载：%s' %(URL))
while True:
    lista= getAllUrl(html);
    #print lista;
    nextPage=getNextUrl(html)
    #print nextPage[0]
    nextUrl=''
    #i('下一页%s'%(nextPage))

    if len(nextPage)&lt;=0:
        e('地址：%s，未找到下一页，程序退出' %(nextPage))
        break;

    nextUrl=nextPage[0]
    nextUrl=URL+'/'+nextUrl[nextUrl.index('href="')+6:nextUrl.index('" target')]
    #print nextPage
    for a in lista:
        downGotoUrl=''
        try:
            #print a.decode('utf-8')
            downGotoUrl=(URL+''+a[a.index('href="')+6:a.index('"&gt;')])
            downGotoUrl=downGotoUrl.replace(URL,'http://www.cssmoban.com')
            #print downGotoUrl
            downHtml=getUrlContent(downGotoUrl)
            #print downHtml
            downTitleList= getDownTitle(downHtml)
            downTitle=''
            if len(downTitleList)&gt;0:
                downTitle=downTitleList[0]
            #print downTitle
            downUrlList= getDownUrl(downHtml)
            downUrl=''
            if len(downUrlList)&gt;0:
                downUrl=downUrlList[0]
            downUrl= downUrl[downUrl.index('href="')+6:downUrl.index('" target')]
            #print downUrl
            i('开始下载：%s,文件名：%s' %(downUrl,downTitle))

            download(downTitle,downUrl)
            i('%s下载完成，保存文件名：%s' %(downUrl,downTitle))
        except Exception,e:
            e('地址：%s下载失败，失败信息：' %(downGotoUrl))
            e(str(e))

    i('-----------------------------------------')
    i('执行下一页：%s' %(nextUrl))
    html= getUrlContent(nextUrl)

原文地址：https://blog.csdn.net/wiker_yong/article/details/25844349