Python入门-编写抓取网站图片的爬虫-正则表达式

//生命太短 我用Python！

//Python真是让一直用c++的村里孩子长知识了！

这个仅仅是一个测试，成功抓取了某网站1000多张图片。

下一步要做一个大新闻 大工程

 #config = utf-8
 
 import urllib
 import urllib2
 import re
 import os
 
 global CNT
 CNT = 0
 
 def getHtml(url):
 
 #! /usr/bin/env python
     # -*- coding=utf-8 -*-
     # @Author pythontab.com
     #url="http://pythontab.com"
     req_header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
     'Accept':'text/html;q=0.9,*/*;q=0.8',
     'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
     'Accept-Encoding':'gzip',
     'Connection':'close',
     'Referer':None #注意如果依然不能抓取的话，这里可以设置抓取网站的host
     }
     req_header_2 = {
     'User-Agent':'Mozilla/5.0 (Windows NT 6.1; rv:28.0) Gecko/20100101 Firefox/28.0'
     }
 
     req_timeout = 5
     #status = urllib.urlopen(url).code
     #print status
     #if status != 200:
     #    print 'Http Error %s' % status
     #    return False
 
     req = urllib2.Request(url,None,req_header_2)
     resp = urllib2.urlopen(req,None,req_timeout)
     html = resp.read()
     return html
 
 def getAllUrl(html):
     reg = r'<a href="(.+)" target='
     theurl = re.compile(reg)
     url = re.findall(theurl,html)
     return url
 
 def getNext(html):
     reg = r"<a href='.+pai/(.+).html'"
     nxtre = re.compile(reg)
     nxt = re.findall(nxtre,html)
     return nxt[0]
 
 def getName(html):
     reg = r'<title>(.+)</title>'
     nare = re.compile(reg)
     name = re.findall(nare,html)
     return name[0]
 
 def getImg(name,html):
     global CNT
 
     reg = r'<img src="(.{0,80}\.jpg)" border="0"'
     imgre = re.compile(reg)
     imglist = re.findall(imgre,html)
 
     reg = r'src="(.{0,80}\.jpeg)" border'
     imgre = re.compile(reg)
     imglist.extend(re.findall(imgre,html))
 
     reg = r"<img src='(.{0,80}\.jpg)'"
     imgre = re.compile(reg)
     imglist.extend(re.findall(imgre,html))
 
     reg = r"<img src='(.{0,80}\.jepg)'"
     imgre = re.compile(reg)
     imglist.extend(re.findall(imgre,html))
 
     local = '.\%s-[%sp]' % (name,len(imglist))
     if os.path.exists(unicode(local,'utf-8')):
         return unicode(local,'utf-8')+u'was existed'
 
     os.mkdir(unicode(local,'utf-8'))
 
     x = 0
     for imgurl in imglist:
         print imgurl
         urllib.urlretrieve(imgurl,unicode(local+'\%s.jpg' % x,'utf-8'))
         x+=1
         CNT+=1
 
     return unicode('%s: get %s pthoto(s)' % (name,x),'utf-8')
 
 def getAll(num):
     global CNT
     nxt = 164680
     while num > 0:
 
         url = '---%s.html' % nxt
         print nxt
         html = getHtml(url)
         nxt -= 1
         num -= 1
         if html == False:
             print 'Error'
             continue
 
         print getImg(getName(html),html)
 
     return 'done! %s photos!' % str(CNT)
 
 def getAll_update(index):
     global CNT
     num = CNT
     urls = getAllUrl(getHtml(index))
 
     for url in urls:
         html = getHtml('---'+url)
         print getImg(getName(html),html)
     return 'done! %s photos!' % str(CNT-num)
 
 #print getAll(10)
 #html = getHtml('---')
 #print getNext(html)
 
 x = 3
 while x < 50:
     print getAll_update('---' % x)
     x+=1
 
 #print getAll_update('---')

header 伪装成浏览器

正则表达式 http://www.cnblogs.com/huxi/archive/2010/07/04/1771073.html    //我也是刚刚学

基本都是一路百度写出来的