爬行百度标题&URL案例

思路：

　　先将需要获取的匹配出，然后可以用"永真"（即while True:）来遍历使得URL可以一直自增变化（百度点击下一页URL的pn参数就增加10）每增加10就爬行一遍URL然后提取一次数据。

#-*-coding:UTF-8-*-

import sys,re,requests,graphics,Tkinter
import easygui as gui

string = raw_input("string is :")
pn = 0
while True:
    url = "http://www.baidu.com/s?wd=%s&pn=%d" % (string, pn)
    pn += 10
    html = requests.get(url).text
    # html = """
    # <div class="c-tools" id="tools_2269957611132062659_2" data-tools='{"title":"织梦CMS 官方网站 - 内容管理系统 - 上海卓卓网络科技有限公司","url":"http://www.baidu.com/link?url=gXtstOFbadX8Lia_Fwwl_AS8VUgXEfqcHe4bpP6Paj-BIGvrYgaUwI4BXvB2M4vg"}'><a class="c-tip-icon"><i class="c-icon c-icon-triangle-down-g"></i></a></div>
    # """
    res = "<div .*? data-tools=(.*?)>.*?</div>"
    con = re.findall(res, html)
    for i in con:
        d = eval(i.strip("'"))#将正则匹配到的json格式的数据转换为字典，eval即为转换。
        print "title:" + d[u'title'] + "  " + d['url']

    num = raw_input(u"e or q:")
    if num == "q":
        exit()

后期又修改了一下．

 #!/usr/bin/env python
 #encoding:utf-8
 #by i3ekr

 import sys,re,requests,time,json
 print """

                        #G
                        #K
                       .Et
                       :#
                     : ##
                     ##Dj K
                    .####G###
                    E;#####f;
                     ########
                     #######.
                     .i#L#,t
                     DEDECMS               

 """
 string = raw_input("string is :")
 pn = 0
 nn = 0
 r = requests.session()
 head = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko'}
 while True:
     url = "http://www.baidu.com/s?wd=%s&pn=%d" % (string, pn)
     html = r.get(url, headers=head).text
     res = "<div .*? data-tools=(.*?)>.*?</div>"
     con = re.findall(res, html)
     pn += 10
     nn += 1
     try:
       for i in con:
           a = eval(eval(i))
           b = r.get(a.get("url"), headers=head)
           print "[%s] %s"%(nn,b.url)
     except Exception as e:
       pass