python3爬虫.4.下载煎蛋网妹子图

开始我学习爬虫的目标 ----> 煎蛋网

通过设置User-Agent获取网页，发现本该是图片链接的地方被一个js函数代替了

于是全局搜索到该函数

function jandan_load_img(b){
    var d = $(b);
    var f = d.next("span.img-hash");
    var e = f.text();
    f.remove();
    var c = jdPwA5ybKhQWGy2rZybAF2StIBxrQ6NvYC(e, "pAqWmGD1GsmY5kVokg1a2eyQ3Shj1Usq");
    var a = $('<a href = "'+c.replace(/(\/\/\w+\.sinaimg\.cn\/)(\w+)(\/.+\.(gif|jpg|jpeg))/, "$1large$3")+'" target = "_blank" class = "view_img_link">[查看原图]</a>');
    d.before(a);
    d.before("<br>");
    d.removeAttr("onload");
    d.attr("src", location.protocol+c.replace(/(\/\/\w+\.sinaimg\.cn\/)(\w+)(\/.+\.gif)/, "$1thumb180$3"));
    if(/\.gif$/.test(c)){
        d.attr("org_src", location.protocol+c);
        b.onload = function(){
            add_img_loading_mask(this, load_sina_gif)
        }
    }
}

该函数提取span.img-hsah传到另一个函数中，继续查找，有两个定义，于是选择靠后的那个

 var jdTzcXZnL0V2WZZ8eq9786xeOdkyoBXlDR=function(m,r,d){
     var e="DECODE";
     var r=r?r:"";
     var d=d?d:0;
     var q=4;
     r=md5(r);
     var o=md5(r.substr(0,16));
     var n=md5(r.substr(16,16));
     if(q){if(e=="DECODE"){var l=m.substr(0,q)}}
     else{var l=""}
     var c=o+md5(o+l);
     var k;
     if(e=="DECODE"){m=m.substr(q);
     k=base64_decode(m)}
     var h=new Array(256);
     for(var g=0;g<;g++){h[g]=g}
     var b=new Array();
     for(var g=0;g<256;g++){b[g]=c.charCodeAt(g%c.length)}
     for(var f=g=0;g<256;g++){f=(f+h[g]+b[g])%256;
     tmp=h[g];
     h[g]=h[f];
     h[f]=tmp}
     var t="";
     k=k.split("");
     for(var p=f=g=0;
     g<k.length;
     g++){p=(p+1)%256;
     f=(f+h[p])%256;
     tmp=h[p];
     h[p]=h[f];
     h[f]=tmp;
     t+=chr(ord(k[g])^(h[(h[p]+h[f])%256]))}
     if(e=="DECODE"){if((t.substr(0,10)==0||t.substr(0,10)-time()>0)&&t.substr(10,16)==md5(t.substr(26)+n).substr(0,16)){t=t.substr(26)}
     else{t=""}
     }
     return t
 };

参考文章:　　http://www.tendcode.com/article/jiandan-meizi-spider/

其中有对js的函数的改写

最后代码如下

 # -*- coding = UTF-8 -*-
 '''
 目标：煎蛋网妹子图
 2018/4/22
 环境：pyhton3
 
 '''
 
 import urllib.request           #使用url处理包，urllib.request模块是用来打开和读取URLs的
 import re                       #使用正则表达式
 import hashlib                  #
 import base64                   #
 from bs4 import BeautifulSoup   #
 import time                     #time
 import logging                  #log
 import sys                      #
 
 '''
 下载单张图片到制定的文件夹下
 '''
 def load_img(imgurl, file):
     name = imgurl.split('/')[-1]
     item = urllib.request.urlretrieve('http:'+imgurl,\
         #'C:\\Users\\74172\\Pictures\\jandan2\\%s'%(name))
         file+'\\%s'%(name))
     print(name+' is loaded')
 
 '''
 md5加密
 '''
 def _md5(value):
     m = hashlib.md5()
     m.update(value.encode('utf-8'))
     return m.hexdigest()
 
 '''
 bash64解码
 注意 原字符串长度报错问题
 '''
 def _base64_decode(data):
     missing_padding = 4 - len(data) % 4
     if missing_padding:
         data += '=' * missing_padding
     return base64.b64decode(data)
 
 '''
 解密获取图片链接
 '''
 def get_imgurl(m, r='', d=0):
     e = "DECODE"
     q = 4
     r = _md5(r)
     o = _md5(r[0:0 + 16])
     n = _md5(r[16:16 + 16])
     l = m[0:q]
     c = o + _md5(o + l)
     m = m[q:]
     k = _base64_decode(m)
     h = list(range(256))
     b = [ord(c[g % len(c)]) for g in range(256)]
 
     f = 0
     for g in range(0, 256):
         f = (f + h[g] + b[g]) % 256
         tmp = h[g]
         h[g] = h[f]
         h[f] = tmp
 
     t = ""
     p, f = 0, 0
     for g in range(0, len(k)):
         p = (p + 1) % 256
         f = (f + h[p]) % 256
         tmp = h[p]
         h[p] = h[f]
         h[f] = tmp
         t += chr(k[g] ^ (h[(h[p] + h[f]) % 256]))
     t = t[26:]
     return t
 
 '''
 获取关键字符串
 '''
 def get_r(js_url):
     js_respon = urllib.request.urlopen(js_url)
     js = js_respon.read().decode('utf-8')
     _r = re.findall('c=[\w\d]+\(e,"(.*?)"\)', js)
     return _r
 
 '''
 获取一个页面的所有图片的链接
 '''
 def get_urls(url,pages,file):
     page = 0
     imagNum = 0
     headers = {
         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
         'Host': 'jandan.net'
     }
     #########################################
     while page < pages:
         req = urllib.request.Request(url, headers=headers)
         respon = urllib.request.urlopen(req)
         html = respon.read().decode('utf-8')
         ##########################################
         js_url = 'http:' + re.findall('<script src="(//cdn.jandan.net/static/min/[\w\d]+\.\d+\.js)"></script>', html)[-1]
         _r = get_r(js_url)[0]
         soup = BeautifulSoup(html, 'lxml')
         tags = soup.select('.img-hash')
         for tag in tags:
             img_hash = tag.text
             img_url = get_imgurl(img_hash,_r)
             print(imagNum,'------>',img_url)
             imagNum = imagNum+1
             load_img(img_url,file)
         ############################################
         nextUrl = re.findall(r'Older Comments" href=".+?.#comments"',html)[0]
         print('page#',90-page,'---->done!')
         url = 'http:' + nextUrl[22:-1]
         page += 1
         time.sleep(10)
     print('done all!')
     print('located---->',file)
 
 if __name__ == '__main__':
     url = 'http://jandan.net/ooxx/'
     pages = 1　　　　　　
     file = 'C:\\jandan_meizi'
     get_urls(url,pages,file)

//************2018.05.03*******************************

刚下班回来爬图，发现中间有个报错，

是get_url一次返回的多张图片链接，于是的改段

     for tag in tags:
             img_hash = tag.text
             img_urls = get_imgurl(img_hash,_r)
             img_urls = re.findall(r'//wx.+?.jpg',img_urls)
             for img_url in img_urls:
                 print(imagNum,'------>',img_url)
                 imagNum = imagNum+1
                 load_img(img_url,file)    

//************2018.5.23***********************************

又报错了，暂时没有解决。