nodejs爬虫笔记(二)---代理设置

node爬虫代理设置

最近想爬取YouTube上面的视频信息，利用nodejs爬虫笔记(一)的方法,代码和错误如下

var request = require(‘request’);
var cheerio = require(‘cheerio’);****
var url = ‘https://www.youtube.com ‘;
function crawler(url,callback){
    var list = [];
    request(url,function(err,res){
        if(err){
            return callback(err);
        }
        var $ = cheerio(res.body.toString());
        var s = $(’*’).text();
        console.log(‘s=’+s);
    });
    callback(null,list);
};

crawler(url,function(err,list){
    if(err){
        return console.log(err);
    }
    console.log(list);
});

错误
{ [Error: connect ETIMEDOUT 8.7.198.45:443

]
code: ‘ETIMEDOUT’,
errno: ‘ETIMEDOUT’,
syscall: ‘connect’,
address: '8.7.198.45',
port: 443 }

由于国内访问youtube的时候需要翻墙，而在代码里我们需要通过设置代理才能获取页面信息。

1、通过nodejs的http/https模块

具体使用可以参考http://nodejs.cn/api/http.html#http_http_request_options_callback，我使用的lantern作为翻墙工具。

var http = require('http'); // 使用http模块，也可以换成https模块
var opt = {
    host: '127.0.0.1', // 这里是代理服务器的地址
    port: '57939', // 这里是代理服务器的端口号
    method: 'GET', // 这里是发送的方法
    path: 'https://www.youtube.com', // 这里是访问的路径
    headers: {
        //请求头(可以利用Google浏览器打开youtube首页，点击network 查看请求头，把相关信息复制过来即可)

　　　　　　'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
　　　　　　'Accept-Encoding':'gzip, deflate, sdch, br',
　　　　　　'Accept-Language':'zh-CN,zh;q=0.8,zh-TW;q=0.6',
　　　　　　'Cache-Control':'max-age=0',
　　　　　　'Cookie':'_ga=GA1.2.1653214693.1476773935; VISITOR_INFO1_LIVE=T3BczuPUIQo; SID=5QR6XEldVgveXzFtqjIcD480cHE18gBRd3xPo398vndcc5JNxOAZ-TgVp5jQx　　       3CR-ePvgA.; HSID=APr2I8UwM-A-Lypbd; SSID=Ap4H3Td1nrV__-9tN; APISID=8bHyFV90pNBU5Z9p/A2DlJa2MyJLL4-RKP; SAPISID=4tZf4GDX7Dt5bNAt/A5vhaZe_DLzn            -ECul; CONSENT=YES+CN.zh-CN+20160904-14-0; YSC=XVHk_pArWhE; PREF=cvdm=grid&f1=50000000&f6=1&f5=30&al=zh-CN&gl=HK',
　　　　　　'Upgrade-insecure-requests':'1',
　　　　　　'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36',
　　　　　　'X-Chrome-Uma-Enabled':'1',
　　　　　　'X-Client-Data':'CJa2yQEIorbJAQjBtskBCKmdygE=',
　　　　　　'Connection': 'keep-alive'

}

};

var body = '';
var req = http.request(opt, function(res) {
    console.log("Got response: " + res.statusCode);
    res.on('data', function(d) {
        body += d;
    }).on('end', function() {
        //console.log(res);
        console.info('============');

        console.log(body)
    });

}).on('error', function(e) {
    console.log("Got error: " + e.message);
})

req.end();

2、使用SuperAgent以及superagent-proxy模块

为了使用方便以及加快开发的速度，我们就会引入模块。SuperAgent也是一个可以封装好的http模块，功能和Request模块也差不多。如果要使用代理模块的还，还需要额外的拓展模块SuperAgent-Proxy。

SuperAgent官网地址
SuperAgent-proxy官网地址

var request =require('superagent');
require('superagent-proxy')(request);

var fs = require('fs');

var proxy = 'http://127.0.0.1:57939';

var header = {
    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Encoding':'gzip, deflate, sdch, br',
    'Accept-Language':'zh-CN,zh;q=0.8,zh-TW;q=0.6',
    'Cache-Control':'max-age=0',
    'Cookie':'_ga=GA1.2.1653214693.1476773935; VISITOR_INFO1_LIVE=T3BczuPUIQo; SID=5QR6XEldVgveXzFtqjIcD480cHE18gBRd3xPo398vndcc5JNxOAZ-TgVp5jQx3CR-ePvgA.; HSID=APr2I8UwM-A-Lypbd; SSID=Ap4H3Td1nrV__-9tN; APISID=8bHyFV90pNBU5Z9p/A2DlJa2MyJLL4-RKP; SAPISID=4tZf4GDX7Dt5bNAt/A5vhaZe_DLzn-ECul; CONSENT=YES+CN.zh-CN+20160904-14-0; YSC=XVHk_pArWhE; PREF=cvdm=grid&f1=50000000&f6=1&f5=30&al=zh-CN&gl=HK',
    'Upgrade-insecure-requests':'1',
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36',
    'X-Chrome-Uma-Enabled':'1',
    'X-Client-Data':'CJa2yQEIorbJAQjBtskBCKmdygE=',
    'Connection': 'keep-alive'
};

request
    .get('https://www.youtube.com')
    .set('header',header)
    .proxy(proxy)
    .end(onresponse);

function onresponse(err,res){
    res.setEncoding('utf-8'); //防止中文乱码
    if(err){
        console.log(err);
    }else{
        console.log('status:'+res.status);
        //console.log(res.headers);
        console.log(res.text);
        //将res.text写入json文件
        fs.writeFile(__dirname+'/data/home.json',JSON.stringify({
            status: 0,
            data: res.text
        }),function(err){
            if(err){
                return console.log(err);
            }
            console.log('完成');
        });
    }
}

运行后会出现以下结果：

status:200
-sessionlink="itct=CJUBEJQ1GAEiEwjxtYyGsv7UAhUFKioKHf57D1Yojh4yCmctaGlnaC10cnZaD0ZFd2hhdF90b193YXRjaA"  title="欅って、書けない 2017年7月9日 #87" aria-describedby="description-id-25113" dir="ltr">欅って、書けない 2017年7月9日 #87</a><span class="accessible-description" id="description-id-25113"> - 長さ: 27:16。</span></h3><div class="yt-lockup-byline yt-ui-ellipsis yt-ui-ellipsis-2"><a href="/channel/UCmIRpoe4yk09Lfx16doL3SA" class="g-hovercard yt-uix-sessionlink       spf-link " data-sessionlink="itct=CJUBEJQ1GAEiEwjxtYyGsv7UAhUFKioKHf57D1Yojh4" data-ytid="UCmIRpoe4yk09Lfx16doL3SA" >5時に夢中</a></div><div class="yt-lockup-meta "><ul class="yt-lockup-meta-info"><li>視聴回数 63,297 回</li><li>17 時間前</li></ul></div></div></div></div></li><li class="yt-shelf-grid-item yt-uix-shelfslider-item"><div class="yt-lockup yt-lockup-grid yt-lockup-video vve-check clearfix" data-context-item-id="6OwlCt4aKfM" data-visibility-tracking="CJQBEJQ1GAIiEwjxtYyGsv7UAhUFKioKHf57D1Yojh5A89Po8K2hifboAQ=="><div class="yt-lockup-dismissable"><div class="yt-lockup-thumbnail contains-addto"><a aria-hidden="true"  href="/watch?v=6OwlCt4aKfM" class=" yt-uix-sessionlink      spf-link " data-sessionlink="itct=CJQBEJQ1GAIiEwjxtYyGsv7UAhUFKioKHf57D1Yojh4yCmctaGlnaC10cnZaD0ZFd2hhdF90b193YXRjaA" ><div class="yt-thumb video-thumb"><span class="yt-thumb-simple">
  <img width="196" alt="" data-ytimg="1" src="https://i.ytimg.com/vi/6OwlCt4aKfM/hqdefault.jpg?sqp=-oaymwEWCMQBEG5IWvKriqkDCQgBFQAAiEIYAQ==&amp;rs=AOn4CLAwVQhAiFdPT0nRTMFB8rX989yXuA" height="110" onload=";window.__ytRIL &amp;&amp; __ytRIL(this)" >
<span class="video-time" aria-hidden="true">6:44</span></span></div></a>  <span class="thumb-menu dark-overflow-action-menu video-actions">
    <button aria-haspopup="true" aria-expanded="false" onclick=";return false;" class="yt-uix-button-reverse flip addto-watch-queue-menu spf-nolink hide-until-delayloaded yt-uix-button yt-uix-button-dark-overflow-action-menu yt-uix-button-size-default yt-uix-button-has-icon no-icon-markup yt-uix-button-empty" type="button" ><span class="yt-uix-button-arrow yt-sprite"></span><ul class="watch-queue-thumb-menu yt-uix-button-menu yt-uix-button-menu-dark-overflow-action-menu hid"><li role="menuitem" class="overflow-menu-choice addto-watch-queue-menu-choice addto-watch-queue-play-next yt-uix-button-menu-item" data-action="play-next" onclick=";return false;"  data-video-ids="6OwlCt4aKfM"><span class="addto-watch-queue-menu-text">次に再生</span></li><li role="menuitem" class="overflow-menu-choice addto-watch-queue-menu-choice addto-watch-queue-play-now yt-uix-button-menu-item" data-action="play-now" onclick=";return false;"  data-video-ids="6OwlCt4aKfM"><span class="addto-watch-queue-menu-text">今すぐ再生</span></li></ul></button>
  </span>

  <button class="yt-uix-button yt-uix-button-size-small yt-uix-button-default yt-uix-button-empty yt-uix-button-has-icon no-icon-markup addto-button video-actions spf-nolink hide-until-delayloaded addto-watch-later-button-sign-in yt-uix-tooltip" type="button" onclick=";return false;" title="後で見る" role="button" data-button-menu-id="shared-addto-watch-later-login" data-video-ids="6OwlCt4aKfM"><span class="yt-uix-button-arrow yt-sprite"></span></button>
  <button class="yt-uix-button yt-uix-button-size-small yt-uix-button-default yt-uix-button-empty yt-uix-button-has-icon no-icon-markup addto-button addto-queue-button video-actions spf-nolink hide-until-delayloaded addto-tv-queue-button yt-uix-tooltip" type="button" onclick=";return false;" title="キュー" data-style="tv-queue" data-video-ids="6OwlCt4aKfM"></button>
</div><div class="yt-lockup-content"><h3 class="yt-lockup-title "><a href="/watch?v=6OwlCt4aKfM" class=" yt-ui-ellipsis yt-ui-ellipsis-2 yt-uix-sessionlink      spf-link " data-sessionlink="itct=CJQBEJQ1GAIiEwjxtYyGsv7UAhUFKioKHf57D1Yojh4yCmctaGlnaC10cnZaD0ZFd2hhdF90b193YXRjaA"  title="シャンプー中に血のり垂らしたらとんでもないパニックになったwww" aria-describedby="description-id-467161" dir="ltr">シャンプー中に血のり垂らしたらとんでもないパニックになったwww</a><span class="accessible-description" id="description-id-467161"> - 長さ: 6:44。</span></h3><div class="yt-lockup-byline yt-ui-ellipsis yt-ui-ellipsis-2"><a href="/channel/UCpOjLndjOqMoffA-fr8cbKA" class="g-hovercard yt-uix-sessionlink       spf-link " data-sessionlink="itct=CJQBEJQ1GAIiEwjxtYyGsv7UAhUFKioKHf57D1Yojh4" data-ytid="UCpOjLndjOqMoffA-fr8cbKA" >水溜りボンド</a>&nbsp;<span title="確認済み" class="yt-uix-tooltip yt-channel-title-icon-verified yt-sprite"></span></div><div class="yt-lockup-meta "><ul class="yt-lockup-meta-info"><li>視聴回数 1,969,790 回</li><li>22 時間前</li></ul></div></div></div></div></li><li class="yt-shelf-grid-item yt-uix-shelfslider-item"><div class="yt-lockup yt-lockup-grid yt-lockup-video vve-check clearfix" data-context-item-id="t-saIu9AD58" data-visibility-tracking="CJMBEJQ1GAMiEwjxtYyGsv7UAhUFKioKHf57D1Yojh5An5-A-q7ExvW3AQ=="><div class="yt-lockup-dismissable"><div class="yt-lockup-thumbnail contains-addto"><a aria-hidden="true"  href="/watch?v=t-saIu9AD58" class=" yt-uix-sessionlink      spf-link " data-sessionlink="itct=CJMBEJQ1GAMiEwjxtYyGsv7UAhUFKioKHf57D1Yojh4yCmctaGlnaC10cnZaD0ZFd2hhdF90b193YXRjaA" ><div class="yt-thumb video-thumb"><span class="yt-thumb-simple">
  <img width="196" alt="" data-ytimg="1" src="https://i.ytimg.com/vi/t-saIu9AD58/hqdefault.jpg?sqp=-oaymwEWCMQBEG5IWvKriqkDCQgBFQAAiEIYAQ==&amp;rs=AOn4CLCICoGcnfWiQ4rKmXIbmq1fspGKiA" height="110" onload=";window.__ytRIL &amp;&amp; __ytRIL(this)" >
<span class="video-time" aria-hidden="true">8:40</span></span></div></a>  <span class="thumb-menu dark-overflow-action-menu video-actions">
    <button aria-haspopup="true" aria-expanded="false" onclick=";return false;" class="yt-uix-button-reverse flip addto-watch-queue-menu spf-nolink hide-until-delayloaded yt-uix-button yt-uix-button-dark-overflow-action-menu yt-uix-button-size-default yt-uix-button-has-icon no-icon-markup yt-uix-button-empty" type="button" ><span class="yt-uix-button-arrow yt-sprite"></span><ul class="watch-queue-thumb-menu yt-uix-button-menu yt-uix-button-menu-dark-overflow-action-menu hid"><li role="menuitem" class="overflow-menu-choice addto-watch-queue-menu-choice addto-watch-queue-play-next yt-uix-button-menu-item" data-action="play-next" onclick=";return false;"  data-video-ids="t-saIu9AD58"><span class="addto-watch-queue-menu-text">次に再生</span></li><li role="menuitem" class="overflow-menu-choice addto-watch-queue-menu-choice addto-watch-queue-play-now yt-uix-button-menu-item" data-action="play-now" onclick=";return false;"  data-video-ids="t-saIu9AD58"><span class="addto-watch-queue-menu-text">今すぐ再生</span></li></ul></button>
  </span>
.......太长了就省略了

此时我们已经可以成功连接上YouTube,接下来就可以利用cheerio模块对其进行解析啦。

请求头的获取：

代码中的请求头可以利用浏览器，翻墙后打开YouTube首页，点击检查，然后点击network，刷新后我们会发现header里面包含的请求信息。