node_puppeteer无界爬虫

环境：node----v14.5.0

vscode----2019

依赖库

　　　（需要自行设置好目录结构，否则会报目录错误）

const puppeteer = require("puppeteer")
const axios = require("axios")
const fs = require("fs")
const url = require("url")
const path = require("path")
    //配置无界浏览器的启动参数
let options = {
        defaultViewport: {
            width: 1536,
            height: 824
        },
        headless: false,
        ignoreDefaultArgs: "--enable-automation",
        // ignoreHTTPSErrors: true, // 忽略证书错误
        // devtools: true, // 开发者控制台
    }
    //删除目录文件（更新）
function delDir(path) {
    let files = [];
    if (fs.existsSync(path)) {
        files = fs.readdirSync(path);
        files.forEach((file, index) => {
            let curPath = path + "/" + file;
            if (fs.statSync(curPath).isDirectory()) {
                delDir(curPath); //递归删除文件夹
                console.log("删除文件夹成功")
            } else {
                fs.unlinkSync(curPath); //删除文件
                console.log("删除文件成功")
            }
        });
        fs.rmdirSync(path);
    }
 
}
//下载作用
async function downMusic(eles) {
    let res = await axios.get(eles.src, { responseType: 'stream' });
    let artistIcon = path.extname(eles.src)
    let ws = fs.createWriteStream("./music/" + eles.alt + "/" + eles.alt + artistIcon, { flags: 'w' });
    res.data.pipe(ws)
    res.data.on("close", function() {
        console.log("下载完成！")
        ws.close()
    })
 
}
 
//主函数
async function openChrome() {
    delDir("./music")
    fs.mkdir("./music", (err) => { if (err) { console.log(err) } else { console.log("创建成功:", "./music") } })
    let browser = await puppeteer.launch(options)
    let page = await browser.newPage()
    page.on('requestfailed', function() {
        console.log("连接超时")
    })
    page.on('console', function(args) {
        console.log(args._text)
    })
    await page.goto("https://music.migu.cn/v3/music/artist?tagId=1&type=A&firstLetter=1&page=1")
    let eles = await page.$$eval(".thumb-img.lazy-image", (elements) => {
        let eles = []
        elements.forEach(function(item, i) {
            let info = {
                alt: item.alt,
                src: item.src
            }
            eles.push(info)
        })
 
        return eles;
 
    })
    console.log(eles)
    eles.forEach((item, i) => {
        fs.mkdir("./music/" + item.alt, function(err) {
            if (err) { console.log(err) } else {
 
                downMusic(item)
 
                fs.mkdir("./music/" + item.alt + "/song", (err) => { if (err) { console.log(err) } else { console.log("创建成功:", "./music/" + item.alt + "/song") } })
            }
        })
    })
    let song_list = []
    let artist_a = await page.$$eval(".thumb-link", (elements) => {
        elements.forEach((element) => {
            console.log(element.href)
 
        })
    })
 
}
openChrome()

效果图如下：