爬虫利器 Puppeteer实战全自动，监控登陆后页面的数据

2019独角兽企业重金招聘Python工程师标准>>>

const puppeteer = require('puppeteer');

 // 等待3000毫秒
 const sleep = time => new Promise(resolve => {
     setTimeout(resolve, time);
 })

//const url = `https://movie.douban.com/explore#!type=movie&tag=%E7%BB%8F%E5%85%B8&sort=rank&page_limit=20&page_start=0`;
const url = `http://www.xiaoyaoji.cn/dashboard`;
;(async() => {
    console.log('Start visit');
    // 启动一个浏览器
    const brower = await puppeteer.launch({
        args: ['--no-sandbox'],
        //dumpio: false,
        headless: true
    });

    const page = await brower.newPage()   // 开启一个新页面
    // 去豆瓣那个页面
    await page.goto(url, {
        waitUntil: 'networkidle2'  // 网络空闲说明已加载完毕
    });

    //await sleep(5000);
    // 点击搜索框拟人输入 鬼才会想起
    const name = '[email protected]';
    await page.type('.text.pristine.untouched', name, {delay: 0});
    const pwd = '密码';
    await page.type('.text.pristine.untouched', pwd, {delay: 1});
    const inputElement = await page.$('input[type=submit]');
    await inputElement.click();

    await page.waitForNavigation();
   
    console.log(page.url());
    await page.goto(url, {
        waitUntil: 'networkidle2'  // 网络空闲说明已加载完毕
    });

    // 结果
    const BRANDS_INFO_SELECTOR = 'div.mc.home-projects';
    const result = await page.evaluate(sel => {
        // 拿到页面上的jQuery
        const ulList = Array.from($(sel).find('div.col-sm-2'));
        const ctn = ulList.map(v => {
            const title = $(v).html();
                return {
                    title: title,
                };
        });
        return ctn;
    },BRANDS_INFO_SELECTOR);

    brower.close();
    console.log(result);
    console.log('End visit');
    // 关闭浏览器
    
})();

 

转载于:https://my.oschina.net/Alexmyj/blog/1919934