puppeteer: Can't await `networkidle` after `page.click` for JS sites

Ideally there would be some way to wait for the network to complete loading after the button is clicked.

There is:

await Promise.all([
  page.click('.load_more a'),
  page.waitForNavigation({ waitUntil: 'networkidle' })
]);

As a side note, you don’t need cheerio to iterate DOM. Instead of:

var html = await page.content();
var $ = cheerio.load(html);
var projectCount = $('div[data-project]').length;
console.log(projectCount, ' projects');

you can just do:

var length = await page.$$eval('div[data-project]', divs => divs.length);
console.log(length);

@aslushnikov That doesn’t work … the waitForNavigation winds up freezing the page if the network is already idle. Also without putting await in front of each of the actions it raises an error of UnhandledPromiseRejectionWarning

@biznickman so based on your script, here’s what I have to reproduce the issue:

const puppeteer = require('.');

(async() => {
  const browser = await puppeteer.launch({headless: false});
  const page = await browser.newPage();
  await page.emulate({
        'viewport': {
          'width': 1400,
          'height': 1000,
          'isMobile': false
        },
        'userAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
  })

  await page.goto('https://www.kickstarter.com/discover/advanced?sort=newest', {waitUntil: 'networkidle0'});

  await page.click('.load_more a');
  var projectCount = (await page.$$('div[data-project]')).length;
  console.log(projectCount, ' projects');

  // Test after wait, separate helper function.
  await page.waitFor(3000);
  console.log('done sleeping');

  projectCount = (await page.$$('div[data-project]')).length;
  console.log(projectCount, ' projects');
})();

This indeed outputs 12 projects, and then 24 projects. The page.waitForNavigation doesn’t work since the click doesn’t commit any navigation - it just triggers some javascript to update page’s DOM.

I’d suggest to use page.waitForFunction to correctly wait for the click to process:

const puppeteer = require('.');

(async() => {
  const browser = await puppeteer.launch({headless: false});
  const page = await browser.newPage();
  await page.emulate({
        'viewport': {
          'width': 1400,
          'height': 1000,
          'isMobile': false
        },
        'userAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
  })

  await page.goto('https://www.kickstarter.com/discover/advanced?sort=newest', {waitUntil: 'networkidle0'});

  await page.click('.load_more a');
  await page.waitForFunction(() => document.querySelectorAll('div[data-project]').length === 24, {
    polling: 'mutation'
  });

  let projectCount = (await page.$$('div[data-project]')).length;
  console.log(projectCount, ' projects');
})();

Might be they need to handle Promises properly. This might fix the issues. There are many such issues. Even when you use .goto or any such navigation, it fail to return promises properly. Need to compare codes between 0.12 and 0.13 versions

For reference, I was able to solve this using the custom networkidle function in #3083

async function clickExtendedImplementation(selector) {

    waitForNetworkIdle = function (page, timeout, maxInflightRequests = 0) {
        page.on('request', onRequestStarted);
        page.on('requestfinished', onRequestFinished);
        page.on('requestfailed', onRequestFinished);
    
        let inflight = 0;
        let fulfill;
        let promise = new Promise(x => fulfill = x);
        let timeoutId = setTimeout(onTimeoutDone, timeout);
        return promise;
    
        function onTimeoutDone() {
            page.removeListener('request', onRequestStarted);
            page.removeListener('requestfinished', onRequestFinished);
            page.removeListener('requestfailed', onRequestFinished);
            fulfill();
        }
    
        function onRequestStarted() {
            ++inflight;
            if (inflight > maxInflightRequests)
                clearTimeout(timeoutId);
        }
    
        function onRequestFinished() {
            if (inflight === 0)
                return;
            --inflight;
            if (inflight === maxInflightRequests)
                timeoutId = setTimeout(onTimeoutDone, timeout);
        }
    }

    await Promise.all([
        this.click(selector),
        waitForNetworkIdle(page, 500, 0) // equivalent to 'networkidle0'
    ]);
}

Add reference to fuction after init page

(async () => {
    // Init puppeteer
    const browser = await puppeteer.launch();
    const page = await browser.newPage();

    // Add extension method
    page.clickExtended = clickExtendedImplementation;

    // Use
    page.clickExtended("#button")
})

Same problem here. Have you figured it out whats wrong?

@ojotoxy Please upvote #3083.

@aslushnikov ERROR: “networkidle” option is no longer supported. Use “networkidle2” instead

And when I use networkidle2/networkidle0, the behaviour is notas per expectation. It doesn’t await for navigation

I am experiencing the same problem. While waiting for navigation after button click, I am not ABLE TO AWAIT UNTIL navigation completes. This was working in 0.12.0. But now in 0.13, its not working