puppeteer-extra: Doesn't play nice with Webpack

Work-around is to import and apply the plugins manually:

import puppeteerVanilla from 'puppeteer';
import { addExtra } from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
import AcceptLanguagePlugin from 'puppeteer-extra-plugin-stealth/evasions/accept-language';
import ChromeRuntimePlugin from 'puppeteer-extra-plugin-stealth/evasions/chrome.runtime';
import ConsoleDebugPlugin from 'puppeteer-extra-plugin-stealth/evasions/console.debug';
import IFrameContentWindowPlugin from 'puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow';
import MediaCodecsPlugin from 'puppeteer-extra-plugin-stealth/evasions/media.codecs';
import NavigatorLanguagesPlugin from 'puppeteer-extra-plugin-stealth/evasions/navigator.languages';
import NavigatorPermissionsPlugin from 'puppeteer-extra-plugin-stealth/evasions/navigator.permissions';
import NavigatorPlugins from 'puppeteer-extra-plugin-stealth/evasions/navigator.plugins';
import WebdriverPlugin from 'puppeteer-extra-plugin-stealth/evasions/navigator.webdriver';
import UserAgentPlugin from 'puppeteer-extra-plugin-stealth/evasions/user-agent';
import WebglVendorPlugin from 'puppeteer-extra-plugin-stealth/evasions/webgl.vendor';
import WindowOuterDimensionsPlugin from 'puppeteer-extra-plugin-stealth/evasions/window.outerdimensions';

async () => {
  const puppeteer = addExtra(puppeteerVanilla);

  const plugins = [
    StealthPlugin(),
    AcceptLanguagePlugin(),
    ChromeRuntimePlugin(),
    ConsoleDebugPlugin(),
    IFrameContentWindowPlugin(),
    MediaCodecsPlugin(),
    NavigatorLanguagesPlugin(),
    NavigatorPermissionsPlugin(),
    NavigatorPlugins(),
    WebdriverPlugin(),
    UserAgentPlugin(),
    WebglVendorPlugin(),
    WindowOuterDimensionsPlugin(),
  ];

  const browser = await puppeteer.launch();

  for (const plugin of plugins) {
    await plugin.onBrowser(browser);
  }

  const [ page ] = await browser.pages();

  for (const plugin of plugins) {
    await plugin.onPageCreated(page);
  }

  // ...
};

I got this working locally but when I used webpack to bundle it and send it over to aws lambda, this line

StealthPlugin();

results to the error below. Adding “kind-of”: “^6.0.2” to the project’s package.json does not resolve the problem.

Error: Cannot find module ‘kind-of’ at a (/var/task/index.js:145:1835) at Function.o [as typeOf] (/var/task/index.js:145:1440) at i (/var/task/index.js:145:854) at e.exports (/var/task/index.js:145:371) at new n (/var/task/index.js:139:191) at new i (/var/task/index.js:133:12241) at e.exports (/var/task/index.js:133:12910) at Object.startBrowser (/var/task/index.js:127:99655) at Runtime.t.handler (/var/task/index.js:127:86330) at processTicksAndRejections (internal/process/task_queues.js:93:5) { code: ‘MODULE_NOT_FOUND’ }

I should note that while working on the playwright-supporting rewrite I realized that for improved type-safety it would make sense to ditch the dynamic custom imports, so I’m most likely doing that. This would mean that this webpack issue should evaporate in the next major release. 😄

@berstend I am getting the error:

Error: Cannot find module 'kind-of'
    at webpackEmptyContext (D:\node_projects\vueapp\dist_electron\index.js:34338:10)
    at Function.getter [as typeOf] (D:\node_projects\vueapp\dist_electron\index.js:91020:29)
    at cloneDeep (D:\node_projects\vueapp\dist_electron\index.js:34370:17)
    at mergeDeep (D:\node_projects\vueapp\dist_electron\index.js:100090:16)
    at new PuppeteerExtraPlugin (D:\node_projects\vueapp\dist_electron\index.js:159981:22)
    at new StealthPlugin (D:\node_projects\vueapp\dist_electron\index.js:159809:5)
    at defaultExport (D:\node_projects\vueapp\dist_electron\index.js:159905:31)
    at Module../src/main/gen.js (D:\node_projects\vueapp\dist_electron\index.js:221678:15)
    at __webpack_require__ (D:\node_projects\vueapp\dist_electron\index.js:20:30)
    at Module../src/main/main.js (D:\node_projects\vueapp\dist_electron\index.js:221764:62)

This is my vue.config.js

module.exports = {
    configureWebpack: {
        // Configuration applied to all builds
        target: "electron-renderer",
        devtool: 'source-map',
        module:{
            rules:[
                {
                    test: /\.js$/,
                    use: "unlazy-loader"
                },
                {
                    // regex for the files that are problematic
                    test: /node_modules\/puppeteer-extra\/dist\/index\.esm\.js/,
                    loader: 'string-replace-loader',
                    options: {
                        // match a require function call where the argument isn't a string
                        // also capture the first character of the args so we can ignore it later
                        search: 'require[(]([^\'"])',
                        // replace the 'require(' with a '__non_webpack_require__(', meaning it will require the files at runtime
                        // $1 grabs the first capture group from the regex, the one character we matched and don't want to lose
                        replace: '__non_webpack_require__($1',
                        flags: 'g'
                    }
                }
            ]
        }
    },
    pluginOptions: {
        electronBuilder: {
            builderOptions: {
                // options placed here will be merged with default configuration and passed to electron-builder
            }
        }
    }
}

babel.config.js:

module.exports = {
  presets: [
    '@vue/cli-plugin-babel/preset'
  ]
}

This only happens if i use StealthPlugin() code

I’m getting a similar problem to this with serverless-bundle. Any ideas how to work around this with serverless-bundle?

we still need to do dynamic require() under the hood

Why is this?

It’s always good to question assumptions, I’m a big believer in that 😃

Let’s take the stealth plugin as an example: it comes with a set of “evasions” (which are just regular plugins) and acts as an “umbrella” plugin, so the user doesn’t need to add the specific evasions one-by-one.

One feature is that the user can add or remove evasions on that list, before puppeteer-extra will require these files (and thereby code mods).

I’m not aware of a way to accomplish that without dynamic require(). 😃

The stealth plugin could require all evasions statically. This would cover most use-cases.

Users who need something more custom could manually import the ones that they need. This wouldn’t be much code and the full import list could be copied from the stealth plugin then edited.

Got it 😃 Anyway, puppeteer-extra should still be able to work with Webpack. It might be that the internal dependency system is just not aware of the bundler already taking care of the dependencies and a flag to disable the internal dependency resolution is sufficient.

I use (electron-forge+webpack+typescript+puppeteer-extra) got this error:

App threw an error during load
Error: Cannot find module 'is-plain-object'
    at webpackEmptyContext (/Path/To/Your/Project/electron-webpack/.webpack/main/index.js:17478:10)

I finally got it work, here is what I did: add “externals” in webpack.main.config.ts

export const mainConfig: Configuration = {
   .....
   .....
  externals: {
    "puppeteer": "puppeteer",
    "puppeteer-extra": "puppeteer-extra",
    "puppeteer-extra-plugin-stealth": "puppeteer-extra-plugin-stealth",
  }
};

I also use electron-forge+webpack+typescript+puppeteer-extra. This worked, thank you!

I use (electron-forge+webpack+typescript+puppeteer-extra) got this error:

App threw an error during load
Error: Cannot find module 'is-plain-object'
    at webpackEmptyContext (/Path/To/Your/Project/electron-webpack/.webpack/main/index.js:17478:10)

I finally got it work, here is what I did: add “externals” in webpack.main.config.ts

export const mainConfig: Configuration = {
   .....
   .....
  externals: {
    "puppeteer": "puppeteer",
    "puppeteer-extra": "puppeteer-extra",
    "puppeteer-extra-plugin-stealth": "puppeteer-extra-plugin-stealth",
  }
};

Hi guys, I made it working. Want to share what I got. Hope it works to y’all too.

Inside vue.config.js:

var webpack = require('webpack')

module.exports = {
  transpileDependencies: [
    'vuetify'
  ],
  pluginOptions: {
    electronBuilder: {
      externals: [
        'puppeteer',
        'puppeteer-extra',
        'puppeteer-extra-plugin-stealth'
      ],
      nodeModulesPath: ['../../node_modules', './node_modules'],
      nodeIntegration: true,
      builderOptions: {
        asarUnpack: 'node_modules/puppeteer/.local-chromium/**/*',
        copyright: 'Copyright © 2021',
        nsis: {
          oneClick: false,
          allowToChangeInstallationDirectory: true
        },
        publish: [
          {
            provider: '<your provider>',
            repo: '<your repo here>',
            owner: '<owner>',
            token: '<token here>',
            releaseType: 'draft',
            publishAutoUpdate: true,
            private: true
          }
        ]
      }
    }
  },
  configureWebpack: {
    externals: {
      puppeteer: "require('puppeteer')",
      'puppeteer-extra': "require('puppeteer-extra')",
      'puppeteer-extra-plugin-stealth': "require('puppeteer-extra-plugin-stealth')"
    }
  }
}

Basically you have to expose these packages and include those inside devDependencies and dependencies to make it working.

just upgrade clone-deep version

@Seblor this is out of the scope of this webpack issue but the iframe.contentWindow evasion is known to cause issues, more context and workaround here: https://github.com/berstend/puppeteer-extra/issues/137#issuecomment-685680787

Which is weird since I saw in an other issue that this flag was supposed to be included in the puppeteer-extra stealth plugin.

It is, as you can see in the code of the navigator.webdriver evasion. Make sure you use the latest versions of everything.

@berstend None of the solutions above worked for me. Do you have any update on the rework ?

Did you try https://github.com/berstend/puppeteer-extra/issues/93#issuecomment-563159177 ? It is highly likely to work because it is pure ES6 with static imports.

@jstormail2 thanks for sharing your workaround 👍

I must admit listing the plugins manually isn’t the most prettiest thing in the world but great if it fixes the webpack issue. 😃

cc @Nisthar

I got this working locally but when I used webpack to bundle it and send it over to aws lambda, this line

StealthPlugin();

results to the error below. Adding “kind-of”: “^6.0.2” to the project’s package.json does not resolve the problem.

Error: Cannot find module ‘kind-of’ at a (/var/task/index.js:145:1835) at Function.o [as typeOf] (/var/task/index.js:145:1440) at i (/var/task/index.js:145:854) at e.exports (/var/task/index.js:145:371) at new n (/var/task/index.js:139:191) at new i (/var/task/index.js:133:12241) at e.exports (/var/task/index.js:133:12910) at Object.startBrowser (/var/task/index.js:127:99655) at Runtime.t.handler (/var/task/index.js:127:86330) at processTicksAndRejections (internal/process/task_queues.js:93:5) { code: ‘MODULE_NOT_FOUND’ }

I had the same issue. unlazy-loader solved this for me…

What is the purpose of the internal dependency management?

Could a simpler design work like this?

import puppeteerExtra from 'puppeteer-extra';
import MyPlugin from 'my-plugin';

const puppeteer = pupeteerExtra({
  plugins: [
    MyPlugin(),
  ],
});

async () => {
  const browser = await puppeteer.launch();

  // etc...
};

I use Webpack with Node because it’s a simpler way to use Babel, bundle node_modules and minify code.

webpack.config.js:

const path = require('path');
const TerserPlugin = require('terser-webpack-plugin');

const { env } = process;

const isProduction = env['NODE_ENV'] === 'production';

const mode = isProduction ?
  'production' :
  'development';

console.log({ mode });

module.exports = {
  entry: [
    '@babel/polyfill',
    './src/index.js',
  ],
  target: 'node',
  devtool: isProduction ? false : 'source-map',
  mode,
  output: {
    path: path.join(__dirname, 'build'),
    filename: 'index.js'
  },
  module: {
    rules: [
      {
        test: /\.m?js$/,
        exclude: /(node_modules|bower_components)/,
        use: {
          loader: 'babel-loader',
          options: {
            babelrc: true,
          },
        },
      },
      {
        test: /\.js$/,
        loader: 'unlazy-loader'
      }
    ],
  },
  optimization: {
    minimize: isProduction,
    minimizer: [
       new TerserPlugin(),
    ],
  },
  resolve: {
    alias: {
      'pg-native': path.resolve(__dirname, 'aliases/pg-native'),
    },
  },
};