vite: Importing a file from a web worker that imports said web worker causes Vite to build forever

Describe the bug

I have my project set up where I have an index file that exports everything in the folder. This is so I can solve a circular dependency.

When I created a web worker, I had it import from the index file. One of these files in the index directly imported the web worker using ?worker. This worked fine in development. When I went to build, however, Vite seemed to get stuck in an infinite loop and only stopped when it ran out of memory.

Reproduction

~~https://github.com/hf02/vite-infinite-build~~ Much simpler reproduction, thanks to @zxch3n: https://stackblitz.com/edit/vitejs-vite-u59lcw?file=worker.js

System Info

System:
    OS: Windows 10 10.0.22000
    CPU: (16) x64 AMD Ryzen 7 3700X 8-Core Processor
    Memory: 9.38 GB / 15.95 GB
  Binaries:
    Node: 16.14.0 - ~\AppData\Local\nvs\default\node.EXE
    npm: 8.3.1 - ~\AppData\Local\nvs\default\npm.CMD
  Browsers:
    Edge: Spartan (44.22000.120.0), Chromium (98.0.1108.56)
    Internet Explorer: 11.0.22000.120

Used Package Manager

pnpm

Logs

PS C:\Users\user\Desktop\vite-project> # i replaced my windows username with a generic "user"
PS C:\Users\user\Desktop\vite-project> pnpx vite build --debug
  vite:config no config file found. +0ms
  vite:config using resolved config: {
  vite:config   root: 'C:/Users/user/Desktop/vite-project',
  vite:config   base: '/',
  vite:config   mode: 'production',
  vite:config   configFile: undefined,
  vite:config   logLevel: undefined,
  vite:config   clearScreen: undefined,
  vite:config   build: {
  vite:config     target: [ 'es2019', 'edge88', 'firefox78', 'chrome87', 'safari13.1' ],
  vite:config     polyfillModulePreload: true,
  vite:config     outDir: 'dist',
  vite:config     assetsDir: 'assets',
  vite:config     assetsInlineLimit: 4096,
  vite:config     cssCodeSplit: true,
  vite:config     cssTarget: [ 'es2019', 'edge88', 'firefox78', 'chrome87', 'safari13.1' ],
  vite:config     sourcemap: false,
  vite:config     rollupOptions: {},
  vite:config     minify: 'esbuild',
  vite:config     terserOptions: {},
  vite:config     write: true,
  vite:config     emptyOutDir: null,
  vite:config     manifest: false,
  vite:config     lib: false,
  vite:config     ssr: false,
  vite:config     ssrManifest: false,
  vite:config     reportCompressedSize: true,
  vite:config     chunkSizeWarningLimit: 500,
  vite:config     watch: null,
  vite:config     commonjsOptions: { include: [Array], extensions: [Array] },
  vite:config     dynamicImportVarsOptions: { warnOnError: true, exclude: [Array] }
  vite:config   },
  vite:config   configFileDependencies: [],
  vite:config   inlineConfig: {
  vite:config     root: undefined,
  vite:config     base: undefined,
  vite:config     mode: undefined,
  vite:config     configFile: undefined,
  vite:config     logLevel: undefined,
  vite:config     clearScreen: undefined,
  vite:config     build: {}
  vite:config   },
  vite:config   resolve: { dedupe: undefined, alias: [ [Object], [Object] ] },
  vite:config   publicDir: 'C:\\Users\\user\\Desktop\\vite-project\\public',
  vite:config   cacheDir: 'C:\\Users\\user\\Desktop\\vite-project\\node_modules\\.vite',
  vite:config   command: 'build',
  vite:config   isProduction: true,
  vite:config   plugins: [
  vite:config     'alias',
  vite:config     'vite:modulepreload-polyfill',
  vite:config     'vite:resolve',
  vite:config     'vite:html-inline-proxy',
  vite:config     'vite:css',
  vite:config     'vite:esbuild',
  vite:config     'vite:json',
  vite:config     'vite:wasm',
  vite:config     'vite:worker',
  vite:config     'vite:worker-import-meta-url',
  vite:config     'vite:asset',
  vite:config     'vite:define',
  vite:config     'vite:css-post',
  vite:config     'vite:watch-package-data',
  vite:config     'vite:build-html',
  vite:config     'commonjs',
  vite:config     'vite:data-uri',
  vite:config     'rollup-plugin-dynamic-import-variables',
  vite:config     'vite:asset-import-meta-url',
  vite:config     'vite:build-import-analysis',
  vite:config     'vite:esbuild-transpile',
  vite:config     'vite:terser',
  vite:config     'vite:reporter',
  vite:config     'vite:load-fallback'
  vite:config   ],
  vite:config   server: {
  vite:config     preTransformRequests: true,
  vite:config     fs: { strict: true, allow: [Array], deny: [Array] }
  vite:config   },
  vite:config   preview: {
  vite:config     port: undefined,
  vite:config     strictPort: undefined,
  vite:config     host: undefined,
  vite:config     https: undefined,
  vite:config     open: undefined,
  vite:config     proxy: undefined,
  vite:config     cors: undefined,
  vite:config     headers: undefined
  vite:config   },
  vite:config   env: { BASE_URL: '/', MODE: 'production', DEV: false, PROD: true },
  vite:config   assetsInclude: [Function: assetsInclude],
  vite:config   logger: {
  vite:config     hasWarned: false,
  vite:config     info: [Function: info],
  vite:config     warn: [Function: warn],
  vite:config     warnOnce: [Function: warnOnce],
  vite:config     error: [Function: error],
  vite:config     clearScreen: [Function: clearScreen],
  vite:config     hasErrorLogged: [Function: hasErrorLogged]
  vite:config   },
  vite:config   packageCache: Map(0) { set: [Function (anonymous)] },
  vite:config   createResolver: [Function: createResolver],
  vite:config   optimizeDeps: {
  vite:config     esbuildOptions: { keepNames: undefined, preserveSymlinks: undefined }
  vite:config   },
  vite:config   worker: {
  vite:config     format: 'iife',
  vite:config     plugins: [
  vite:config       [Object], [Object], [Object],
  vite:config       [Object], [Object], [Object],
  vite:config       [Object], [Object], [Object],
  vite:config       [Object], [Object], [Object],
  vite:config       [Object], [Object], [Object],
  vite:config       [Object], [Object], [Object],
  vite:config       [Object], [Object], [Object],
  vite:config       [Object], [Object], [Object]
  vite:config     ],
  vite:config     rollupOptions: {}
  vite:config   }
  vite:config } +5ms
vite v2.8.4 building for production...
transforming (85241) src\lib\worker\theWorker.ts
<--- Last few GCs --->

[1580:000002AACF9E4580]   152266 ms: Mark-sweep 3980.6 (4139.2) -> 3968.6 (4141.2) MB, 1286.2 / 0.0 ms  (average mu = 0.131, current mu = 0.080) task scavenge might not succeed
[1580:000002AACF9E4580]   153712 ms: Mark-sweep 3982.3 (4141.4) -> 3970.8 (4143.4) MB, 1333.3 / 0.0 ms  (average mu = 0.105, current mu = 0.078) task scavenge might not succeed


<--- JS stacktrace --->

FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory
 1: 00007FF63F237B7F v8::internal::CodeObjectRegistry::~CodeObjectRegistry+114079
 2: 00007FF63F1C4546 DSA_meth_get_flags+65542
 3: 00007FF63F1C53FD node::OnFatalError+301
 4: 00007FF63FAFB29E v8::Isolate::ReportExternalAllocationLimitReached+94
 5: 00007FF63FAE587D v8::SharedArrayBuffer::Externalize+781
 6: 00007FF63F988C4C v8::internal::Heap::EphemeronKeyWriteBarrierFromCode+1468
 7: 00007FF63F9958F9 v8::internal::Heap::PublishPendingAllocations+1129
 8: 00007FF63F9928CA v8::internal::Heap::PageFlagsAreConsistent+2842
 9: 00007FF63F985529 v8::internal::Heap::CollectGarbage+2137
10: 00007FF63F935A95 v8::internal::IndexGenerator::~IndexGenerator+22165
11: 00007FF63F15607F std::basic_ostream<char,std::char_traits<char> >::operator<<+80079
12: 00007FF63F154896 std::basic_ostream<char,std::char_traits<char> >::operator<<+73958
13: 00007FF63F29708B uv_async_send+331
14: 00007FF63F29681C uv_loop_init+1292
15: 00007FF63F2969BA uv_run+202
16: 00007FF63F265825 node::SpinEventLoop+309
17: 00007FF63F17CE83 v8::internal::Isolate::stack_guard+53827
18: 00007FF63F1FA40C node::Start+220
19: 00007FF63F01894C RC4_options+348236
20: 00007FF64007FE68 v8::internal::compiler::RepresentationChanger::Uint32OverflowOperatorFor+14472
21: 00007FF9B1D354E0 BaseThreadInitThunk+16
22: 00007FF9B360485B RtlUserThreadStart+43
PS C:\Users\user\Desktop\vite-project>

Validations

Follow our Code of Conduct
Read the Contributing Guidelines.
Read the docs.
Check that there isn’t already an issue that reports the same bug to avoid creating a duplicate.
Make sure this is a Vite issue and not a framework-specific issue. For example, if it’s a Vue SFC related bug, it should likely be reported to https://github.com/vuejs/core instead.
Check that this is a concrete bug. For Q&A open a GitHub Discussion or join our Discord Chat Server.
The provided reproduction is a minimal reproducible example of the bug.

About this issue

Original URL
State: closed
Created 2 years ago
Reactions: 18
Comments: 26 (9 by maintainers)

Commits related to this issue

fixed build failure caused by https://github.com/vitejs/vite/issues/7015 — committed to Pyrolistical/hexpanse by Pyrolistical 2 years ago

Most upvoted comments

From the error logs, it seems that this happens from trying to bundle workers as CommonJS?

Would the error go away if worker constructor files were instead just treated as entry points for ESM bundling? That’s basically how Vite works in dev, and how I used to assume Vite built code as well.

I understand that some apps will want backwards compat through opt-in or opt-out, but all modern browsers and runtimes support module workers.

lgarron on Oct 6, 2023

I had the same issue, and have found a workaround after attempting for several days. Hope this can help someone.

The key idea is forcing the vite bypassing new Worker/SharedWorker(...)

let scriptForWorker = "/path/to/myWorker.js"
const worker = new Worker(scriptForWorker, { type: 'module' })

The scriptForWorker is a true variable, so the vite have no idea about how to process it, and will just keep it as a simple string. And then, let rollup build the myWorker.js for us.

modify vite.config, let rollup build ‘myWorker.js’
modify vite.config, let vite generate manifest.json

import { defineConfig } from 'vite';
export default defineConfig({
    build: {
        rollupOptions: {
            input: {
                main: "index.html",
                worker: "path/to/worker"
            }
        },
        manifest: true
    }
});

modify the source code about new Worker(...)

let scriptForWorker = "path/to/worker.ts";
if(!import.meta.env.DEV){
    // in production, we get the js from manifest.json
    // in `npm run dev` mode, using orignal path directly
    const response = await fetch("/manifest.json");
    if(response.ok){
        const manifest = await response.json();
        scriptForWorker = manifest[scriptForWorker].file;
    }
    else{
        return Promise.reject(response.statusText);
    }
}

// create the Worker
const worker = new Worker(scriptForWorker, { type:'module' });
// or the SharedWorker
const sharedWorker = new SharedWorker(scriptForWorker, { type:'module' });

building

npm run build
# or
npm run dev

AceBear on Nov 6, 2022

While I appreciate a partial fix in https://github.com/vitejs/vite/pull/16103 , I’d like to note that that it does far from solving the issue in general.

Trying to run npx vite build using a build of eef9da13d0028161eacc0ea699988814f29a56e4 on my repro from https://github.com/vitejs/vite/issues/14499 still fails the same way, unfortunately.

https://github.com/vitejs/vite/pull/16103 states that:

But I think it’s fine because doing that is not popular and will make the code complex.

I appreciate that it might not be possible to fix the current situation super easily, but it’s not a matter of popular conventions. Some of us don’t have a choice about the recursive dynamic imports in our code, unless we want our code to fail in other bundlers. 😢

Would it be possible to reopen one of these issues to keep track of the general problem?

Thanks, »Lucas

lgarron on Mar 6, 2024

I’ve attempted to create an experimental plugin that would make Vite chunk module workers instead of bundling. It seems to work for my use case and solves the circular dependency issue (Gist of plugin source).

There’s a few caveats that showed up:

Only works for workers instantiated like new Worker(new URL('path', import.meta.url), type: { 'module' })
module preload will need to be disabled in your vite config. Otherwise preload code is injected into the worker which breaks it.
You need to be careful that the module that instantiates the worker does not bundle any code that includes Web APIs that are not supported by workers. As your worker will import that bundle and it could break.

There’s likely other things that could break because the plugin simply tricks Vite into thinking the worker is a dynamic import of an ES module.

jamsinclair on Dec 4, 2023

This issue is still present in v5.0.0-beta.11. As mentioned in https://github.com/vitejs/vite/issues/14499#issuecomment-1740267849, it can be circumvented by doing:

Details

import { defineConfig } from 'vite';

// https://github.com/vitejs/vite/blob/ec7ee22cf15bed05a6c55693ecbac27cfd615118/packages/vite/src/node/plugins/workerImportMetaUrl.ts#L127-L128
const workerImportMetaUrlRE =
    /\bnew\s+(?:Worker|SharedWorker)\s*\(\s*(new\s+URL\s*\(\s*('[^']+'|"[^"]+"|`[^`]+`)\s*,\s*import\.meta\.url\s*\))/g

// https://vitejs.dev/config/
export default defineConfig({
    build: {
        target: 'esnext',
    },
    server: {
        host: 'localhost',
        headers: {
            'Cross-Origin-Embedder-Policy': 'require-corp',
            'Cross-Origin-Opener-Policy': 'same-origin',
            'Cross-Origin-Resource-Policy': 'cross-origin'
        }
    },
    optimizeDeps: {
        esbuildOptions: {
            target: 'esnext'
        }
    },
    worker: {
        format: 'es',
        // https://github.com/vitejs/vite/issues/7015
        // https://github.com/vitejs/vite/issues/14499#issuecomment-1740267849
        plugins: () => [
            {
                name: 'Disable nested workers',
                enforce: 'pre',
                transform(code: string, id: string) {
                    if (code.includes('new Worker') && code.includes('new URL') && code.includes('import.meta.url')) {
                        return code.replace(workerImportMetaUrlRE, `((() => { throw new Error('Nested workers are disabled') })()`);
                    }
                }
            }
        ]
    }
});

Note that for Emscripten-generated code this would mean that the locateFile handler on the incoming module would have to be overridden to provide support for nested workers.

kleisauke on Nov 22, 2023