usearch: Bug: usearch@2.10.x nodejs installation fails

Describe the bug

NPM install fails on arm64 linux. I presume it couldn’t find a new prebuilt binary (exciting new feature from @sroussey), and fell back to compiling from scratch, which it can no longer do because the source and gyp files have been removed from the npm package.

Run npm ci
npm WARN deprecated @npmcli/move-file@2.0.1: This functionality has been moved to @npmcli/fs
npm WARN deprecated @aws-sdk/middleware-retry@3.374.0: This package has moved to @smithy/middleware-retry
npm WARN deprecated @aws-sdk/config-resolver@3.374.0: This package has moved to @smithy/config-resolver
npm WARN deprecated querystring@0.2.0: The querystring API is considered Legacy. new code should use the URLSearchParams API instead.
npm WARN deprecated @aws-sdk/smithy-client@3.374.0: This package has moved to @smithy/smithy-client
npm WARN deprecated @astrojs/webapi@3.0.0: This package is not used by Astro any more and is no longer maintained. In Astro 3.0 polyfills are part of a core module.
npm ERR! code 1
npm ERR! path /home/runner/actions-runner/_work/project/project/node_modules/usearch
npm ERR! command failed
npm ERR! command sh -c node-gyp-build
npm ERR! gyp info it worked if it ends with ok
npm ERR! gyp info using node-gyp@10.1.0
npm ERR! gyp info using node@20.11.1 | linux | arm64
npm ERR! gyp info find Python using Python version 3.10.12 found at "/usr/bin/python3"
npm ERR! gyp http GET https://nodejs.org/download/release/v20.11.1/node-v20.11.1-headers.tar.gz
npm ERR! gyp http 200 https://nodejs.org/download/release/v20.11.1/node-v20.11.1-headers.tar.gz
npm ERR! gyp http GET https://nodejs.org/download/release/v20.11.1/SHASUMS256.txt
npm ERR! gyp http 200 https://nodejs.org/download/release/v20.11.1/SHASUMS256.txt
npm ERR! gyp info spawn /usr/bin/python3
npm ERR! gyp info spawn args [
npm ERR! gyp info spawn args '/home/runner/actions-runner/_work/project/project/node_modules/node-gyp/gyp/gyp_main.py',
npm ERR! gyp info spawn args 'binding.gyp',
npm ERR! gyp info spawn args '-f',
npm ERR! gyp info spawn args 'make',
npm ERR! gyp info spawn args '-I',
npm ERR! gyp info spawn args '/home/runner/actions-runner/_work/project/project/node_modules/usearch/build/config.gypi',
npm ERR! gyp info spawn args '-I',
npm ERR! gyp info spawn args '/home/runner/actions-runner/_work/project/project/node_modules/node-gyp/addon.gypi',
npm ERR! gyp info spawn args '-I',
npm ERR! gyp info spawn args '/home/runner/.cache/node-gyp/20.11.1/include/node/common.gypi',
npm ERR! gyp info spawn args '-Dlibrary=shared_library',
npm ERR! gyp info spawn args '-Dvisibility=default',
npm ERR! gyp info spawn args '-Dnode_root_dir=/home/runner/.cache/node-gyp/20.11.1',
npm ERR! gyp info spawn args '-Dnode_gyp_dir=/home/runner/actions-runner/_work/project/project/node_modules/node-gyp',
npm ERR! gyp info spawn args '-Dnode_lib_file=/home/runner/.cache/node-gyp/20.11.1/<(target_arch)/node.lib',
npm ERR! gyp info spawn args '-Dmodule_root_dir=/home/runner/actions-runner/_work/project/project/node_modules/usearch',
npm ERR! gyp info spawn args '-Dnode_engine=v8',
npm ERR! gyp info spawn args '--depth=.',
npm ERR! gyp info spawn args '--no-parallel',
npm ERR! gyp info spawn args '--generator-output',
npm ERR! gyp info spawn args 'build',
npm ERR! gyp info spawn args '-Goutput_dir=.'
npm ERR! gyp info spawn args ]
npm ERR! gyp: binding.gyp not found (cwd: /home/runner/actions-runner/_work/project/project/node_modules/usearch) while trying to load binding.gyp
npm ERR! gyp ERR! configure error 
npm ERR! gyp ERR! stack Error: `gyp` failed with exit code: 1
npm ERR! gyp ERR! stack at ChildProcess.<anonymous> (/home/runner/actions-runner/_work/project/project/node_modules/node-gyp/lib/configure.js:297:18)
npm ERR! gyp ERR! stack at ChildProcess.emit (node:events:518:28)
npm ERR! gyp ERR! stack at ChildProcess._handle.onexit (node:internal/child_process:294:12)
npm ERR! gyp ERR! System Linux 5.15.0-84-generic
npm ERR! gyp ERR! command "/opt/hostedtoolcache/node/20.11.1/arm64/bin/node" "/home/runner/actions-runner/_work/project/project/node_modules/node-gyp/bin/node-gyp.js" "rebuild"
npm ERR! gyp ERR! cwd /home/runner/actions-runner/_work/project/project/node_modules/usearch
npm ERR! gyp ERR! node -v v20.11.1
npm ERR! gyp ERR! node-gyp -v v10.1.0
npm ERR! gyp ERR! not ok

npm ERR! A complete log of this run can be found in: /home/runner/.npm/_logs/2024-04-01T21_50_25_304Z-debug-0.log
Error: Process completed with exit code 1.

Steps to reproduce

On an arm64 machine, run

docker run -it --entrypoint /bin/bash --rm public.ecr.aws/lambda/nodejs:20-arm64

Inside the container, run

dnf install -y python3 make gcc-c++
npm init -y
npm i usearch

View the error:

npm ERR! code 1
npm ERR! path /var/task/node_modules/usearch
npm ERR! command failed
npm ERR! command sh -c node-gyp-build
npm ERR! gyp info it worked if it ends with ok
npm ERR! gyp info using node-gyp@9.4.0
npm ERR! gyp info using node@20.9.0 | linux | arm64
npm ERR! gyp info find Python using Python version 3.9.16 found at "/usr/bin/python3"
npm ERR! gyp http GET https://nodejs.org/download/release/v20.9.0/node-v20.9.0-headers.tar.gz
npm ERR! gyp http 200 https://nodejs.org/download/release/v20.9.0/node-v20.9.0-headers.tar.gz
npm ERR! gyp http GET https://nodejs.org/download/release/v20.9.0/SHASUMS256.txt
npm ERR! gyp http 200 https://nodejs.org/download/release/v20.9.0/SHASUMS256.txt
npm ERR! gyp info spawn /usr/bin/python3
npm ERR! gyp info spawn args [
npm ERR! gyp info spawn args   '/var/lang/lib/node_modules/npm/node_modules/node-gyp/gyp/gyp_main.py',
npm ERR! gyp info spawn args   'binding.gyp',
npm ERR! gyp info spawn args   '-f',
npm ERR! gyp info spawn args   'make',
npm ERR! gyp info spawn args   '-I',
npm ERR! gyp info spawn args   '/var/task/node_modules/usearch/build/config.gypi',
npm ERR! gyp info spawn args   '-I',
npm ERR! gyp info spawn args   '/var/lang/lib/node_modules/npm/node_modules/node-gyp/addon.gypi',
npm ERR! gyp info spawn args   '-I',
npm ERR! gyp info spawn args   '/root/.cache/node-gyp/20.9.0/include/node/common.gypi',
npm ERR! gyp info spawn args   '-Dlibrary=shared_library',
npm ERR! gyp info spawn args   '-Dvisibility=default',
npm ERR! gyp info spawn args   '-Dnode_root_dir=/root/.cache/node-gyp/20.9.0',
npm ERR! gyp info spawn args   '-Dnode_gyp_dir=/var/lang/lib/node_modules/npm/node_modules/node-gyp',
npm ERR! gyp info spawn args   '-Dnode_lib_file=/root/.cache/node-gyp/20.9.0/<(target_arch)/node.lib',
npm ERR! gyp info spawn args   '-Dmodule_root_dir=/var/task/node_modules/usearch',
npm ERR! gyp info spawn args   '-Dnode_engine=v8',
npm ERR! gyp info spawn args   '--depth=.',
npm ERR! gyp info spawn args   '--no-parallel',
npm ERR! gyp info spawn args   '--generator-output',
npm ERR! gyp info spawn args   'build',
npm ERR! gyp info spawn args   '-Goutput_dir=.'
npm ERR! gyp info spawn args ]
npm ERR! gyp: binding.gyp not found (cwd: /var/task/node_modules/usearch) while trying to load binding.gyp
npm ERR! gyp ERR! configure error
npm ERR! gyp ERR! stack Error: `gyp` failed with exit code: 1
npm ERR! gyp ERR! stack     at ChildProcess.onCpExit (/var/lang/lib/node_modules/npm/node_modules/node-gyp/lib/configure.js:325:16)
npm ERR! gyp ERR! stack     at ChildProcess.emit (node:events:514:28)
npm ERR! gyp ERR! stack     at ChildProcess._handle.onexit (node:internal/child_process:294:12)
npm ERR! gyp ERR! System Linux 6.6.12-linuxkit
npm ERR! gyp ERR! command "/var/lang/bin/node" "/var/lang/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js" "rebuild"
npm ERR! gyp ERR! cwd /var/task/node_modules/usearch
npm ERR! gyp ERR! node -v v20.9.0
npm ERR! gyp ERR! node-gyp -v v9.4.0
npm ERR! gyp ERR! not ok

npm ERR! A complete log of this run can be found in: /root/.npm/_logs/2024-04-01T23_50_47_987Z-debug-0.log

Expected behavior

NPM install to work with the linux-arm64 prebuilds

Regardless, it’s important for falling back to compiling from scratch with node-gyp to continue to work.

USearch version

2.10.x

Operating System

Amazon Linux 2023

Hardware architecture

Arm

Which interface are you using?

Other bindings

Contact Details

Ping me in Discord

Is there an existing issue for this?

I have searched the existing issues

Code of Conduct

I agree to follow this project’s Code of Conduct

About this issue

Original URL
State: open
Created 3 months ago
Comments: 69 (69 by maintainers)

Commits related to this issue

Make: Revert CI version Relates to #377 — committed to unum-cloud/usearch by ashvardanian 3 months ago
Build: Released 2.11.7 [skip ci] ## [2.11.7](https://github.com/unum-cloud/usearch/compare/v2.11.6...v2.11.7) (2024-04-15) ### Docs * Mention checkbox in issue request ([3a33d9d](https://github.com... — committed to unum-cloud/usearch by semantic-release-bot 3 months ago
Make: Disable Windows NPM builds Relates to the #377 and the comment: https://github.com/unum-cloud/usearch/issues/377#issuecomment-2057492107 This temporarily disables the failing CI pipeline to ge... — committed to unum-cloud/usearch by ashvardanian 2 months ago
Make: Disable JS prebuild for Windows Relates to https://github.com/unum-cloud/usearch/issues/377 This temporarily disables the failing CI pipeline to generate and update docs. — committed to ashvardanian/SimSIMD by ashvardanian 2 months ago

Most upvoted comments

These don’t include prebuilt binaries (you won’t find a .node file among them):

https://www.npmjs.com/package/better-sqlite3?activeTab=code https://www.npmjs.com/package/duckdb?activeTab=code https://www.npmjs.com/package/hnswlib-node?activeTab=code https://www.npmjs.com/package/@tensorflow/tfjs-node?activeTab=code

As opposed to usearch:

https://www.npmjs.com/package/usearch?activeTab=code

Frustrating.

Anyhow, I will look at ncc source code for ncc and see what they are doing for .node files.

sroussey on Apr 12, 2024

See https://github.com/unum-cloud/usearch/pull/383

sroussey on Apr 3, 2024

I guess I was overzealous on files[] part of the package.json. Will fix.

sroussey on Apr 2, 2024

You could disable for windows which should cause everyone to rebuild on their machine on that platform.

If they don’t allow lifecycle scripts it will fail, same with bun, etc. They will need to make changes to accommodate.

I’m not a big windows build expert but I’ll have another look this weekend. I’m curious if other people are having windows build issues on GitHub actions in 2024. What was the date it started failing? Might be a clue.

sroussey on May 3, 2024

I try to let the bundlers do that, because I’ve had problems with other libraries where the paths change due to bundling and it’s better just to let them handle it.

Yeah, looks like ncc will change it to be relative to the combined js file. So the copy of the prebuilds folder is adjacent to the index.js file (which just happens to be where we expect it to be).

sroussey on Apr 15, 2024

cosmetic changes

Maybe a red herring? It uses windows-latest so maybe that changed instead?

Looking at the changeset, I cannot see why the error would come up. You might try branching that off and do a force push of the last known good and see if that now fails.

sroussey on Apr 12, 2024

Also, I don’t think GitHub has arm64 for those.

sroussey on Apr 11, 2024

Those can cross compile, so faster to do it that way.

sroussey on Apr 11, 2024

Great! I’ll have a look.

Hopefully one of them uses prebuildify. I think the sqlite one (based on my poor memory) relies on a lifecycle script to download a prebuilt binary, and for security reasons people are moving away from having some deep dependency run arbitrary code on install on your machine (and another reason for dev containers). Bun, for example, does not run them.

I remember having issues since typescript doesn’t have __dirname etc. I know some better ways around that now however.

sroussey on Apr 11, 2024

Sure, the easiest way to locally reproduce without getting docker involved is:

mkdir usearch-bundle
cd usearch-bundle
npm init -y
npm install usearch@2.11.1

To get the entrypoint, run node -p "require.resolve('usearch')" or just run the commands below:

Then, try bundling with @vercel/ncc (using --external doesn’t make sense in this case, because our goal is to bundle usearch, not skip it):

npx --yes @vercel/ncc build node_modules/usearch/javascript/dist/cjs/usearch.js -o ncc-usearch-bundled --debug

or with esbuild:

npx --yes esbuild node_modules/usearch/javascript/dist/cjs/usearch.js --bundle --outdir=esbuild-usearch-bundled --loader:.node=file --platform=node

Bundling works great with many other native addons that have prebuilds, such as:

better-sqlite3

	let addon;
	if (nativeBinding == null) {
		addon = DEFAULT_ADDON || (DEFAULT_ADDON = require('bindings')('better_sqlite3.node'));
	} else if (typeof nativeBinding === 'string') {
		// See <https://webpack.js.org/api/module-variables/#__non_webpack_require__-webpack-specific>
		const requireFunc = typeof __non_webpack_require__ === 'function' ? __non_webpack_require__ : require;
		addon = requireFunc(path.resolve(nativeBinding).replace(/(\.node)?$/, '.node'));
	} else {
		// See <https://github.com/WiseLibs/better-sqlite3/issues/972>
		addon = nativeBinding;
	}

duckdb

var binary = require('@mapbox/node-pre-gyp');
var path = require('path');
var binding_path = binary.find(path.resolve(path.join(__dirname,'../package.json')));
var binding = require(binding_path);

module.exports = exports = binding;

hnswlib-node

const addon = require('bindings')('addon');
module.exports = addon;

@tensorflow/tfjs-node
And many more…

It should be possible to add prebuilds without breaking bundling. @sroussey hopefully some of those package examples can be inspiration. I’ll look at it more, but I’m not familiar enough with prebuilds and node-gyp to immediately see what the solution is.

jlarmstrongiv on Apr 11, 2024

At any rate, I don’t use aws lambda, but if you give me some steps to reproduce locally, I’ll have a look.

sroussey on Apr 10, 2024

update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 60 --slave /usr/bin/g++ g++ /usr/bin/g++-12 fixed it, thanks.

sroussey on Apr 3, 2024

Works fine on MacOS though.

sroussey on Apr 3, 2024

I’m home a bit today, so I’ll comment out that section until I have a working version.

sroussey on Apr 3, 2024

bash-5.2# npm install @sroussey/usearch

added 5 packages, and audited 6 packages in 6s

found 0 vulnerabilities

The install passed! Haven’t tried running sample code yet though

jlarmstrongiv on Apr 2, 2024

Oh, they are the same size, so definitely not right.

sroussey on Apr 2, 2024