hypercore: Failed to get hypercore to replicate between server and browser.

With your suggestions I have managed to replicate the hypercore to the browser through a hyperswarm tooled with the dht-relay. I will prepare an example of my code for anyone else that visits this issue and close this issue with it.

Thank you for your help LuKks. I greatly appreciate your guidance through this matter.

Try setting up a dht-relay server using WebSocket: https://github.com/hyperswarm/dht-relay When you have that running then use that relay server i.e. ws://1.2.3.4:8080

For example, create a DHT node where you do node.createServer() so you can replicate from there as you already did. Later, from browser use the same dht-relay library to connect to the WebSocket relay, so you can easily create a DHT instance to do node.connect(serverPublicKey) and easily receive the replication.

My working setup for replicating hypercore over hyperswarm (https://github.com/hyperswarm/hyperswarm) between browser and node server using dht-relay (https://github.com/hyperswarm/dht-relay).

Server Example:

// For making the keyPair consistent. Not necessary but it helped me debug.
import HyperCrypto from "hypercore-crypto";
import crypto from "crypto";

// The important imports
import {createServer} from "http";
import {WebSocketServer} from "ws";
import {relay} from "@hyperswarm/dht-relay"
import RAM from "random-access-memory";
import Hypercore from "hypercore";

// Make httpServer to capture ws requests.
const httpServer = createServer();

const keys = keyPair("testFeed");
const core = new Hypercore(RAM, {
    keyPair: keys,
    valueEncoding: "json"
});

await core.ready();

console.log("CoreKey", core.key.toString("hex"));

await core.append("testFeed index 0 entry");
await core.append("testFeed index 1 entry");

const swarm = new Hyperswarm();
const wss = new WebSocketServer({noServer: true});

wss.on("connection", socket => {
    // Relay to the 'swarm.dht' instance.
    relay(swarm.dht, new WSRelayStream(false, socket));
});

swarm.on("connection", socket => {
    console.log("Got connection from socket. Attempting to replicate socket.")
    core.replicate(socket);
});

// Join core as server.
const disc = swarm.join(core.discoveryKey, {server: true, client: false});
await disc.flushed();

// Listen for http socket.
await new Promise(resolve => httpServer.listen(42001, resolve));

// Upgrade web socket.
httpServer.on('upgrade', function (request, socket, head) {
    wss.handleUpgrade(request, socket, head, async function done(ws) {
        wss.emit("connection", ws, request);
    });
});

// Making topics and keyPairs from string
function makeTopic(string, salt = "") {
    return crypto.createHash('sha256')
        .update(salt + string)
        .digest()
}
function keyPair(seed, salt = "") {
    return HyperCrypto.keyPair(Buffer.from(makeTopic(seed, salt)));
}

I used browserify to pack Hypercore, Hyperswarm, dht-relay using an html import map to load the scripts. I was also testing with different device types (Android/iPhone) so I displayed result to the page for convivence.

Client/Browser:

<script type="importmap">
        {
          "imports": {
            "b4a": "https://ga.jspm.io/npm:b4a@1.6.0/index.js",
            "lodash": "https://ga.jspm.io/npm:lodash@4.17.21/lodash.js",
            "random-access-idb": "https://ga.jspm.io/npm:random-access-idb@1.2.2/index.js",
            "random-access-memory": "https://ga.jspm.io/npm:random-access-memory@6.0.0/index.js",
            "hyperswarm": "./z/js/hyperswarm.js",
            "dht-relay": "./z/js/dhtbrowser.js",
            "dht-relay-ws": "./z/js/dhtbrowserws.js",
            "hypercore": "./z/js/hypercore.js"
          }
        }
</script>
<script async src="https://ga.jspm.io/npm:es-module-shims@1.5.1/dist/es-module-shims.js" crossorigin="anonymous"></script>

<div id="debugMessage"> 
        <h3>Debug Messages:</h3>
</div>

<script type="module">
	import b4a from "b4a";
	import DHT from "dht-relay"; // '@hyperswarm/dht-relay' browserified
	import DHT_WS from "dht-relay-ws"; // '@hyperswarm/dht-relay/lib/transport/ws' browserified
	import RAM from "random-access-memory"; 
	import Hyperswarm from "hyperswarm"; // Hyperswarm browserified
	import Hypercore from "hypercore"; // Hypercore browserified.
	import _ from "lodash";

	// The core key generated from server
	const hypercoreKey = "b08efa235e51ad04969c138805754701189de1758d12990e7c761b19afec380d"; 
	const core = new Hypercore(RAM, hypercoreKey);

	const ws = new WebSocket("ws://xxxxx:42001"); // Address omitted, but port is same as the one in WebSocketServer
	const node = new DHT(new DHT_WS(true, ws));

	const swarm = new Hyperswarm({
		dht: node
	});

	await core.ready();

	addMessage("Core ready.");

	const disc = swarm.join(core.discoveryKey, {server: false, client: true});
	await disc.flushed();

	swarm.on("connection", socket => {
		addMessage("Hyperswarm Socket Connected.");  

		core.replicate(socket);

		core.get(0, {
			onwait() {
				addMessage("Waiting for core[0] to be downloaded..");
			}
		}).then(data => {
			addMessage("test0 " + b4a.toString(data, "utf8"), "color: green");
		});

		core.get(1, {
			onwait() {
				addMessage("Waiting for core[1] to be downloaded..");
			}
		}).then(data => {
			addMessage("test1 " + b4a.toString(data, "utf8"), "color: green");
		});
	});

	// For printing to the page.
	function addMessage(msg, cssText) {
		if (_.isArray(msg)) return _.map(msg, m => addMessage(m, cssText));
		const ele = document.getElementById("debugMessage");
		const newDiv = document.createElement("div");

		if (!_.isEmpty(cssText)) {
			newDiv.style.cssText = (newDiv.style.cssText ?? "") + cssText;
		}

		newDiv.innerHTML = msg;
		ele.appendChild(newDiv);
		return newDiv;
	}

</script>

Ah, very true, I was missing the dht option. So the browser it’s like this:

import Hypercore from 'hypercore'
import RAM from 'random-access-memory'
import Hyperswarm from 'hyperswarm'
import DHT from '@hyperswarm/dht-relay'
import Stream from '@hyperswarm/dht-relay/ws'

const ws = new WebSocket('wss://dht2-relay.leet.ar')
const node = new DHT(new Stream(true, ws))

const coreKey = 'ed041921f1c96f4df676b58bf60b2133abf926ba0ad68bfb400de96aff4fb776'

main()

async function main () {
  const core = new Hypercore(RAM, coreKey, { valueEncoding: 'json' })
  await core.ready()

  const swarm = new Hyperswarm({ dht: node })
  swarm.on('connection', socket => core.replicate(socket))
  const discovery = swarm.join(core.discoveryKey, { server: false, client: true })
  await discovery.flushed()

  console.log('getting first block')
  console.log(await core.get(0))
}

Node.js

const Hypercore = require('hypercore')
const RAM = require('random-access-memory')
const Hyperswarm = require('hyperswarm')

const bootstrap = [
  { host: 'dht1.leet.ar', port: 49737 },
  { host: 'dht2.leet.ar', port: 49737 }
]

const core = new Hypercore(RAM, { valueEncoding: 'json' })

main()

async function main () {
  await core.ready()
  console.log('core key', core.key.toString('hex'))

  await core.append('ab')
  await core.append('cd')

  const swarm = new Hyperswarm({ bootstrap })
  swarm.on('connection', socket => core.replicate(socket))
  swarm.join(core.discoveryKey, { server: true, client: false })
}

Browser:

import Hypercore from 'hypercore'
import RAM from 'random-access-memory'
import DHT from '@hyperswarm/dht-relay'
import Stream from '@hyperswarm/dht-relay/ws'

const ws = new WebSocket('wss://dht2-relay.leet.ar')
const node = new DHT(new Stream(true, ws))

const coreKey = 'ed041921f1c96f4df676b58bf60b2133abf926ba0ad68bfb400de96aff4fb776'

main()

async function main () {
  const core = new Hypercore(RAM, coreKey, { valueEncoding: 'json' })
  await core.ready()

  const stream = node.lookup(core.discoveryKey)

  const found = []
  for await (const data of stream) {
    found.push(data)
  }

  if (!found.length || !found[0].peers.length) {
    console.log('peer not found')
    return
  }

  const peer = found[0].peers[0]
  console.log('found a peer', peer)

  const socket = node.connect(peer.publicKey)
  core.replicate(socket)

  console.log('getting first block')
  console.log(await core.get(0))
}

(this works but it’s not the best or correct way to do it)

Node.js:

const Hypercore = require('hypercore')
const RAM = require('random-access-memory')
const DHT = require('@hyperswarm/dht')

const bootstrap = [
  { host: 'dht1.leet.ar', port: 49737 },
  { host: 'dht2.leet.ar', port: 49737 }
]

const core = new Hypercore(RAM, { valueEncoding: 'json' })

main()

async function main () {
  await core.ready()
  console.log('core key', core.key.toString('hex'))

  await core.append('ab')
  await core.append('cd')

  const node = new DHT({ bootstrap })
  const server = node.createServer()
  server.on('connection', (socket) => core.replicate(socket))
  await server.listen()
  console.log('server public key', server.publicKey.toString('hex'))
}

Browser (I say React because I don’t know the effects of browserify vs webpack or whichever React uses):

import Hypercore from 'hypercore'
import RAM from 'random-access-memory'
import b4a from 'b4a'
import DHT from '@hyperswarm/dht-relay'
import Stream from '@hyperswarm/dht-relay/ws'

const ws = new WebSocket('wss://dht2-relay.leet.ar')
const node = new DHT(new Stream(true, ws))

const coreKey = 'b01df32c7a473b735f1974f060fbedb3a410d2db987932560ce62b8ad640f5c5'
const serverPublicKey = 'c007833b3288ddc1d139bef11f2aa617c9176cd74b0eb027130ae3e176caf712'

main()

async function main () {
  const core = new Hypercore(RAM, coreKey, { valueEncoding: 'json' })
  await core.ready()

  const socket = node.connect(b4a.from(serverPublicKey, 'hex'))
  core.replicate(socket)

  console.log('getting first block')
  console.log(await core.get(0))
}

Worst case scenario, create a repo to reproduce the issue so I can try

Otherwise, servers/clients may not found themselves because they would be in completely different setups/networks, but you will notice it so no problem

Cool, let me know!

Keep in mind that my DHT network is isolated from the mainnet network, that means if you use my relay then you have to set the bootstrap list.

For example, in Node.js:

const node = new DHT({
  bootstrap: [
    { host: 'dht1.leet.ar', port: 49737 },
    { host: 'dht2.leet.ar', port: 49737 }
  ]
})

In the browser:

const ws = new WebSocket('wss://dht2-relay.leet.ar')

But you can setup a relay server using the default bootstrap list (i.e. not setting the bootstrap variable)!