README.md

@@ -1,2 +1,2 @@
-# kernel-swarm
+# kernel-dht
-Skynet kernel module for DHT requests using the @lumeweb/swarm-rpc-client package
+Skynet kernel module for DHT requests using the @lumeweb/dht-rpc-client package

build.js

@@ -3,12 +3,12 @@ import esbuild from "esbuild";
 esbuild.buildSync({
   entryPoints: ["src/index.ts"],
   outfile: "dist/index.js",
-  format: "iife",
+  format: "esm",
   bundle: true,
   legalComments: "external",
   //  minify: true
   define: {
     global: "self",
   },
-  inject: ["process.js"],
+    inject:['process.js']
 });

package.json

@@ -1,5 +1,5 @@
 {
-    "name": "@lumeweb/kernel-swarm",
+    "name": "@lumeweb/kernel-dht",
     "author": {
         "name": "Hammer Technologies LLC",
         "email": "contact@lumeweb.com"
@@ -9,48 +9,41 @@
         "format": "prettier -w src",
         "build-script": "tsc --project tsconfig.build.json && mv dist-build/build.js dist-build/build.mjs",
         "compile": "npm run build-script && node build.js",
-        "build": "npm run compile && node ./dist-build/build.mjs dev"
+        "build": "rimraf node_modules/@hyperswarm/secret-stream/node_modules node_modules/@lumeweb/dht-web/node_modules && npm run compile && node ./dist-build/build.mjs dev"
     },
     "type": "module",
     "dependencies": {
-        "@lumeweb/hyperswarm-web": "git+https://git.lumeweb.com/LumeWeb/hyperswarm-web.git",
+        "@lumeweb/dht-web": "https://github.com/LumeWeb/dht-web.git",
-        "@noble/ed25519": "^1.7.1",
+        "hyperswarm": "^4.0.2",
-        "b4a": "^1.6.1",
+        "libkmodule": "^0.2.12",
-        "hyperswarm": "^4.3.7",
-        "libkmodule": "^0.2.53",
         "libskynet": "^0.0.62",
-        "noise-handshake": "github:LumeWeb/noise-handshake",
+        "noise-handshake": "https://github.com/LumeWeb/noise-handshake.git",
-        "randombytes": "github:LumeWeb/randombytes-browser"
+        "randombytes": "https://github.com/LumeWeb/randombytes-browser.git"
     },
     "devDependencies": {
-        "@rollup/plugin-commonjs": "^22.0.2",
+        "@rollup/plugin-commonjs": "^22.0.1",
         "@rollup/plugin-node-resolve": "^13.3.0",
-        "@rollup/plugin-typescript": "^8.5.0",
+        "@rollup/plugin-typescript": "^8.3.3",
         "@screamingvoid/sodium-universal": "^0.1.1",
-        "@scure/bip39": "^1.1.0",
+        "@types/jest": "^28.1.3",
-        "@skynetlabs/skynet-nodejs": "^2.9.0",
-        "@types/b4a": "^1.6.0",
-        "@types/jest": "^28.1.8",
         "@types/read": "^0.0.29",
         "buffer": "^6.0.3",
-        "cli-progress": "^3.11.2",
         "crypto-browserify": "^3.12.0",
-        "esbuild": "^0.14.54",
+        "esbuild": "^0.14.48",
         "inspectpack": "^4.7.1",
-        "jest": "^28.1.3",
+        "jest": "^28.1.1",
-        "jest-puppeteer": "^6.2.0",
+        "jest-puppeteer": "^6.1.0",
-        "libskynetnode": "^0.1.4",
+        "libskynetnode": "^0.1.3",
-        "prettier": "^2.8.3",
+        "prettier": "^2.7.1",
         "process": "^0.11.10",
-        "puppeteer": "^15.5.0",
+        "puppeteer": "^15.2.0",
-        "random-number-csprng": "^1.0.2",
         "read": "^1.0.7",
-        "rollup": "^2.79.1",
+        "rollup": "^2.75.7",
         "rollup-plugin-polyfill-node": "^0.9.0",
         "stream-browserify": "^3.0.0",
-        "ts-loader": "^9.4.2",
+        "ts-loader": "^9.3.1",
-        "typescript": "^4.9.5",
+        "typescript": "^4.7.4",
-        "webpack": "^5.75.0",
+        "webpack": "^5.73.0",
         "webpack-cli": "^4.10.0"
     },
     "browser": {

src-build/build.ts

@@ -1,218 +0,0 @@
-// This is the standard build script for a kernel module.
-
-import * as fs from "fs";
-import read from "read";
-import * as bip39 from "@scure/bip39";
-import { wordlist } from "@scure/bip39/wordlists/english.js";
-//@ts-ignore
-import { SkynetClient } from "@skynetlabs/skynet-nodejs";
-
-// Helper variables to make it easier to return empty values alongside errors.
-const nu8 = new Uint8Array(0);
-const nkp = {
-  publicKey: nu8,
-  secretKey: nu8,
-};
-
-// readFile is a wrapper for fs.readFileSync that handles the try-catch for the
-// caller.
-function readFile(fileName: string): [string, string | null] {
-  try {
-    let data = fs.readFileSync(fileName, "utf8");
-    return [data, null];
-  } catch (err) {
-    return ["", "unable to read file: " + JSON.stringify(err)];
-  }
-}
-
-// readFileBinary is a wrapper for fs.readFileSync that handles the try-catch
-// for the caller.
-function readFileBinary(fileName: string): [Uint8Array, string | null] {
-  try {
-    let data = fs.readFileSync(fileName, null);
-    return [data, null];
-  } catch (err) {
-    return [nu8, "unable to read file: " + JSON.stringify(err)];
-  }
-}
-
-// writeFile is a wrapper for fs.writeFileSync which handles the try-catch in a
-// non-exception way.
-function writeFile(fileName: string, fileData: string): string | null {
-  try {
-    fs.writeFileSync(fileName, fileData);
-    return null;
-  } catch (err) {
-    return "unable to write file: " + JSON.stringify(err);
-  }
-}
-
-// handlePass handles all portions of the script that occur after the password
-// has been requested. If no password needs to be requested, handlePass will be
-// called with a null input. We need to structure the code this way because the
-// password reader is async and we can only access the password when using a
-// callback.
-function handlePass(password: string) {
-  try {
-    // If we are running prod and the seed file does not exist, we
-    // need to confirm the password and also warn the user to use a
-    // secure password.
-    if (!fs.existsSync(seedFile) && process.argv[2] === "prod") {
-      // The file does not exist, we need to confirm the
-      // password.
-      console.log();
-      console.log(
-        "No production entry found for module. Creating new production module..."
-      );
-      console.log(
-        "If someone can guess the password, they can push arbitrary changes to your module."
-      );
-      console.log("Please use a secure password.");
-      console.log();
-      read(
-        { prompt: "Confirm Password: ", silent: true },
-        function (err: any, confirmPassword: string) {
-          if (err) {
-            console.error("unable to fetch password:", err);
-            process.exit(1);
-          }
-          if (password !== confirmPassword) {
-            console.error("passwords do not match");
-            process.exit(1);
-          }
-          handlePassConfirm(moduleSalt, password);
-        }
-      );
-    } else {
-      // If the seed file does exist, or if we are using dev,
-      // there's no need to confirm the password but we do
-      // need to pass the logic off to the handlePassConfirm
-      // callback.
-      handlePassConfirm(moduleSalt, password);
-    }
-  } catch (err) {
-    console.error("Unable to read seedFile:", err);
-    process.exit(1);
-  }
-}
-
-// handlePassConfirm handles the full script after the confirmation password
-// has been provided. If not confirmation password is needed, this function
-// will be called anyway using the unconfirmed password as input.
-function handlePassConfirm(seed: string, password: string) {
-  // Create the seedFile if it does not exist. For dev we just save the
-  // seed to disk outright, because this is a dev build and therefore not
-  // security sensitive. Also the dev seed does not get pushed to the
-  // github repo.
-  //
-  // For prod, we use the seed to create a new seed (called the shield)
-  // which allows us to verify that the developer has provided the right
-  // password when deploying the module. The shield does get pushed to
-  // the github repo so that the production module is the same on all
-  // devices.
-  if (!fs.existsSync(seedFile) && process.argv[2] !== "prod") {
-    // Generate the seed phrase and write it to the file.
-    let seedPhrase = bip39.generateMnemonic(wordlist);
-    let errWF = writeFile(seedFile, seedPhrase);
-    if (errWF !== null) {
-      console.error("unable to write file:", errWF);
-      process.exit(1);
-    }
-  } else if (!fs.existsSync(seedFile) && process.argv[2] === "prod") {
-    // Generate the seed phrase.
-    let seedPhrase = bip39.generateMnemonic(wordlist);
-    // Write the registry link to the file.
-  }
-
-  // Load or verify the seed. If this is prod, the password is used to
-  // create and verify the seed. If this is dev, we just load the seed
-  // with no password.
-  let seedPhrase: string;
-  let registryLink: string;
-  if (process.argv[2] === "prod") {
-    // Generate the seed phrase from the password.
-    seedPhrase = bip39.generateMnemonic(wordlist);
-  } else {
-    let [sp, errRF] = readFile(seedFile);
-    if (errRF !== null) {
-      console.error("unable to read seed phrase for dev command from disk");
-      process.exit(1);
-    }
-    seedPhrase = sp;
-  }
-
-  let metadata = {
-    Filename: "index.js",
-  };
-  const client = new SkynetClient("https://web3portal.com");
-  client
-    .uploadFile("dist/index.js")
-    .then((result: any) => {
-      console.log("Immutable Link for kernel:", result);
-    })
-    .catch((err: any) => {
-      console.error("unable to upload file", err);
-      process.exit(1);
-    });
-}
-
-// Add a newline for readability.
-console.log();
-
-// Check for a 'dev' or 'prod' input to the script.
-if (process.argv.length !== 3) {
-  console.error("need to provide either 'dev' or 'prod' as an input");
-  process.exit(1);
-}
-
-// Create the build folder if it does not exist.
-if (!fs.existsSync("build")) {
-  fs.mkdirSync("build");
-}
-
-// Determine the seed file.
-let seedFile: string;
-if (process.argv[2] === "prod") {
-  seedFile = "module-skylink";
-} else if (process.argv[2] === "dev") {
-  seedFile = "build/dev-seed";
-} else {
-  console.error("need to provide either 'dev' or 'prod' as an input");
-  process.exit(1);
-}
-
-// If doing a prod deployment, check whether the salt file exists. If it does
-// not, create it.
-let moduleSalt: string;
-if (!fs.existsSync(".module-salt")) {
-  moduleSalt = bip39.generateMnemonic(wordlist);
-  let errWF = writeFile(".module-salt", moduleSalt);
-  if (errWF !== null) {
-    console.error("unable to write module salt file:", errWF);
-    process.exit(1);
-  }
-} else {
-  let [ms, errRF] = readFile(".module-salt");
-  if (errRF !== null) {
-    console.error("unable to read moduleSalt");
-    process.exit(1);
-  }
-  ms = ms.replace(/\n$/, "");
-  moduleSalt = ms;
-}
-
-// Need to get a password if this is a prod build.
-if (process.argv[2] === "prod") {
-  read(
-    { prompt: "Password: ", silent: true },
-    function (err: any, password: string) {
-      if (err) {
-        console.error("unable to fetch password:", err);
-        process.exit(1);
-      }
-      handlePass(password);
-    }
-  );
-} else {
-  handlePass("");
-}

src/index.ts

@@ -1,21 +1,20 @@
 // @ts-ignore
-import Hyperswarm from "@lumeweb/hyperswarm-web";
+import DHT from "@lumeweb/dht-web";
 import type { ActiveQuery } from "libkmodule";
 import { addHandler, getSeed, handleMessage } from "libkmodule";
 import { handlePresentSeed as handlePresentSeedModule } from "libkmodule/dist/seed.js";
 import type { Buffer } from "buffer";
-import * as ed from "@noble/ed25519";
+import { hexToBuf } from "libskynet";
-import b4a from "b4a";
 
 interface DhtConnection {
-  swarm: number;
+  dht: number;
   conn: any;
 }
 
 const connections = new Map<number, DhtConnection>();
-const swarmInstances = new Map<number, Hyperswarm>();
+const dhtInstances = new Map<number, DHT>();
 
-let defaultSwarm: Hyperswarm;
+let defaultDht: DHT;
 
 let moduleReadyResolve: Function;
 let moduleReady: Promise<void> = new Promise((resolve) => {
@@ -23,67 +22,120 @@ let moduleReady: Promise<void> = new Promise((resolve) => {
 });
 
 onmessage = handleMessage;
-function idFactory(start = 1) {
+function idFactory(start = 1, step = 1, limit = 2 ** 32) {
   let id = start;
 
   return function nextId() {
     const nextId = id;
-    id += 1;
+    id += step;
+    if (id >= limit) id = start;
     return nextId;
   };
 }
 
-const getSwarmId = idFactory();
+const nextId = idFactory(1);
-const getSocketId = idFactory();
 
 addHandler("presentSeed", handlePresentSeed);
-addHandler("joinPeer", handleJoinPeer);
+addHandler("openDht", handleOpenDht);
-addHandler("getPeerByPubkey", handleGetPeerByPubkey);
+addHandler("closeDht", handleCloseDht);
+addHandler("connect", handleConnect);
 addHandler("listenSocketEvent", handleListenSocketEvent, {
   receiveUpdates: true,
 });
 addHandler("socketExists", handleSocketExists);
 addHandler("close", handleCloseSocketEvent);
-addHandler("socketWrite", handleWriteSocketEvent);
+addHandler("write", handleWriteSocketEvent);
 addHandler("addRelay", handleAddRelay);
 addHandler("removeRelay", handleRemoveRelay);
 addHandler("clearRelays", handleClearRelays);
 addHandler("getRelays", handleGetRelays);
+addHandler("getRelayServers", handleGetRelayServers);
 addHandler("ready", handleReady);
 
 async function handlePresentSeed(aq: ActiveQuery) {
-  handlePresentSeedModule({
+  const keyPair = aq.callerInput.myskyRootKeypair;
-    callerInput: {
+  handlePresentSeedModule({ callerInput: { seed: keyPair } } as ActiveQuery);
-      seed: {
+  if (!defaultDht) {
-        publicKey: await ed.getPublicKey(aq.callerInput.rootKey),
+    defaultDht = dhtInstances.get(await createDht()) as DHT;
-        secretKey: aq.callerInput.rootKey,
-      },
-    },
-  } as ActiveQuery);
-
-  if (!defaultSwarm) {
-    defaultSwarm = swarmInstances.get(await createSwarm()) as Hyperswarm;
   }
   moduleReadyResolve();
 }
 
-async function createSwarm(): Promise<number> {
+async function handleOpenDht(aq: ActiveQuery) {
-  const swarmInstance = new Hyperswarm({ keyPair: await getSeed() });
+  const id = await createDht();
-  const id = getSwarmId();
+  aq.respond({ dht: id });
-  swarmInstances.set(id, swarmInstance);
+}
 
-  swarmInstance.on("connection", (peer) => {
+async function handleCloseDht(aq: ActiveQuery) {
-    const socketId = getSocketId();
+  const { dht = null } = aq.callerInput;
-    connections.set(socketId, { swarm: id, conn: peer });
 
-    peer.on("close", () => {
+  if (!dht) {
-      connections.delete(socketId);
+    aq.reject("Invalid DHT id");
-    });
+    return;
+  }
+
+  if (dht === defaultDht) {
+    aq.reject("Cannot close default DHT");
+    return;
+  }
+
+  dhtInstances.delete(dht);
+  Array.from(connections.values())
+    .filter((item) => item.dht === dht)
+    .forEach((item) => {
+      item.conn.end();
     });
 
+  aq.respond();
+}
+
+async function createDht(): Promise<number> {
+  const dhtInstance = new DHT({ keyPair: await getSeed() });
+  const id = nextId();
+  dhtInstances.set(id, dhtInstance);
   return id;
 }
 
+async function handleConnect(aq: ActiveQuery) {
+  const { pubkey, options = {} } = aq.callerInput;
+
+  let socket: any;
+
+  const dht = await getDht(aq);
+
+  try {
+    // @ts-ignore
+    socket = await dht.connect(
+      typeof pubkey === "string" ? hexToBuf(pubkey).shift() : pubkey,
+      options
+    );
+  } catch (e: any) {
+    aq.reject(e);
+    return;
+  }
+
+  const id = nextId();
+
+  socket.on("open", () => {
+    let dhtId: any = [...dhtInstances.entries()].filter(
+      (item) => item[1] === dht
+    );
+    dhtId = dhtId.shift()[0];
+
+    setDhtConnection(id, dhtId as number, socket);
+    aq.respond({ id });
+  });
+
+  socket.on("end", () => {
+    deleteDhtConnection(id);
+  });
+
+  socket.on("error", (e: any) => {
+    deleteDhtConnection(id);
+    aq.reject(e);
+  });
+}
+
 function handleListenSocketEvent(aq: ActiveQuery) {
   const { event = null } = aq.callerInput;
 
@@ -131,7 +183,7 @@ function handleListenSocketEvent(aq: ActiveQuery) {
 async function handleSocketExists(aq: ActiveQuery) {
   const { id = null } = aq.callerInput;
 
-  aq.respond(connections.has(Number(id)));
+  aq.respond(hasDhtConnection(Number(id)));
 }
 
 function handleCloseSocketEvent(aq: ActiveQuery) {
@@ -167,32 +219,32 @@ function handleWriteSocketEvent(aq: ActiveQuery) {
 function validateConnection(aq: ActiveQuery): any | boolean {
   const { id = null } = aq.callerInput;
 
-  if (!id || !connections.has(id)) {
+  if (!id || !hasDhtConnection(id)) {
     aq.reject("Invalid connection id");
     return false;
   }
 
-  return connections.get(id)?.conn;
+  return getDhtConnection(id)?.conn;
 }
 
-async function getSwarm(aq: ActiveQuery): Promise<Hyperswarm> {
+async function getDht(aq: ActiveQuery): Promise<DHT> {
   await moduleReady;
-  let swarm;
+  let dht;
   if ("callerInput" in aq && aq.callerInput) {
-    swarm = aq.callerInput.swarm ?? null;
+    dht = aq.callerInput.dht ?? null;
 
-    if (swarm && !swarmInstances.has(swarm)) {
+    if (dht && !dhtInstances.has(dht)) {
-      const error = "Invalid swarm id";
+      const error = "Invalid DHT id";
       aq.reject(error);
       throw new Error(error);
     }
   }
 
-  if (!swarm) {
+  if (!dht) {
-    return defaultSwarm;
+    return defaultDht;
   }
 
-  return swarmInstances.get(swarm) as Hyperswarm;
+  return dhtInstances.get(dht) as DHT;
 }
 
 async function handleAddRelay(aq: ActiveQuery) {
@@ -203,7 +255,7 @@ async function handleAddRelay(aq: ActiveQuery) {
     return;
   }
 
-  const dht = await getSwarm(aq);
+  const dht = await getDht(aq);
 
   aq.respond(await dht.addRelay(pubkey));
 }
@@ -216,13 +268,13 @@ async function handleRemoveRelay(aq: ActiveQuery) {
     return;
   }
 
-  const dht = await getSwarm(aq);
+  const dht = await getDht(aq);
 
   aq.respond(dht.removeRelay(pubkey));
 }
 
 async function handleClearRelays(aq: ActiveQuery) {
-  const dht = await getSwarm(aq);
+  const dht = await getDht(aq);
 
   dht.clearRelays();
 
@@ -230,61 +282,29 @@ async function handleClearRelays(aq: ActiveQuery) {
 }
 
 async function handleGetRelays(aq: ActiveQuery) {
-  aq.respond(await (await getSwarm(aq)).relays);
+  aq.respond(await (await getDht(aq)).relays);
 }
-
+async function handleGetRelayServers(aq: ActiveQuery) {
-async function handleJoinPeer(aq: ActiveQuery) {
+  aq.respond(await (await getDht(aq)).relayServers);
-  const { topic = null } = aq.callerInput;
-
-  const swarm = await getSwarm(aq);
-
-  if (!topic) {
-    aq.reject("invalid topic");
-    return;
-  }
-  if (!b4a.isBuffer(topic)) {
-    aq.reject("topic must be a buffer");
-    return;
-  }
-
-  // @ts-ignore
-  swarm.join(topic);
-  aq.respond();
-}
-async function handleGetPeerByPubkey(aq: ActiveQuery) {
-  const { pubkey = null } = aq.callerInput;
-
-  const swarm = await getSwarm(aq);
-
-  if (!pubkey) {
-    aq.reject("invalid topic");
-    return;
-  }
-
-  if (!b4a.isBuffer(pubkey)) {
-    aq.reject("pubkey must be a buffer");
-    return;
-  }
-
-  // @ts-ignore
-  if (!swarm._allConnections.has(pubkey)) {
-    aq.reject("peer does not exist");
-    return;
-  }
-
-  // @ts-ignore
-  const peer = swarm._allConnections.get(pubkey);
-
-  aq.respond(
-    [...connections.entries()].filter((conn) => {
-      return conn[1].conn === peer;
-    })[0][0]
-  );
 }
 
 async function handleReady(aq: ActiveQuery) {
-  const swarm = await getSwarm(aq);
+  await (await getDht(aq)).ready();
-  // @ts-ignore
-  await swarm.ready();
   aq.respond();
 }
+
+function setDhtConnection(id: number, dht: number, conn: any) {
+  connections.set(id, { dht, conn });
+}
+
+function getDhtConnection(id: number) {
+  return connections.get(id);
+}
+
+function hasDhtConnection(id: number) {
+  return connections.has(id);
+}
+
+function deleteDhtConnection(id: number) {
+  connections.delete(id);
+}