*Rewrite to use new design

README.md

@@ -1,2 +1,2 @@
-# kernel-dht
+# kernel-swarm
-Skynet kernel module for DHT requests using the @lumeweb/dht-rpc-client package
+Skynet kernel module for DHT requests using the @lumeweb/swarm-rpc-client package

build.js

@@ -3,12 +3,12 @@ import esbuild from "esbuild";
 esbuild.buildSync({
   entryPoints: ["src/index.ts"],
   outfile: "dist/index.js",
-  format: "esm",
+  format: "iife",
   bundle: true,
   legalComments: "external",
   //  minify: true
   define: {
     global: "self",
   },
-    inject:['process.js']
+  inject: ["process.js"],
 });

package.json

@@ -1,5 +1,5 @@
 {
-    "name": "@lumeweb/kernel-dht",
+    "name": "@lumeweb/kernel-swarm",
     "author": {
         "name": "Hammer Technologies LLC",
         "email": "contact@lumeweb.com"
@@ -9,41 +9,48 @@
         "format": "prettier -w src",
         "build-script": "tsc --project tsconfig.build.json && mv dist-build/build.js dist-build/build.mjs",
         "compile": "npm run build-script && node build.js",
-        "build": "rimraf node_modules/@hyperswarm/secret-stream/node_modules node_modules/@lumeweb/dht-web/node_modules && npm run compile && node ./dist-build/build.mjs dev"
+        "build": "npm run compile && node ./dist-build/build.mjs dev"
     },
     "type": "module",
     "dependencies": {
-        "@lumeweb/dht-web": "https://github.com/LumeWeb/dht-web.git",
+        "@lumeweb/hyperswarm-web": "git+https://git.lumeweb.com/LumeWeb/hyperswarm-web.git",
-        "hyperswarm": "^4.0.2",
+        "@noble/ed25519": "^1.7.1",
-        "libkmodule": "^0.2.12",
+        "b4a": "^1.6.1",
+        "hyperswarm": "^4.3.7",
+        "libkmodule": "^0.2.53",
         "libskynet": "^0.0.62",
-        "noise-handshake": "https://github.com/LumeWeb/noise-handshake.git",
+        "noise-handshake": "github:LumeWeb/noise-handshake",
-        "randombytes": "https://github.com/LumeWeb/randombytes-browser.git"
+        "randombytes": "github:LumeWeb/randombytes-browser"
     },
     "devDependencies": {
-        "@rollup/plugin-commonjs": "^22.0.1",
+        "@rollup/plugin-commonjs": "^22.0.2",
         "@rollup/plugin-node-resolve": "^13.3.0",
-        "@rollup/plugin-typescript": "^8.3.3",
+        "@rollup/plugin-typescript": "^8.5.0",
         "@screamingvoid/sodium-universal": "^0.1.1",
-        "@types/jest": "^28.1.3",
+        "@scure/bip39": "^1.1.0",
+        "@skynetlabs/skynet-nodejs": "^2.9.0",
+        "@types/b4a": "^1.6.0",
+        "@types/jest": "^28.1.8",
         "@types/read": "^0.0.29",
         "buffer": "^6.0.3",
+        "cli-progress": "^3.11.2",
         "crypto-browserify": "^3.12.0",
-        "esbuild": "^0.14.48",
+        "esbuild": "^0.14.54",
         "inspectpack": "^4.7.1",
-        "jest": "^28.1.1",
+        "jest": "^28.1.3",
-        "jest-puppeteer": "^6.1.0",
+        "jest-puppeteer": "^6.2.0",
-        "libskynetnode": "^0.1.3",
+        "libskynetnode": "^0.1.4",
-        "prettier": "^2.7.1",
+        "prettier": "^2.8.3",
         "process": "^0.11.10",
-        "puppeteer": "^15.2.0",
+        "puppeteer": "^15.5.0",
+        "random-number-csprng": "^1.0.2",
         "read": "^1.0.7",
-        "rollup": "^2.75.7",
+        "rollup": "^2.79.1",
         "rollup-plugin-polyfill-node": "^0.9.0",
         "stream-browserify": "^3.0.0",
-        "ts-loader": "^9.3.1",
+        "ts-loader": "^9.4.2",
-        "typescript": "^4.7.4",
+        "typescript": "^4.9.5",
-        "webpack": "^5.73.0",
+        "webpack": "^5.75.0",
         "webpack-cli": "^4.10.0"
     },
     "browser": {

src-build/build.ts

@@ -0,0 +1,218 @@
+// This is the standard build script for a kernel module.
+
+import * as fs from "fs";
+import read from "read";
+import * as bip39 from "@scure/bip39";
+import { wordlist } from "@scure/bip39/wordlists/english.js";
+//@ts-ignore
+import { SkynetClient } from "@skynetlabs/skynet-nodejs";
+
+// Helper variables to make it easier to return empty values alongside errors.
+const nu8 = new Uint8Array(0);
+const nkp = {
+  publicKey: nu8,
+  secretKey: nu8,
+};
+
+// readFile is a wrapper for fs.readFileSync that handles the try-catch for the
+// caller.
+function readFile(fileName: string): [string, string | null] {
+  try {
+    let data = fs.readFileSync(fileName, "utf8");
+    return [data, null];
+  } catch (err) {
+    return ["", "unable to read file: " + JSON.stringify(err)];
+  }
+}
+
+// readFileBinary is a wrapper for fs.readFileSync that handles the try-catch
+// for the caller.
+function readFileBinary(fileName: string): [Uint8Array, string | null] {
+  try {
+    let data = fs.readFileSync(fileName, null);
+    return [data, null];
+  } catch (err) {
+    return [nu8, "unable to read file: " + JSON.stringify(err)];
+  }
+}
+
+// writeFile is a wrapper for fs.writeFileSync which handles the try-catch in a
+// non-exception way.
+function writeFile(fileName: string, fileData: string): string | null {
+  try {
+    fs.writeFileSync(fileName, fileData);
+    return null;
+  } catch (err) {
+    return "unable to write file: " + JSON.stringify(err);
+  }
+}
+
+// handlePass handles all portions of the script that occur after the password
+// has been requested. If no password needs to be requested, handlePass will be
+// called with a null input. We need to structure the code this way because the
+// password reader is async and we can only access the password when using a
+// callback.
+function handlePass(password: string) {
+  try {
+    // If we are running prod and the seed file does not exist, we
+    // need to confirm the password and also warn the user to use a
+    // secure password.
+    if (!fs.existsSync(seedFile) && process.argv[2] === "prod") {
+      // The file does not exist, we need to confirm the
+      // password.
+      console.log();
+      console.log(
+        "No production entry found for module. Creating new production module..."
+      );
+      console.log(
+        "If someone can guess the password, they can push arbitrary changes to your module."
+      );
+      console.log("Please use a secure password.");
+      console.log();
+      read(
+        { prompt: "Confirm Password: ", silent: true },
+        function (err: any, confirmPassword: string) {
+          if (err) {
+            console.error("unable to fetch password:", err);
+            process.exit(1);
+          }
+          if (password !== confirmPassword) {
+            console.error("passwords do not match");
+            process.exit(1);
+          }
+          handlePassConfirm(moduleSalt, password);
+        }
+      );
+    } else {
+      // If the seed file does exist, or if we are using dev,
+      // there's no need to confirm the password but we do
+      // need to pass the logic off to the handlePassConfirm
+      // callback.
+      handlePassConfirm(moduleSalt, password);
+    }
+  } catch (err) {
+    console.error("Unable to read seedFile:", err);
+    process.exit(1);
+  }
+}
+
+// handlePassConfirm handles the full script after the confirmation password
+// has been provided. If not confirmation password is needed, this function
+// will be called anyway using the unconfirmed password as input.
+function handlePassConfirm(seed: string, password: string) {
+  // Create the seedFile if it does not exist. For dev we just save the
+  // seed to disk outright, because this is a dev build and therefore not
+  // security sensitive. Also the dev seed does not get pushed to the
+  // github repo.
+  //
+  // For prod, we use the seed to create a new seed (called the shield)
+  // which allows us to verify that the developer has provided the right
+  // password when deploying the module. The shield does get pushed to
+  // the github repo so that the production module is the same on all
+  // devices.
+  if (!fs.existsSync(seedFile) && process.argv[2] !== "prod") {
+    // Generate the seed phrase and write it to the file.
+    let seedPhrase = bip39.generateMnemonic(wordlist);
+    let errWF = writeFile(seedFile, seedPhrase);
+    if (errWF !== null) {
+      console.error("unable to write file:", errWF);
+      process.exit(1);
+    }
+  } else if (!fs.existsSync(seedFile) && process.argv[2] === "prod") {
+    // Generate the seed phrase.
+    let seedPhrase = bip39.generateMnemonic(wordlist);
+    // Write the registry link to the file.
+  }
+
+  // Load or verify the seed. If this is prod, the password is used to
+  // create and verify the seed. If this is dev, we just load the seed
+  // with no password.
+  let seedPhrase: string;
+  let registryLink: string;
+  if (process.argv[2] === "prod") {
+    // Generate the seed phrase from the password.
+    seedPhrase = bip39.generateMnemonic(wordlist);
+  } else {
+    let [sp, errRF] = readFile(seedFile);
+    if (errRF !== null) {
+      console.error("unable to read seed phrase for dev command from disk");
+      process.exit(1);
+    }
+    seedPhrase = sp;
+  }
+
+  let metadata = {
+    Filename: "index.js",
+  };
+  const client = new SkynetClient("https://web3portal.com");
+  client
+    .uploadFile("dist/index.js")
+    .then((result: any) => {
+      console.log("Immutable Link for kernel:", result);
+    })
+    .catch((err: any) => {
+      console.error("unable to upload file", err);
+      process.exit(1);
+    });
+}
+
+// Add a newline for readability.
+console.log();
+
+// Check for a 'dev' or 'prod' input to the script.
+if (process.argv.length !== 3) {
+  console.error("need to provide either 'dev' or 'prod' as an input");
+  process.exit(1);
+}
+
+// Create the build folder if it does not exist.
+if (!fs.existsSync("build")) {
+  fs.mkdirSync("build");
+}
+
+// Determine the seed file.
+let seedFile: string;
+if (process.argv[2] === "prod") {
+  seedFile = "module-skylink";
+} else if (process.argv[2] === "dev") {
+  seedFile = "build/dev-seed";
+} else {
+  console.error("need to provide either 'dev' or 'prod' as an input");
+  process.exit(1);
+}
+
+// If doing a prod deployment, check whether the salt file exists. If it does
+// not, create it.
+let moduleSalt: string;
+if (!fs.existsSync(".module-salt")) {
+  moduleSalt = bip39.generateMnemonic(wordlist);
+  let errWF = writeFile(".module-salt", moduleSalt);
+  if (errWF !== null) {
+    console.error("unable to write module salt file:", errWF);
+    process.exit(1);
+  }
+} else {
+  let [ms, errRF] = readFile(".module-salt");
+  if (errRF !== null) {
+    console.error("unable to read moduleSalt");
+    process.exit(1);
+  }
+  ms = ms.replace(/\n$/, "");
+  moduleSalt = ms;
+}
+
+// Need to get a password if this is a prod build.
+if (process.argv[2] === "prod") {
+  read(
+    { prompt: "Password: ", silent: true },
+    function (err: any, password: string) {
+      if (err) {
+        console.error("unable to fetch password:", err);
+        process.exit(1);
+      }
+      handlePass(password);
+    }
+  );
+} else {
+  handlePass("");
+}

src/index.ts

@@ -1,20 +1,21 @@
 // @ts-ignore
-import DHT from "@lumeweb/dht-web";
+import Hyperswarm from "@lumeweb/hyperswarm-web";
 import type { ActiveQuery } from "libkmodule";
 import { addHandler, getSeed, handleMessage } from "libkmodule";
 import { handlePresentSeed as handlePresentSeedModule } from "libkmodule/dist/seed.js";
 import type { Buffer } from "buffer";
-import { hexToBuf } from "libskynet";
+import * as ed from "@noble/ed25519";
+import b4a from "b4a";
 
 interface DhtConnection {
-  dht: number;
+  swarm: number;
   conn: any;
 }
 
 const connections = new Map<number, DhtConnection>();
-const dhtInstances = new Map<number, DHT>();
+const swarmInstances = new Map<number, Hyperswarm>();
 
-let defaultDht: DHT;
+let defaultSwarm: Hyperswarm;
 
 let moduleReadyResolve: Function;
 let moduleReady: Promise<void> = new Promise((resolve) => {
@@ -22,120 +23,67 @@ let moduleReady: Promise<void> = new Promise((resolve) => {
 });
 
 onmessage = handleMessage;
-function idFactory(start = 1, step = 1, limit = 2 ** 32) {
+function idFactory(start = 1) {
   let id = start;
 
   return function nextId() {
     const nextId = id;
-    id += step;
+    id += 1;
-    if (id >= limit) id = start;
     return nextId;
   };
 }
 
-const nextId = idFactory(1);
+const getSwarmId = idFactory();
+const getSocketId = idFactory();
 
 addHandler("presentSeed", handlePresentSeed);
-addHandler("openDht", handleOpenDht);
+addHandler("joinPeer", handleJoinPeer);
-addHandler("closeDht", handleCloseDht);
+addHandler("getPeerByPubkey", handleGetPeerByPubkey);
-addHandler("connect", handleConnect);
 addHandler("listenSocketEvent", handleListenSocketEvent, {
   receiveUpdates: true,
 });
 addHandler("socketExists", handleSocketExists);
 addHandler("close", handleCloseSocketEvent);
-addHandler("write", handleWriteSocketEvent);
+addHandler("socketWrite", handleWriteSocketEvent);
 addHandler("addRelay", handleAddRelay);
 addHandler("removeRelay", handleRemoveRelay);
 addHandler("clearRelays", handleClearRelays);
 addHandler("getRelays", handleGetRelays);
-addHandler("getRelayServers", handleGetRelayServers);
 addHandler("ready", handleReady);
 
 async function handlePresentSeed(aq: ActiveQuery) {
-  const keyPair = aq.callerInput.myskyRootKeypair;
+  handlePresentSeedModule({
-  handlePresentSeedModule({ callerInput: { seed: keyPair } } as ActiveQuery);
+    callerInput: {
-  if (!defaultDht) {
+      seed: {
-    defaultDht = dhtInstances.get(await createDht()) as DHT;
+        publicKey: await ed.getPublicKey(aq.callerInput.rootKey),
+        secretKey: aq.callerInput.rootKey,
+      },
+    },
+  } as ActiveQuery);
+
+  if (!defaultSwarm) {
+    defaultSwarm = swarmInstances.get(await createSwarm()) as Hyperswarm;
   }
   moduleReadyResolve();
 }
 
-async function handleOpenDht(aq: ActiveQuery) {
+async function createSwarm(): Promise<number> {
-  const id = await createDht();
+  const swarmInstance = new Hyperswarm({ keyPair: await getSeed() });
-  aq.respond({ dht: id });
+  const id = getSwarmId();
-}
+  swarmInstances.set(id, swarmInstance);
 
-async function handleCloseDht(aq: ActiveQuery) {
+  swarmInstance.on("connection", (peer) => {
-  const { dht = null } = aq.callerInput;
+    const socketId = getSocketId();
+    connections.set(socketId, { swarm: id, conn: peer });
 
-  if (!dht) {
+    peer.on("close", () => {
-    aq.reject("Invalid DHT id");
+      connections.delete(socketId);
-    return;
+    });
-  }
-
-  if (dht === defaultDht) {
-    aq.reject("Cannot close default DHT");
-    return;
-  }
-
-  dhtInstances.delete(dht);
-  Array.from(connections.values())
-    .filter((item) => item.dht === dht)
-    .forEach((item) => {
-      item.conn.end();
   });
 
-  aq.respond();
-}
-
-async function createDht(): Promise<number> {
-  const dhtInstance = new DHT({ keyPair: await getSeed() });
-  const id = nextId();
-  dhtInstances.set(id, dhtInstance);
   return id;
 }
 
-async function handleConnect(aq: ActiveQuery) {
-  const { pubkey, options = {} } = aq.callerInput;
-
-  let socket: any;
-
-  const dht = await getDht(aq);
-
-  try {
-    // @ts-ignore
-    socket = await dht.connect(
-      typeof pubkey === "string" ? hexToBuf(pubkey).shift() : pubkey,
-      options
-    );
-  } catch (e: any) {
-    aq.reject(e);
-    return;
-  }
-
-  const id = nextId();
-
-  socket.on("open", () => {
-    let dhtId: any = [...dhtInstances.entries()].filter(
-      (item) => item[1] === dht
-    );
-    dhtId = dhtId.shift()[0];
-
-    setDhtConnection(id, dhtId as number, socket);
-    aq.respond({ id });
-  });
-
-  socket.on("end", () => {
-    deleteDhtConnection(id);
-  });
-
-  socket.on("error", (e: any) => {
-    deleteDhtConnection(id);
-    aq.reject(e);
-  });
-}
-
 function handleListenSocketEvent(aq: ActiveQuery) {
   const { event = null } = aq.callerInput;
 
@@ -183,7 +131,7 @@ function handleListenSocketEvent(aq: ActiveQuery) {
 async function handleSocketExists(aq: ActiveQuery) {
   const { id = null } = aq.callerInput;
 
-  aq.respond(hasDhtConnection(Number(id)));
+  aq.respond(connections.has(Number(id)));
 }
 
 function handleCloseSocketEvent(aq: ActiveQuery) {
@@ -219,32 +167,32 @@ function handleWriteSocketEvent(aq: ActiveQuery) {
 function validateConnection(aq: ActiveQuery): any | boolean {
   const { id = null } = aq.callerInput;
 
-  if (!id || !hasDhtConnection(id)) {
+  if (!id || !connections.has(id)) {
     aq.reject("Invalid connection id");
     return false;
   }
 
-  return getDhtConnection(id)?.conn;
+  return connections.get(id)?.conn;
 }
 
-async function getDht(aq: ActiveQuery): Promise<DHT> {
+async function getSwarm(aq: ActiveQuery): Promise<Hyperswarm> {
   await moduleReady;
-  let dht;
+  let swarm;
   if ("callerInput" in aq && aq.callerInput) {
-    dht = aq.callerInput.dht ?? null;
+    swarm = aq.callerInput.swarm ?? null;
 
-    if (dht && !dhtInstances.has(dht)) {
+    if (swarm && !swarmInstances.has(swarm)) {
-      const error = "Invalid DHT id";
+      const error = "Invalid swarm id";
       aq.reject(error);
       throw new Error(error);
     }
   }
 
-  if (!dht) {
+  if (!swarm) {
-    return defaultDht;
+    return defaultSwarm;
   }
 
-  return dhtInstances.get(dht) as DHT;
+  return swarmInstances.get(swarm) as Hyperswarm;
 }
 
 async function handleAddRelay(aq: ActiveQuery) {
@@ -255,7 +203,7 @@ async function handleAddRelay(aq: ActiveQuery) {
     return;
   }
 
-  const dht = await getDht(aq);
+  const dht = await getSwarm(aq);
 
   aq.respond(await dht.addRelay(pubkey));
 }
@@ -268,13 +216,13 @@ async function handleRemoveRelay(aq: ActiveQuery) {
     return;
   }
 
-  const dht = await getDht(aq);
+  const dht = await getSwarm(aq);
 
   aq.respond(dht.removeRelay(pubkey));
 }
 
 async function handleClearRelays(aq: ActiveQuery) {
-  const dht = await getDht(aq);
+  const dht = await getSwarm(aq);
 
   dht.clearRelays();
 
@@ -282,29 +230,61 @@ async function handleClearRelays(aq: ActiveQuery) {
 }
 
 async function handleGetRelays(aq: ActiveQuery) {
-  aq.respond(await (await getDht(aq)).relays);
+  aq.respond(await (await getSwarm(aq)).relays);
 }
-async function handleGetRelayServers(aq: ActiveQuery) {
+
-  aq.respond(await (await getDht(aq)).relayServers);
+async function handleJoinPeer(aq: ActiveQuery) {
+  const { topic = null } = aq.callerInput;
+
+  const swarm = await getSwarm(aq);
+
+  if (!topic) {
+    aq.reject("invalid topic");
+    return;
+  }
+  if (!b4a.isBuffer(topic)) {
+    aq.reject("topic must be a buffer");
+    return;
+  }
+
+  // @ts-ignore
+  swarm.join(topic);
+  aq.respond();
+}
+async function handleGetPeerByPubkey(aq: ActiveQuery) {
+  const { pubkey = null } = aq.callerInput;
+
+  const swarm = await getSwarm(aq);
+
+  if (!pubkey) {
+    aq.reject("invalid topic");
+    return;
+  }
+
+  if (!b4a.isBuffer(pubkey)) {
+    aq.reject("pubkey must be a buffer");
+    return;
+  }
+
+  // @ts-ignore
+  if (!swarm._allConnections.has(pubkey)) {
+    aq.reject("peer does not exist");
+    return;
+  }
+
+  // @ts-ignore
+  const peer = swarm._allConnections.get(pubkey);
+
+  aq.respond(
+    [...connections.entries()].filter((conn) => {
+      return conn[1].conn === peer;
+    })[0][0]
+  );
 }
 
 async function handleReady(aq: ActiveQuery) {
-  await (await getDht(aq)).ready();
+  const swarm = await getSwarm(aq);
+  // @ts-ignore
+  await swarm.ready();
   aq.respond();
 }
-
-function setDhtConnection(id: number, dht: number, conn: any) {
-  connections.set(id, { dht, conn });
-}
-
-function getDhtConnection(id: number) {
-  return connections.get(id);
-}
-
-function hasDhtConnection(id: number) {
-  return connections.has(id);
-}
-
-function deleteDhtConnection(id: number) {
-  connections.delete(id);
-}