refactor: update to new sdks and portal

.github/workflows/ci.yml

@@ -0,0 +1,13 @@
+name: Build/Publish
+
+on:
+    push:
+        branches:
+            - master
+            - develop
+            - develop-*
+
+jobs:
+    main:
+        uses: lumeweb/github-node-deploy-workflow/.github/workflows/main.yml@master
+        secrets: inherit

.presetterrc.json

@@ -0,0 +1,8 @@
+{
+  "preset": [
+    "@lumeweb/presetter-kernel-module-preset"
+  ],
+    "config": {
+        "official": true
+    }
+}

build.js

@@ -1,14 +0,0 @@
-import esbuild from "esbuild";
-
-esbuild.buildSync({
-  entryPoints: ["src/index.ts"],
-  outfile: "dist/index.js",
-  format: "iife",
-  bundle: true,
-  legalComments: "external",
-  //  minify: true
-  define: {
-    global: "self",
-  },
-  inject: ["process.js", "polyfill.js"],
-});

package.json

@@ -4,64 +4,36 @@
         "name": "Hammer Technologies LLC",
         "email": "contact@lumeweb.com"
     },
+    "repository": {
+        "type": "git",
+        "url": "gitea@git.lumeweb.com:LumeWeb/kernel-swarm.git"
+    },
     "scripts": {
-        "test": "jest",
+        "prepare": "presetter bootstrap",
-        "format": "prettier -w src",
+        "build": "run build",
-        "build-script": "tsc --project tsconfig.build.json && mv dist-build/build.js dist-build/build.mjs",
+        "semantic-release": "semantic-release"
-        "compile": "npm run build-script && node build.js",
-        "build": "npm run compile && node ./dist-build/build.mjs dev"
     },
     "type": "module",
     "dependencies": {
-        "@lumeweb/hyperswarm-web": "git+https://git.lumeweb.com/LumeWeb/hyperswarm-web.git",
+        "@lumeweb/hyperswarm-web": "0.0.2-develop.3",
-        "@lumeweb/rpc": "git+https://git.lumeweb.com/LumeWeb/rpc.git",
+        "@lumeweb/libkernel": "^0.1.0-develop.7",
-        "@noble/ed25519": "^1.7.3",
+        "@lumeweb/rpc": "0.0.2-develop.1",
-        "@peculiar/webcrypto": "git+https://git.lumeweb.com/LumeWeb/webcrypto.git",
+        "@noble/curves": "^1.1.0",
+        "@peculiar/webcrypto": "^1.4.3",
         "async-mutex": "^0.4.0",
-        "b4a": "^1.6.3",
+        "b4a": "^1.6.4",
         "eventemitter2": "^6.4.9",
-        "hyperswarm": "^4.4.0",
+        "hyperswarm": "^4.5.1",
-        "libkmodule": "^0.2.53",
+        "noise-handshake": "^3.0.3",
-        "libskynet": "^0.0.62",
-        "noise-handshake": "^3.0.2",
         "p-defer": "^4.0.0",
-        "protomux": "git+https://git.lumeweb.com/LumeWeb/kernel-protomux.git",
+        "protomux": "^3.5.0"
-        "randombytes": "github:LumeWeb/randombytes-browser"
     },
     "devDependencies": {
-        "@rollup/plugin-commonjs": "^22.0.2",
+        "@lumeweb/presetter-kernel-module-preset": "^0.1.0-develop.20",
-        "@rollup/plugin-node-resolve": "^13.3.0",
-        "@rollup/plugin-typescript": "^8.5.0",
-        "@screamingvoid/sodium-universal": "^0.1.1",
-        "@scure/bip39": "^1.2.0",
-        "@skynetlabs/skynet-nodejs": "^2.9.0",
         "@types/b4a": "^1.6.0",
-        "@types/jest": "^28.1.8",
+        "presetter": "*"
-        "@types/read": "^0.0.29",
-        "buffer": "^6.0.3",
-        "cli-progress": "^3.12.0",
-        "crypto-browserify": "^3.12.0",
-        "esbuild": "^0.14.54",
-        "inspectpack": "^4.7.1",
-        "jest": "^28.1.3",
-        "jest-puppeteer": "^6.2.0",
-        "libskynetnode": "^0.1.4",
-        "prettier": "^2.8.7",
-        "process": "^0.11.10",
-        "puppeteer": "^15.5.0",
-        "random-number-csprng": "^1.0.2",
-        "read": "^1.0.7",
-        "rollup": "^2.79.1",
-        "rollup-plugin-polyfill-node": "^0.9.0",
-        "stream-browserify": "^3.0.0",
-        "ts-loader": "^9.4.2",
-        "typescript": "^4.9.5",
-        "webpack": "^5.79.0",
-        "webpack-cli": "^4.10.0"
     },
-    "browser": {
+    "version": "",
-        "sodium-universal": "@screamingvoid/sodium-universal",
+    "readme": "ERROR: No README data found!",
-        "crypto": "crypto-browserify",
+    "_id": "@lumeweb/kernel-swarm@"
-        "stream": "stream-browserify"
-    }
 }

patches/protomux+3.5.0.patch

@@ -0,0 +1,45 @@
+diff --git a/node_modules/protomux/index.js b/node_modules/protomux/index.js
+index 07bf0ad..c8db59c 100644
+--- a/node_modules/protomux/index.js
++++ b/node_modules/protomux/index.js
+@@ -195,34 +195,34 @@ class Channel {
+       type,
+       encoding,
+       onmessage,
+-      recv (state, session) {
++      async recv (state, session) {
+         session._track(m.onmessage(encoding.decode(state), session))
+       }, 
+-      send (m, session = s) {
++      async send (m, session = s) {
+         if (session.closed === true) return false
+ 
+         const mux = session._mux
+         const state = { buffer: null, start: 0, end: typeLen }
+ 
+         if (mux._batch !== null) {
+-          encoding.preencode(state, m)
++          await encoding.preencode(state, m)
+           state.buffer = mux._alloc(state.end)
+ 
+           c.uint.encode(state, type)
+-          encoding.encode(state, m)
++          await encoding.encode(state, m)
+ 
+           mux._pushBatch(session._localId, state.buffer)
+           return true
+         }
+ 
+         c.uint.preencode(state, session._localId)
+-        encoding.preencode(state, m)
++        await encoding.preencode(state, m)
+ 
+         state.buffer = mux._alloc(state.end)
+ 
+         c.uint.encode(state, session._localId)
+         c.uint.encode(state, type)
+-        encoding.encode(state, m)
++        await encoding.encode(state, m)
+ 
+         mux.drained = mux.stream.write(state.buffer)
+ 

polyfill.js

@@ -1,11 +0,0 @@
-import { Crypto } from "@peculiar/webcrypto";
-
-let globalCrypto = self.crypto;
-if (!globalCrypto.subtle) {
-  let subtleCrypto = new Crypto().subtle;
-  Object.defineProperty(globalCrypto, "subtle", {
-    get() {
-      return subtleCrypto;
-    },
-  });
-}

process.js

@@ -1,3 +0,0 @@
-export const process = require('process');
-
-

src-build/build.ts

@@ -1,218 +0,0 @@
-// This is the standard build script for a kernel module.
-
-import * as fs from "fs";
-import read from "read";
-import * as bip39 from "@scure/bip39";
-import { wordlist } from "@scure/bip39/wordlists/english.js";
-//@ts-ignore
-import { SkynetClient } from "@skynetlabs/skynet-nodejs";
-
-// Helper variables to make it easier to return empty values alongside errors.
-const nu8 = new Uint8Array(0);
-const nkp = {
-  publicKey: nu8,
-  secretKey: nu8,
-};
-
-// readFile is a wrapper for fs.readFileSync that handles the try-catch for the
-// caller.
-function readFile(fileName: string): [string, string | null] {
-  try {
-    let data = fs.readFileSync(fileName, "utf8");
-    return [data, null];
-  } catch (err) {
-    return ["", "unable to read file: " + JSON.stringify(err)];
-  }
-}
-
-// readFileBinary is a wrapper for fs.readFileSync that handles the try-catch
-// for the caller.
-function readFileBinary(fileName: string): [Uint8Array, string | null] {
-  try {
-    let data = fs.readFileSync(fileName, null);
-    return [data, null];
-  } catch (err) {
-    return [nu8, "unable to read file: " + JSON.stringify(err)];
-  }
-}
-
-// writeFile is a wrapper for fs.writeFileSync which handles the try-catch in a
-// non-exception way.
-function writeFile(fileName: string, fileData: string): string | null {
-  try {
-    fs.writeFileSync(fileName, fileData);
-    return null;
-  } catch (err) {
-    return "unable to write file: " + JSON.stringify(err);
-  }
-}
-
-// handlePass handles all portions of the script that occur after the password
-// has been requested. If no password needs to be requested, handlePass will be
-// called with a null input. We need to structure the code this way because the
-// password reader is async and we can only access the password when using a
-// callback.
-function handlePass(password: string) {
-  try {
-    // If we are running prod and the seed file does not exist, we
-    // need to confirm the password and also warn the user to use a
-    // secure password.
-    if (!fs.existsSync(seedFile) && process.argv[2] === "prod") {
-      // The file does not exist, we need to confirm the
-      // password.
-      console.log();
-      console.log(
-        "No production entry found for module. Creating new production module..."
-      );
-      console.log(
-        "If someone can guess the password, they can push arbitrary changes to your module."
-      );
-      console.log("Please use a secure password.");
-      console.log();
-      read(
-        { prompt: "Confirm Password: ", silent: true },
-        function (err: any, confirmPassword: string) {
-          if (err) {
-            console.error("unable to fetch password:", err);
-            process.exit(1);
-          }
-          if (password !== confirmPassword) {
-            console.error("passwords do not match");
-            process.exit(1);
-          }
-          handlePassConfirm(moduleSalt, password);
-        }
-      );
-    } else {
-      // If the seed file does exist, or if we are using dev,
-      // there's no need to confirm the password but we do
-      // need to pass the logic off to the handlePassConfirm
-      // callback.
-      handlePassConfirm(moduleSalt, password);
-    }
-  } catch (err) {
-    console.error("Unable to read seedFile:", err);
-    process.exit(1);
-  }
-}
-
-// handlePassConfirm handles the full script after the confirmation password
-// has been provided. If not confirmation password is needed, this function
-// will be called anyway using the unconfirmed password as input.
-function handlePassConfirm(seed: string, password: string) {
-  // Create the seedFile if it does not exist. For dev we just save the
-  // seed to disk outright, because this is a dev build and therefore not
-  // security sensitive. Also the dev seed does not get pushed to the
-  // github repo.
-  //
-  // For prod, we use the seed to create a new seed (called the shield)
-  // which allows us to verify that the developer has provided the right
-  // password when deploying the module. The shield does get pushed to
-  // the github repo so that the production module is the same on all
-  // devices.
-  if (!fs.existsSync(seedFile) && process.argv[2] !== "prod") {
-    // Generate the seed phrase and write it to the file.
-    let seedPhrase = bip39.generateMnemonic(wordlist);
-    let errWF = writeFile(seedFile, seedPhrase);
-    if (errWF !== null) {
-      console.error("unable to write file:", errWF);
-      process.exit(1);
-    }
-  } else if (!fs.existsSync(seedFile) && process.argv[2] === "prod") {
-    // Generate the seed phrase.
-    let seedPhrase = bip39.generateMnemonic(wordlist);
-    // Write the registry link to the file.
-  }
-
-  // Load or verify the seed. If this is prod, the password is used to
-  // create and verify the seed. If this is dev, we just load the seed
-  // with no password.
-  let seedPhrase: string;
-  let registryLink: string;
-  if (process.argv[2] === "prod") {
-    // Generate the seed phrase from the password.
-    seedPhrase = bip39.generateMnemonic(wordlist);
-  } else {
-    let [sp, errRF] = readFile(seedFile);
-    if (errRF !== null) {
-      console.error("unable to read seed phrase for dev command from disk");
-      process.exit(1);
-    }
-    seedPhrase = sp;
-  }
-
-  let metadata = {
-    Filename: "index.js",
-  };
-  const client = new SkynetClient("https://web3portal.com");
-  client
-    .uploadFile("dist/index.js")
-    .then((result: any) => {
-      console.log("Immutable Link for kernel:", result);
-    })
-    .catch((err: any) => {
-      console.error("unable to upload file", err);
-      process.exit(1);
-    });
-}
-
-// Add a newline for readability.
-console.log();
-
-// Check for a 'dev' or 'prod' input to the script.
-if (process.argv.length !== 3) {
-  console.error("need to provide either 'dev' or 'prod' as an input");
-  process.exit(1);
-}
-
-// Create the build folder if it does not exist.
-if (!fs.existsSync("build")) {
-  fs.mkdirSync("build");
-}
-
-// Determine the seed file.
-let seedFile: string;
-if (process.argv[2] === "prod") {
-  seedFile = "module-skylink";
-} else if (process.argv[2] === "dev") {
-  seedFile = "build/dev-seed";
-} else {
-  console.error("need to provide either 'dev' or 'prod' as an input");
-  process.exit(1);
-}
-
-// If doing a prod deployment, check whether the salt file exists. If it does
-// not, create it.
-let moduleSalt: string;
-if (!fs.existsSync(".module-salt")) {
-  moduleSalt = bip39.generateMnemonic(wordlist);
-  let errWF = writeFile(".module-salt", moduleSalt);
-  if (errWF !== null) {
-    console.error("unable to write module salt file:", errWF);
-    process.exit(1);
-  }
-} else {
-  let [ms, errRF] = readFile(".module-salt");
-  if (errRF !== null) {
-    console.error("unable to read moduleSalt");
-    process.exit(1);
-  }
-  ms = ms.replace(/\n$/, "");
-  moduleSalt = ms;
-}
-
-// Need to get a password if this is a prod build.
-if (process.argv[2] === "prod") {
-  read(
-    { prompt: "Password: ", silent: true },
-    function (err: any, password: string) {
-      if (err) {
-        console.error("unable to fetch password:", err);
-        process.exit(1);
-      }
-      handlePass(password);
-    }
-  );
-} else {
-  handlePass("");
-}

src/addr.ts

@@ -22,7 +22,7 @@ export function pubKeyToIpv6(publicKey: Uint8Array) {
   addr[prefix.length] = ones;
   addr.set(nodeId, prefix.length + 1);
 
-  const result = [];
+  const result: string[] = [];
   for (let i = 0; i < 8; i++) {
     const num1 = addr[i * 2].toString(16).padStart(2, "0");
     const num2 = addr[i * 2 + 1].toString(16).padStart(2, "0");
@@ -46,7 +46,7 @@ function getLeadingOnes(buf: Uint8Array) {
 }
 
 function getTruncatedNodeID(buf: Uint8Array) {
-  const result = [];
+  const result: number[] = [];
   let done = false;
   let bits = 0;
   let nBits = 0;

src/index.ts

@@ -1,14 +1,18 @@
 // @ts-ignore
 import Hyperswarm from "@lumeweb/hyperswarm-web";
-import type { ActiveQuery } from "libkmodule";
+import type { ActiveQuery } from "@lumeweb/libkernel/module";
-import { addHandler, getSeed, handleMessage } from "libkmodule";
+import {
-import { handlePresentSeed as handlePresentSeedModule } from "libkmodule/dist/seed.js";
+  addHandler,
+  getKey,
+  handleMessage,
+  handlePresentKey as handlePresentKeyModule,
+} from "@lumeweb/libkernel/module";
 import { Buffer } from "buffer";
-import * as ed from "@noble/ed25519";
+import { ed25519 } from "@noble/curves/ed25519";
 import b4a from "b4a";
 import { pubKeyToIpv6 } from "./addr.js";
 import { EventEmitter2 as EventEmitter } from "eventemitter2";
-import { logErr } from "libkmodule/dist";
+import { logErr } from "@lumeweb/libkernel";
 // @ts-ignore
 import Protomux from "protomux";
 import defer, { DeferredPromise } from "p-defer";
@@ -54,7 +58,7 @@ const getSocketId = idFactory();
 const getChannelId = idFactory();
 const getMessageId = idFactory();
 
-addHandler("presentSeed", handlePresentSeed);
+addHandler("presentKey", handlePresentKey);
 addHandler("join", handleJoin);
 addHandler("getPeerByPubkey", handleGetPeerByPubkey);
 
@@ -83,12 +87,12 @@ addHandler("createProtomuxMessage", handleCreateProtomuxMessage, {
 });
 addHandler("createSwarm", handleCreateSwarm);
 
-async function handlePresentSeed(aq: ActiveQuery) {
+async function handlePresentKey(aq: ActiveQuery) {
-  const pubkey = await ed.getPublicKey(aq.callerInput.rootKey);
+  const pubkey = ed25519.getPublicKey(aq.callerInput.rootKey);
-  handlePresentSeedModule({
+  handlePresentKeyModule({
     callerInput: {
       seed: {
-        publicKey: await ed.getPublicKey(aq.callerInput.rootKey),
+        publicKey: ed25519.getPublicKey(aq.callerInput.rootKey),
         secretKey: b4a.concat([aq.callerInput.rootKey, pubkey]),
       },
     },
@@ -101,7 +105,13 @@ async function handlePresentSeed(aq: ActiveQuery) {
 }
 
 async function createSwarm(): Promise<number> {
-  const swarmInstance = new Hyperswarm({ keyPair: await getSeed() });
+  const privateKey = await getKey();
+  const swarmInstance = new Hyperswarm({
+    keyPair: {
+      publicKey: ed25519.getPublicKey(privateKey),
+      secretKey: privateKey,
+    },
+  });
   const id = getSwarmId();
   swarmInstances.set(id, swarmInstance);
 
@@ -206,7 +216,7 @@ async function handleWriteSocketEvent(aq: ActiveQuery) {
 
   if (!message) {
     aq.reject("empty message");
-    return false;
+    return;
   }
 
   await socket.mutex?.waitForUnlock();
@@ -447,7 +457,7 @@ async function handleCreateProtomuxChannel(aq: ActiveQuery) {
     }
     return (...args: any) => {
       args = args.filter(
-        (item: any) => item?.constructor.name.toLowerCase() !== "channel"
+        (item: any) => item?.constructor.name.toLowerCase() !== "channel",
       );
 
       if (name === "destroy") {
@@ -559,7 +569,7 @@ async function handleCreateProtomuxMessage(aq: ActiveQuery) {
     };
   };
 
-  aq.setReceiveUpdate?.((data) => {
+  aq.setReceiveUpdate?.((data: any) => {
     if (data.action === "send") {
       message.send(...data.args);
     }
@@ -571,7 +581,7 @@ async function handleCreateProtomuxMessage(aq: ActiveQuery) {
   if (data.onmessage) {
     data.onmessage = (...args: any) => {
       args = args.filter(
-        (item: any) => item?.constructor.name.toLowerCase() !== "channel"
+        (item: any) => item?.constructor.name.toLowerCase() !== "channel",
       );
       aq.sendUpdate({
         action: "onmessage",

tsconfig.build.json

@@ -1,14 +0,0 @@
-{
-	"compilerOptions": {
-		"target": "esnext",
-		"module": "esnext",
-		"moduleResolution": "node",
-		"allowSyntheticDefaultImports": true,
-		"declaration": true,
-		"outDir": "./dist-build",
-		"strict": true,
-        "esModuleInterop": true
-	},
-	"include": ["src-build"],
-	"exclude": ["node_modules", "**/__tests__/*"]
-}

tsconfig.json

@@ -1,12 +0,0 @@
-{
-	"compilerOptions": {
-		"target": "esnext",
-		"declaration": true,
-		"moduleResolution": "node",
-		"outDir": "./build",
-		"strict": true,
-        "allowSyntheticDefaultImports": true
-    },
-	"include": ["src"],
-	"exclude": ["node_modules", "**/__tests__/*"]
-}