LumeWeb
/
kernel-swarm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: update to new sdks and portal

This commit is contained in:
Derrick Hammer 2023-07-01 04:21:09 -04:00
parent 87c7d1e02d
commit cc929a903a
Signed by: pcfreak30
GPG Key ID: C997C339BE476FF2
13 changed files with 22018 additions and 338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,13 @@
name: Build/Publish
on:
push:
branches:
- master
- develop
- develop-*
jobs:
main:
uses: lumeweb/github-node-deploy-workflow/.github/workflows/main.yml@master
secrets: inherit

8
.presetterrc.json Normal file
View File

@ -0,0 +1,8 @@
{
"preset": [
"@lumeweb/presetter-kernel-module-preset"
],
"config": {
"official": true
}
}

14
build.js
View File

@ -1,14 +0,0 @@
import esbuild from "esbuild";
esbuild.buildSync({
entryPoints: ["src/index.ts"],
outfile: "dist/index.js",
format: "iife",
bundle: true,
legalComments: "external",
// minify: true
define: {
global: "self",
},
inject: ["process.js", "polyfill.js"],
});

21904
npm-shrinkwrap.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

70
package.json
View File

@ -4,64 +4,36 @@
"name": "Hammer Technologies LLC",
"email": "contact@lumeweb.com"
},
"repository": {
"type": "git",
"url": "gitea@git.lumeweb.com:LumeWeb/kernel-swarm.git"
},
"scripts": {
"test": "jest",
"format": "prettier -w src",
"build-script": "tsc --project tsconfig.build.json && mv dist-build/build.js dist-build/build.mjs",
"compile": "npm run build-script && node build.js",
"build": "npm run compile && node ./dist-build/build.mjs dev"
"prepare": "presetter bootstrap",
"build": "run build",
"semantic-release": "semantic-release"
},
"type": "module",
"dependencies": {
"@lumeweb/hyperswarm-web": "git+https://git.lumeweb.com/LumeWeb/hyperswarm-web.git",
"@lumeweb/rpc": "git+https://git.lumeweb.com/LumeWeb/rpc.git",
"@noble/ed25519": "^1.7.3",
"@peculiar/webcrypto": "git+https://git.lumeweb.com/LumeWeb/webcrypto.git",
"@lumeweb/hyperswarm-web": "0.0.2-develop.3",
"@lumeweb/libkernel": "^0.1.0-develop.7",
"@lumeweb/rpc": "0.0.2-develop.1",
"@noble/curves": "^1.1.0",
"@peculiar/webcrypto": "^1.4.3",
"async-mutex": "^0.4.0",
"b4a": "^1.6.3",
"b4a": "^1.6.4",
"eventemitter2": "^6.4.9",
"hyperswarm": "^4.4.0",
"libkmodule": "^0.2.53",
"libskynet": "^0.0.62",
"noise-handshake": "^3.0.2",
"hyperswarm": "^4.5.1",
"noise-handshake": "^3.0.3",
"p-defer": "^4.0.0",
"protomux": "git+https://git.lumeweb.com/LumeWeb/kernel-protomux.git",
"randombytes": "github:LumeWeb/randombytes-browser"
"protomux": "^3.5.0"
},
"devDependencies": {
"@rollup/plugin-commonjs": "^22.0.2",
"@rollup/plugin-node-resolve": "^13.3.0",
"@rollup/plugin-typescript": "^8.5.0",
"@screamingvoid/sodium-universal": "^0.1.1",
"@scure/bip39": "^1.2.0",
"@skynetlabs/skynet-nodejs": "^2.9.0",
"@lumeweb/presetter-kernel-module-preset": "^0.1.0-develop.20",
"@types/b4a": "^1.6.0",
"@types/jest": "^28.1.8",
"@types/read": "^0.0.29",
"buffer": "^6.0.3",
"cli-progress": "^3.12.0",
"crypto-browserify": "^3.12.0",
"esbuild": "^0.14.54",
"inspectpack": "^4.7.1",
"jest": "^28.1.3",
"jest-puppeteer": "^6.2.0",
"libskynetnode": "^0.1.4",
"prettier": "^2.8.7",
"process": "^0.11.10",
"puppeteer": "^15.5.0",
"random-number-csprng": "^1.0.2",
"read": "^1.0.7",
"rollup": "^2.79.1",
"rollup-plugin-polyfill-node": "^0.9.0",
"stream-browserify": "^3.0.0",
"ts-loader": "^9.4.2",
"typescript": "^4.9.5",
"webpack": "^5.79.0",
"webpack-cli": "^4.10.0"
"presetter": "*"
},
"browser": {
"sodium-universal": "@screamingvoid/sodium-universal",
"crypto": "crypto-browserify",
"stream": "stream-browserify"
}
"version": "",
"readme": "ERROR: No README data found!",
"_id": "@lumeweb/kernel-swarm@"
}

45
patches/protomux+3.5.0.patch Normal file
View File

@ -0,0 +1,45 @@
diff --git a/node_modules/protomux/index.js b/node_modules/protomux/index.js
index 07bf0ad..c8db59c 100644
--- a/node_modules/protomux/index.js
+++ b/node_modules/protomux/index.js
@@ -195,34 +195,34 @@ class Channel {
type,
encoding,
onmessage,
- recv (state, session) {
+ async recv (state, session) {
session._track(m.onmessage(encoding.decode(state), session))
},
- send (m, session = s) {
+ async send (m, session = s) {
if (session.closed === true) return false
const mux = session._mux
const state = { buffer: null, start: 0, end: typeLen }
if (mux._batch !== null) {
- encoding.preencode(state, m)
+ await encoding.preencode(state, m)
state.buffer = mux._alloc(state.end)
c.uint.encode(state, type)
- encoding.encode(state, m)
+ await encoding.encode(state, m)
mux._pushBatch(session._localId, state.buffer)
return true
}
c.uint.preencode(state, session._localId)
- encoding.preencode(state, m)
+ await encoding.preencode(state, m)
state.buffer = mux._alloc(state.end)
c.uint.encode(state, session._localId)
c.uint.encode(state, type)
- encoding.encode(state, m)
+ await encoding.encode(state, m)
mux.drained = mux.stream.write(state.buffer)

11
polyfill.js
View File

@ -1,11 +0,0 @@
import { Crypto } from "@peculiar/webcrypto";
let globalCrypto = self.crypto;
if (!globalCrypto.subtle) {
let subtleCrypto = new Crypto().subtle;
Object.defineProperty(globalCrypto, "subtle", {
get() {
return subtleCrypto;
},
});
}

3
process.js
View File

@ -1,3 +0,0 @@
export const process = require('process');

218
src-build/build.ts
View File

@ -1,218 +0,0 @@
// This is the standard build script for a kernel module.
import * as fs from "fs";
import read from "read";
import * as bip39 from "@scure/bip39";
import { wordlist } from "@scure/bip39/wordlists/english.js";
//@ts-ignore
import { SkynetClient } from "@skynetlabs/skynet-nodejs";
// Helper variables to make it easier to return empty values alongside errors.
const nu8 = new Uint8Array(0);
const nkp = {
publicKey: nu8,
secretKey: nu8,
};
// readFile is a wrapper for fs.readFileSync that handles the try-catch for the
// caller.
function readFile(fileName: string): [string, string | null] {
try {
let data = fs.readFileSync(fileName, "utf8");
return [data, null];
} catch (err) {
return ["", "unable to read file: " + JSON.stringify(err)];
}
}
// readFileBinary is a wrapper for fs.readFileSync that handles the try-catch
// for the caller.
function readFileBinary(fileName: string): [Uint8Array, string | null] {
try {
let data = fs.readFileSync(fileName, null);
return [data, null];
} catch (err) {
return [nu8, "unable to read file: " + JSON.stringify(err)];
}
}
// writeFile is a wrapper for fs.writeFileSync which handles the try-catch in a
// non-exception way.
function writeFile(fileName: string, fileData: string): string | null {
try {
fs.writeFileSync(fileName, fileData);
return null;
} catch (err) {
return "unable to write file: " + JSON.stringify(err);
}
}
// handlePass handles all portions of the script that occur after the password
// has been requested. If no password needs to be requested, handlePass will be
// called with a null input. We need to structure the code this way because the
// password reader is async and we can only access the password when using a
// callback.
function handlePass(password: string) {
try {
// If we are running prod and the seed file does not exist, we
// need to confirm the password and also warn the user to use a
// secure password.
if (!fs.existsSync(seedFile) && process.argv[2] === "prod") {
// The file does not exist, we need to confirm the
// password.
console.log();
console.log(
"No production entry found for module. Creating new production module..."
);
console.log(
"If someone can guess the password, they can push arbitrary changes to your module."
);
console.log("Please use a secure password.");
console.log();
read(
{ prompt: "Confirm Password: ", silent: true },
function (err: any, confirmPassword: string) {
if (err) {
console.error("unable to fetch password:", err);
process.exit(1);
}
if (password !== confirmPassword) {
console.error("passwords do not match");
process.exit(1);
}
handlePassConfirm(moduleSalt, password);
}
);
} else {
// If the seed file does exist, or if we are using dev,
// there's no need to confirm the password but we do
// need to pass the logic off to the handlePassConfirm
// callback.
handlePassConfirm(moduleSalt, password);
}
} catch (err) {
console.error("Unable to read seedFile:", err);
process.exit(1);
}
}
// handlePassConfirm handles the full script after the confirmation password
// has been provided. If not confirmation password is needed, this function
// will be called anyway using the unconfirmed password as input.
function handlePassConfirm(seed: string, password: string) {
// Create the seedFile if it does not exist. For dev we just save the
// seed to disk outright, because this is a dev build and therefore not
// security sensitive. Also the dev seed does not get pushed to the
// github repo.
//
// For prod, we use the seed to create a new seed (called the shield)
// which allows us to verify that the developer has provided the right
// password when deploying the module. The shield does get pushed to
// the github repo so that the production module is the same on all
// devices.
if (!fs.existsSync(seedFile) && process.argv[2] !== "prod") {
// Generate the seed phrase and write it to the file.
let seedPhrase = bip39.generateMnemonic(wordlist);
let errWF = writeFile(seedFile, seedPhrase);
if (errWF !== null) {
console.error("unable to write file:", errWF);
process.exit(1);
}
} else if (!fs.existsSync(seedFile) && process.argv[2] === "prod") {
// Generate the seed phrase.
let seedPhrase = bip39.generateMnemonic(wordlist);
// Write the registry link to the file.
}
// Load or verify the seed. If this is prod, the password is used to
// create and verify the seed. If this is dev, we just load the seed
// with no password.
let seedPhrase: string;
let registryLink: string;
if (process.argv[2] === "prod") {
// Generate the seed phrase from the password.
seedPhrase = bip39.generateMnemonic(wordlist);
} else {
let [sp, errRF] = readFile(seedFile);
if (errRF !== null) {
console.error("unable to read seed phrase for dev command from disk");
process.exit(1);
}
seedPhrase = sp;
}
let metadata = {
Filename: "index.js",
};
const client = new SkynetClient("https://web3portal.com");
client
.uploadFile("dist/index.js")
.then((result: any) => {
console.log("Immutable Link for kernel:", result);
})
.catch((err: any) => {
console.error("unable to upload file", err);
process.exit(1);
});
}
// Add a newline for readability.
console.log();
// Check for a 'dev' or 'prod' input to the script.
if (process.argv.length !== 3) {
console.error("need to provide either 'dev' or 'prod' as an input");
process.exit(1);
}
// Create the build folder if it does not exist.
if (!fs.existsSync("build")) {
fs.mkdirSync("build");
}
// Determine the seed file.
let seedFile: string;
if (process.argv[2] === "prod") {
seedFile = "module-skylink";
} else if (process.argv[2] === "dev") {
seedFile = "build/dev-seed";
} else {
console.error("need to provide either 'dev' or 'prod' as an input");
process.exit(1);
}
// If doing a prod deployment, check whether the salt file exists. If it does
// not, create it.
let moduleSalt: string;
if (!fs.existsSync(".module-salt")) {
moduleSalt = bip39.generateMnemonic(wordlist);
let errWF = writeFile(".module-salt", moduleSalt);
if (errWF !== null) {
console.error("unable to write module salt file:", errWF);
process.exit(1);
}
} else {
let [ms, errRF] = readFile(".module-salt");
if (errRF !== null) {
console.error("unable to read moduleSalt");
process.exit(1);
}
ms = ms.replace(/\n$/, "");
moduleSalt = ms;
}
// Need to get a password if this is a prod build.
if (process.argv[2] === "prod") {
read(
{ prompt: "Password: ", silent: true },
function (err: any, password: string) {
if (err) {
console.error("unable to fetch password:", err);
process.exit(1);
}
handlePass(password);
}
);
} else {
handlePass("");
}

4
src/addr.ts
View File

@ -22,7 +22,7 @@ export function pubKeyToIpv6(publicKey: Uint8Array) {
addr[prefix.length] = ones;
addr.set(nodeId, prefix.length + 1);
const result = [];
const result: string[] = [];
for (let i = 0; i < 8; i++) {
const num1 = addr[i * 2].toString(16).padStart(2, "0");
const num2 = addr[i * 2 + 1].toString(16).padStart(2, "0");
@ -46,7 +46,7 @@ function getLeadingOnes(buf: Uint8Array) {
}
function getTruncatedNodeID(buf: Uint8Array) {
const result = [];
const result: number[] = [];
let done = false;
let bits = 0;
let nBits = 0;

40
src/index.ts
View File

@ -1,14 +1,18 @@
// @ts-ignore
import Hyperswarm from "@lumeweb/hyperswarm-web";
import type { ActiveQuery } from "libkmodule";
import { addHandler, getSeed, handleMessage } from "libkmodule";
import { handlePresentSeed as handlePresentSeedModule } from "libkmodule/dist/seed.js";
import type { ActiveQuery } from "@lumeweb/libkernel/module";
import {
addHandler,
getKey,
handleMessage,
handlePresentKey as handlePresentKeyModule,
} from "@lumeweb/libkernel/module";
import { Buffer } from "buffer";
import * as ed from "@noble/ed25519";
import { ed25519 } from "@noble/curves/ed25519";
import b4a from "b4a";
import { pubKeyToIpv6 } from "./addr.js";
import { EventEmitter2 as EventEmitter } from "eventemitter2";
import { logErr } from "libkmodule/dist";
import { logErr } from "@lumeweb/libkernel";
// @ts-ignore
import Protomux from "protomux";
import defer, { DeferredPromise } from "p-defer";
@ -54,7 +58,7 @@ const getSocketId = idFactory();
const getChannelId = idFactory();
const getMessageId = idFactory();
addHandler("presentSeed", handlePresentSeed);
addHandler("presentKey", handlePresentKey);
addHandler("join", handleJoin);
addHandler("getPeerByPubkey", handleGetPeerByPubkey);
@ -83,12 +87,12 @@ addHandler("createProtomuxMessage", handleCreateProtomuxMessage, {
});
addHandler("createSwarm", handleCreateSwarm);
async function handlePresentSeed(aq: ActiveQuery) {
const pubkey = await ed.getPublicKey(aq.callerInput.rootKey);
handlePresentSeedModule({
async function handlePresentKey(aq: ActiveQuery) {
const pubkey = ed25519.getPublicKey(aq.callerInput.rootKey);
handlePresentKeyModule({
callerInput: {
seed: {
publicKey: await ed.getPublicKey(aq.callerInput.rootKey),
publicKey: ed25519.getPublicKey(aq.callerInput.rootKey),
secretKey: b4a.concat([aq.callerInput.rootKey, pubkey]),
},
},
@ -101,7 +105,13 @@ async function handlePresentSeed(aq: ActiveQuery) {
}
async function createSwarm(): Promise<number> {
const swarmInstance = new Hyperswarm({ keyPair: await getSeed() });
const privateKey = await getKey();
const swarmInstance = new Hyperswarm({
keyPair: {
publicKey: ed25519.getPublicKey(privateKey),
secretKey: privateKey,
},
});
const id = getSwarmId();
swarmInstances.set(id, swarmInstance);
@ -206,7 +216,7 @@ async function handleWriteSocketEvent(aq: ActiveQuery) {
if (!message) {
aq.reject("empty message");
return false;
return;
}
await socket.mutex?.waitForUnlock();
@ -447,7 +457,7 @@ async function handleCreateProtomuxChannel(aq: ActiveQuery) {
}
return (...args: any) => {
args = args.filter(
(item: any) => item?.constructor.name.toLowerCase() !== "channel"
(item: any) => item?.constructor.name.toLowerCase() !== "channel",
);
if (name === "destroy") {
@ -559,7 +569,7 @@ async function handleCreateProtomuxMessage(aq: ActiveQuery) {
};
};
aq.setReceiveUpdate?.((data) => {
aq.setReceiveUpdate?.((data: any) => {
if (data.action === "send") {
message.send(...data.args);
}
@ -571,7 +581,7 @@ async function handleCreateProtomuxMessage(aq: ActiveQuery) {
if (data.onmessage) {
data.onmessage = (...args: any) => {
args = args.filter(
(item: any) => item?.constructor.name.toLowerCase() !== "channel"
(item: any) => item?.constructor.name.toLowerCase() !== "channel",
);
aq.sendUpdate({
action: "onmessage",

14
tsconfig.build.json
View File

@ -1,14 +0,0 @@
{
"compilerOptions": {
"target": "esnext",
"module": "esnext",
"moduleResolution": "node",
"allowSyntheticDefaultImports": true,
"declaration": true,
"outDir": "./dist-build",
"strict": true,
"esModuleInterop": true
},
"include": ["src-build"],
"exclude": ["node_modules", "**/__tests__/*"]
}

12
tsconfig.json
View File

@ -1,12 +0,0 @@
{
"compilerOptions": {
"target": "esnext",
"declaration": true,
"moduleResolution": "node",
"outDir": "./build",
"strict": true,
"allowSyntheticDefaultImports": true
},
"include": ["src"],
"exclude": ["node_modules", "**/__tests__/*"]
}