From ba4c6dcb9552eeb7cf4b87e31d2f878b1eb83198 Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Fri, 1 Sep 2023 18:59:06 -0400 Subject: [PATCH] feat: add initial support to query the network for a registry entry for a module --- npm-shrinkwrap.json | 214 ++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + src/coreModules.ts | 46 ++++++++++ src/message.ts | 2 +- src/queries.ts | 97 ++++++++++++++++---- 5 files changed, 344 insertions(+), 16 deletions(-) create mode 100644 src/coreModules.ts diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 01c86d4..4bdb51a 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -11,6 +11,7 @@ "@lumeweb/libkernel": "0.1.0-develop.35" }, "devDependencies": { + "@lumeweb/libs5": "^0.1.0-develop.25", "@lumeweb/presetter-kernel-module-preset": "^0.1.0-develop.43", "binconv": "^0.2.0", "presetter": "*", @@ -1727,6 +1728,31 @@ "web-streams-polyfill": "^3.2.1" } }, + "node_modules/@lumeweb/libs5": { + "version": "0.1.0-develop.25", + "resolved": "https://registry.npmjs.org/@lumeweb/libs5/-/libs5-0.1.0-develop.25.tgz", + "integrity": "sha512-kqvMrAU3fZLFwAqVwv0btu7dx0wBN/XkCGV9LivCoUz2DIT0T63W1Xv4LinpqDIZmbiXNmYt0CVQAhfe53b2UA==", + "dev": true, + "dependencies": { + "@noble/curves": "^1.1.0", + "@noble/hashes": "^1.3.1", + "detect-node": "^2.1.0", + "level": "^8.0.0", + "multiformats": "^12.0.1", + "p-defer": "^4.0.0", + "ws": "^8.13.0" + } + }, + "node_modules/@lumeweb/libs5/node_modules/multiformats": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.0.tgz", + "integrity": "sha512-/qTOKKnU8nwcVURjRcS+UN0QYgdS5BPZzY10Aiciu2SqncyCVMGV8KtD83EBFmsuJDsSEmT4sGvzcTkCoMw0sQ==", + "dev": true, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/@lumeweb/libweb": { "version": "0.2.0-develop.30", "resolved": "https://registry.npmjs.org/@lumeweb/libweb/-/libweb-0.2.0-develop.30.tgz", @@ -3469,6 +3495,24 @@ "node": ">=6.5" } }, + "node_modules/abstract-level": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/abstract-level/-/abstract-level-1.0.3.tgz", + "integrity": "sha512-t6jv+xHy+VYwc4xqZMn2Pa9DjcdzvzZmQGRjTFc8spIbRGHgBrEKbPq+rYXc7CCo0lxgYvSgKVg9qZAhpVQSjA==", + "dev": true, + "dependencies": { + "buffer": "^6.0.3", + "catering": "^2.1.0", + "is-buffer": "^2.0.5", + "level-supports": "^4.0.0", + "level-transcoder": "^1.0.1", + "module-error": "^1.0.1", + "queue-microtask": "^1.2.3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/acorn": { "version": "8.9.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.9.0.tgz", @@ -4002,6 +4046,18 @@ "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==", "dev": true }, + "node_modules/browser-level": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/browser-level/-/browser-level-1.0.1.tgz", + "integrity": "sha512-XECYKJ+Dbzw0lbydyQuJzwNXtOpbMSq737qxJN11sIRTErOMShvDpbzTlgju7orJKvx4epULolZAuJGLzCmWRQ==", + "dev": true, + "dependencies": { + "abstract-level": "^1.0.2", + "catering": "^2.1.1", + "module-error": "^1.0.2", + "run-parallel-limit": "^1.1.0" + } + }, "node_modules/browser-resolve": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/browser-resolve/-/browser-resolve-2.0.0.tgz", @@ -4446,6 +4502,15 @@ "cdl": "bin/cdl.js" } }, + "node_modules/catering": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/catering/-/catering-2.1.1.tgz", + "integrity": "sha512-K7Qy8O9p76sL3/3m7/zLKbRkyOlSZAgzEaLhyj2mXS8PsCud2Eo4hAb8aLtZqHh0QGqLcb9dlJSu6lHRVENm1w==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -4548,6 +4613,23 @@ "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==", "peer": true }, + "node_modules/classic-level": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/classic-level/-/classic-level-1.3.0.tgz", + "integrity": "sha512-iwFAJQYtqRTRM0F6L8h4JCt00ZSGdOyqh7yVrhhjrOpFhmBjNlRUey64MCiyo6UmQHMJ+No3c81nujPv+n9yrg==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "abstract-level": "^1.0.2", + "catering": "^2.1.0", + "module-error": "^1.0.1", + "napi-macros": "^2.2.2", + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/clean-stack": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", @@ -8281,6 +8363,29 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "engines": { + "node": ">=4" + } + }, "node_modules/is-builtin-module": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz", @@ -9594,6 +9699,45 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/level": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/level/-/level-8.0.0.tgz", + "integrity": "sha512-ypf0jjAk2BWI33yzEaaotpq7fkOPALKAgDBxggO6Q9HGX2MRXn0wbP1Jn/tJv1gtL867+YOjOB49WaUF3UoJNQ==", + "dev": true, + "dependencies": { + "browser-level": "^1.0.1", + "classic-level": "^1.2.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/level" + } + }, + "node_modules/level-supports": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-4.0.1.tgz", + "integrity": "sha512-PbXpve8rKeNcZ9C1mUicC9auIYFyGpkV9/i6g76tLgANwWhtG2v7I4xNBUlkn3lE2/dZF3Pi0ygYGtLc4RXXdA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/level-transcoder": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/level-transcoder/-/level-transcoder-1.0.1.tgz", + "integrity": "sha512-t7bFwFtsQeD8cl8NIoQ2iwxA0CL/9IFw7/9gAjOonH0PWTTiRfY7Hq+Ejbsxh86tXobDQ6IOiddjNYIfOBs06w==", + "dev": true, + "dependencies": { + "buffer": "^6.0.3", + "module-error": "^1.0.1" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -10838,6 +10982,15 @@ "node": ">=0.10.0" } }, + "node_modules/module-error": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/module-error/-/module-error-1.0.2.tgz", + "integrity": "sha512-0yuvsqSCv8LbaOKhnsQ/T5JhyFlCYLPXK3U2sgV10zoKQwzs/MyfuQUOZQ1V/6OCOJsK/TRgNVrPuPDqtdMFtA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -10888,6 +11041,12 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/napi-macros": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.2.2.tgz", + "integrity": "sha512-hmEVtAGYzVQpCKdbQea4skABsdXW4RUh5t5mJ2zzqowJS2OyXZTU1KhDVFhx+NlWZ4ap9mqR9TcDO3LTTttd+g==", + "dev": true + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -10985,6 +11144,17 @@ "node": "^12.13 || ^14.13 || >=16" } }, + "node_modules/node-gyp-build": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.1.tgz", + "integrity": "sha512-24vnklJmyRS8ViBNI8KbtK/r/DmXQMRiOMXTNz2nrTnAYUwjmEEbnnpB/+kt+yWRv73bPsSPRFddrcIbAxSiMQ==", + "dev": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-gyp/node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -16258,6 +16428,29 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/run-parallel-limit": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/run-parallel-limit/-/run-parallel-limit-1.1.0.tgz", + "integrity": "sha512-jJA7irRNM91jaKc3Hcl1npHsFLOXOoTkPCUL1JEa1R82O2miplXXRaGdjW/KM/98YQWDhJLiSs793CnXfblJUw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, "node_modules/rxjs": { "version": "7.8.1", "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", @@ -19421,6 +19614,27 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/ws": { + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", + "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", + "dev": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/package.json b/package.json index 86f39f8..606afa6 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "url": "gitea@git.lumeweb.com:LumeWeb/kernel.git" }, "devDependencies": { + "@lumeweb/libs5": "^0.1.0-develop.25", "@lumeweb/presetter-kernel-module-preset": "^0.1.0-develop.43", "binconv": "^0.2.0", "presetter": "*", diff --git a/src/coreModules.ts b/src/coreModules.ts new file mode 100644 index 0000000..2b447c3 --- /dev/null +++ b/src/coreModules.ts @@ -0,0 +1,46 @@ +import { modules } from "#queries.js"; +import { internalModuleCall } from "./queries.js"; +import { SignedRegistryEntry } from "@lumeweb/libs5"; +import { base58btc } from "multiformats/bases/base58"; + +const CORE_MODULES = { + swarm: "", + peerDiscoveryRegistry: "", + ircPeerDiscovery: "", + s5: "", +}; + +export default CORE_MODULES; + +export async function networkReady() { + for (const module of [ + CORE_MODULES.peerDiscoveryRegistry, + CORE_MODULES.ircPeerDiscovery, + CORE_MODULES.swarm, + ]) { + if (!moduleLoaded(module)) { + return false; + } + } + + const resolvers = await internalModuleCall( + CORE_MODULES.peerDiscoveryRegistry, + "getRelays", + ); + + return resolvers.length > 0; +} + +function moduleLoaded(module: string) { + return module in modules; +} + +export async function resolveModuleRegistryEntry(pubkey: string) { + const signedEntry = (await internalModuleCall( + CORE_MODULES.s5, + "getRegistryEntry", + { pubkey }, + )) as SignedRegistryEntry; + + return base58btc.encode(signedEntry.data); +} diff --git a/src/message.ts b/src/message.ts index fcaf43c..e5de5ce 100644 --- a/src/message.ts +++ b/src/message.ts @@ -93,7 +93,7 @@ export const handleIncomingMessage = function (event: any) { } else { domain = new URL(event.origin).hostname; } - handleModuleCall(event, event.source, domain, false); + handleModuleCall(event, event.source, domain, false, false); return; } if (event.data.method === "queryUpdate") { diff --git a/src/queries.ts b/src/queries.ts index cc70edc..4b6fd40 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -19,6 +19,8 @@ import { import type { moduleQuery, presentKeyData } from "@lumeweb/libkernel/module"; import { readableStreamToUint8Array } from "binconv"; import { getSavedRegistryEntry } from "./registry.js"; +import { defer } from "@lumeweb/libkernel/module"; +import { networkReady, resolveModuleRegistryEntry } from "./coreModules.js"; // WorkerLaunchFn is the type signature of the function that launches the // worker to set up for processing a query. @@ -40,6 +42,8 @@ interface Module { // OpenQuery holds all of the information necessary for managing an open query. interface OpenQuery { isWorker: boolean; + isInternal: boolean; + internalHandler?: (message: OpenQueryResponse) => void; domain: string; source: any; dest: Worker; @@ -47,13 +51,19 @@ interface OpenQuery { origin: string; } +interface OpenQueryResponse { + nonce: string; + method: string; + data: any; + err?: any; +} + // Define the stateful variables for managing the modules. We track the set of // queries that are in progress, the set of skapps that are known to the // kernel, the set of modules that we've downloaded, and the set of modules // that are actively being downloaded. let queriesNonce = 0; -const queries = {} as any; -const skapps = {} as any; +const queries = {} as { [module: string]: OpenQuery }; const modules = {} as any; const modulesLoading = {} as any; @@ -178,7 +188,7 @@ function handleWorkerMessage(event: MessageEvent, mod: Module, worker: Worker) { // Handle a call from the worker to another module. if (event.data.method === "moduleCall") { - handleModuleCall(event, worker, mod.domain, true); + handleModuleCall(event, worker, mod.domain, true, false); return; } @@ -290,11 +300,12 @@ function launchWorker(mod: Module): [Worker, Err] { // handleModuleCall will handle a callModule message sent to the kernel from an // extension or webpage. -function handleModuleCall( +async function handleModuleCall( event: MessageEvent, messagePortal: any, callerDomain: string, isWorker: boolean, + isInternal: false | ((message: OpenQueryResponse) => void), ) { if (!("data" in event.data) || !("module" in event.data.data)) { logErr( @@ -345,7 +356,7 @@ function handleModuleCall( if (typeof event.data.data.method !== "string") { logErr( "moduleCall", - "recieved moduleCall with malformed method", + "received moduleCall with malformed method", event.data, ); respondErr( @@ -365,7 +376,7 @@ function handleModuleCall( event, messagePortal, isWorker, - "presentSeed is a priviledged method, only root is allowed to use it", + "presentSeed is a privileged method, only root is allowed to use it", ); return; } @@ -384,8 +395,7 @@ function handleModuleCall( let finalModule = moduleDomain; // Can change with overrides. if (isRegistryEntry) { - finalModule = getSavedRegistryEntry(moduleDomain); - if (!finalModule) { + const registryFail = () => { logErr("moduleCall", "received moduleCall with no known registry entry"); respondErr( event, @@ -393,7 +403,23 @@ function handleModuleCall( isWorker, "registry entry for module is not found", ); - return; + }; + + finalModule = getSavedRegistryEntry(moduleDomain); + if (!finalModule) { + if (!(await networkReady())) { + registryFail(); + return; + } + let resolvedModule; + + try { + resolvedModule = await resolveModuleRegistryEntry(finalModule); + } catch (e) { + registryFail(); + return; + } + finalModule = resolvedModule; } } // Define a helper function to create a new query to the module. It will @@ -420,6 +446,8 @@ function handleModuleCall( queriesNonce = queriesNonce + 1; queries[nonce] = { isWorker, + isInternal: !!isInternal, + internalHandler: isInternal ?? undefined, domain: callerDomain, source: messagePortal, dest: worker, @@ -614,11 +642,14 @@ function handleModuleResponse( // We are sending either a response message or a responseUpdate message, // all other possibilities have been handled. - const sourceIsWorker = queries[event.data.nonce].isWorker; - const sourceNonce = queries[event.data.nonce].nonce; - const source = queries[event.data.nonce].source; - const origin = queries[event.data.nonce].origin; - const msg: any = { + const query = queries[event.data.nonce]; + const sourceIsWorker = query.isWorker; + const sourceIsInternal = query.isInternal; + const internalHandler = query.internalHandler; + const sourceNonce = query.nonce; + const source = query.source; + const origin = query.origin; + const msg: OpenQueryResponse = { nonce: sourceNonce, method: event.data.method, data: event.data.data, @@ -629,8 +660,11 @@ function handleModuleResponse( msg["err"] = event.data.err; delete queries[event.data.nonce]; } - if (sourceIsWorker === true) { + + if (sourceIsWorker) { source.postMessage(msg); + } else if (sourceIsInternal) { + internalHandler?.(msg); } else { source.postMessage(msg, origin); } @@ -656,6 +690,39 @@ function handleQueryUpdate(event: MessageEvent) { }); } +export async function internalModuleCall( + module: string, + method: string, + params = {}, +): Promise { + const callDefer = defer(); + handleModuleCall( + { + data: { + data: { + module, + method, + data: params, + nonce: 0, + }, + }, + origin: "root", + } as any, + undefined, + "root", + false, + (message) => { + if (message.err) { + callDefer.reject(message.err); + return; + } + callDefer.resolve(message.data.data); + }, + ); + + return callDefer.promise; +} + export { Module, handleModuleCall,