feat: add initial support to query the network for a registry entry for a module

npm-shrinkwrap.json

@@ -11,6 +11,7 @@
                 "@lumeweb/libkernel": "0.1.0-develop.35"
             },
             "devDependencies": {
+                "@lumeweb/libs5": "^0.1.0-develop.25",
                 "@lumeweb/presetter-kernel-module-preset": "^0.1.0-develop.43",
                 "binconv": "^0.2.0",
                 "presetter": "*",
@@ -1727,6 +1728,31 @@
                 "web-streams-polyfill": "^3.2.1"
             }
         },
+        "node_modules/@lumeweb/libs5": {
+            "version": "0.1.0-develop.25",
+            "resolved": "https://registry.npmjs.org/@lumeweb/libs5/-/libs5-0.1.0-develop.25.tgz",
+            "integrity": "sha512-kqvMrAU3fZLFwAqVwv0btu7dx0wBN/XkCGV9LivCoUz2DIT0T63W1Xv4LinpqDIZmbiXNmYt0CVQAhfe53b2UA==",
+            "dev": true,
+            "dependencies": {
+                "@noble/curves": "^1.1.0",
+                "@noble/hashes": "^1.3.1",
+                "detect-node": "^2.1.0",
+                "level": "^8.0.0",
+                "multiformats": "^12.0.1",
+                "p-defer": "^4.0.0",
+                "ws": "^8.13.0"
+            }
+        },
+        "node_modules/@lumeweb/libs5/node_modules/multiformats": {
+            "version": "12.1.0",
+            "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.0.tgz",
+            "integrity": "sha512-/qTOKKnU8nwcVURjRcS+UN0QYgdS5BPZzY10Aiciu2SqncyCVMGV8KtD83EBFmsuJDsSEmT4sGvzcTkCoMw0sQ==",
+            "dev": true,
+            "engines": {
+                "node": ">=16.0.0",
+                "npm": ">=7.0.0"
+            }
+        },
         "node_modules/@lumeweb/libweb": {
             "version": "0.2.0-develop.30",
             "resolved": "https://registry.npmjs.org/@lumeweb/libweb/-/libweb-0.2.0-develop.30.tgz",
@@ -3469,6 +3495,24 @@
                 "node": ">=6.5"
             }
         },
+        "node_modules/abstract-level": {
+            "version": "1.0.3",
+            "resolved": "https://registry.npmjs.org/abstract-level/-/abstract-level-1.0.3.tgz",
+            "integrity": "sha512-t6jv+xHy+VYwc4xqZMn2Pa9DjcdzvzZmQGRjTFc8spIbRGHgBrEKbPq+rYXc7CCo0lxgYvSgKVg9qZAhpVQSjA==",
+            "dev": true,
+            "dependencies": {
+                "buffer": "^6.0.3",
+                "catering": "^2.1.0",
+                "is-buffer": "^2.0.5",
+                "level-supports": "^4.0.0",
+                "level-transcoder": "^1.0.1",
+                "module-error": "^1.0.1",
+                "queue-microtask": "^1.2.3"
+            },
+            "engines": {
+                "node": ">=12"
+            }
+        },
         "node_modules/acorn": {
             "version": "8.9.0",
             "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.9.0.tgz",
@@ -4002,6 +4046,18 @@
             "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==",
             "dev": true
         },
+        "node_modules/browser-level": {
+            "version": "1.0.1",
+            "resolved": "https://registry.npmjs.org/browser-level/-/browser-level-1.0.1.tgz",
+            "integrity": "sha512-XECYKJ+Dbzw0lbydyQuJzwNXtOpbMSq737qxJN11sIRTErOMShvDpbzTlgju7orJKvx4epULolZAuJGLzCmWRQ==",
+            "dev": true,
+            "dependencies": {
+                "abstract-level": "^1.0.2",
+                "catering": "^2.1.1",
+                "module-error": "^1.0.2",
+                "run-parallel-limit": "^1.1.0"
+            }
+        },
         "node_modules/browser-resolve": {
             "version": "2.0.0",
             "resolved": "https://registry.npmjs.org/browser-resolve/-/browser-resolve-2.0.0.tgz",
@@ -4446,6 +4502,15 @@
                 "cdl": "bin/cdl.js"
             }
         },
+        "node_modules/catering": {
+            "version": "2.1.1",
+            "resolved": "https://registry.npmjs.org/catering/-/catering-2.1.1.tgz",
+            "integrity": "sha512-K7Qy8O9p76sL3/3m7/zLKbRkyOlSZAgzEaLhyj2mXS8PsCud2Eo4hAb8aLtZqHh0QGqLcb9dlJSu6lHRVENm1w==",
+            "dev": true,
+            "engines": {
+                "node": ">=6"
+            }
+        },
         "node_modules/chalk": {
             "version": "4.1.2",
             "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -4548,6 +4613,23 @@
             "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==",
             "peer": true
         },
+        "node_modules/classic-level": {
+            "version": "1.3.0",
+            "resolved": "https://registry.npmjs.org/classic-level/-/classic-level-1.3.0.tgz",
+            "integrity": "sha512-iwFAJQYtqRTRM0F6L8h4JCt00ZSGdOyqh7yVrhhjrOpFhmBjNlRUey64MCiyo6UmQHMJ+No3c81nujPv+n9yrg==",
+            "dev": true,
+            "hasInstallScript": true,
+            "dependencies": {
+                "abstract-level": "^1.0.2",
+                "catering": "^2.1.0",
+                "module-error": "^1.0.1",
+                "napi-macros": "^2.2.2",
+                "node-gyp-build": "^4.3.0"
+            },
+            "engines": {
+                "node": ">=12"
+            }
+        },
         "node_modules/clean-stack": {
             "version": "2.2.0",
             "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
@@ -8281,6 +8363,29 @@
                 "url": "https://github.com/sponsors/ljharb"
             }
         },
+        "node_modules/is-buffer": {
+            "version": "2.0.5",
+            "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",
+            "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==",
+            "dev": true,
+            "funding": [
+                {
+                    "type": "github",
+                    "url": "https://github.com/sponsors/feross"
+                },
+                {
+                    "type": "patreon",
+                    "url": "https://www.patreon.com/feross"
+                },
+                {
+                    "type": "consulting",
+                    "url": "https://feross.org/support"
+                }
+            ],
+            "engines": {
+                "node": ">=4"
+            }
+        },
         "node_modules/is-builtin-module": {
             "version": "3.2.1",
             "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz",
@@ -9594,6 +9699,45 @@
                 "url": "https://github.com/chalk/strip-ansi?sponsor=1"
             }
         },
+        "node_modules/level": {
+            "version": "8.0.0",
+            "resolved": "https://registry.npmjs.org/level/-/level-8.0.0.tgz",
+            "integrity": "sha512-ypf0jjAk2BWI33yzEaaotpq7fkOPALKAgDBxggO6Q9HGX2MRXn0wbP1Jn/tJv1gtL867+YOjOB49WaUF3UoJNQ==",
+            "dev": true,
+            "dependencies": {
+                "browser-level": "^1.0.1",
+                "classic-level": "^1.2.0"
+            },
+            "engines": {
+                "node": ">=12"
+            },
+            "funding": {
+                "type": "opencollective",
+                "url": "https://opencollective.com/level"
+            }
+        },
+        "node_modules/level-supports": {
+            "version": "4.0.1",
+            "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-4.0.1.tgz",
+            "integrity": "sha512-PbXpve8rKeNcZ9C1mUicC9auIYFyGpkV9/i6g76tLgANwWhtG2v7I4xNBUlkn3lE2/dZF3Pi0ygYGtLc4RXXdA==",
+            "dev": true,
+            "engines": {
+                "node": ">=12"
+            }
+        },
+        "node_modules/level-transcoder": {
+            "version": "1.0.1",
+            "resolved": "https://registry.npmjs.org/level-transcoder/-/level-transcoder-1.0.1.tgz",
+            "integrity": "sha512-t7bFwFtsQeD8cl8NIoQ2iwxA0CL/9IFw7/9gAjOonH0PWTTiRfY7Hq+Ejbsxh86tXobDQ6IOiddjNYIfOBs06w==",
+            "dev": true,
+            "dependencies": {
+                "buffer": "^6.0.3",
+                "module-error": "^1.0.1"
+            },
+            "engines": {
+                "node": ">=12"
+            }
+        },
         "node_modules/leven": {
             "version": "3.1.0",
             "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz",
@@ -10838,6 +10982,15 @@
                 "node": ">=0.10.0"
             }
         },
+        "node_modules/module-error": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/module-error/-/module-error-1.0.2.tgz",
+            "integrity": "sha512-0yuvsqSCv8LbaOKhnsQ/T5JhyFlCYLPXK3U2sgV10zoKQwzs/MyfuQUOZQ1V/6OCOJsK/TRgNVrPuPDqtdMFtA==",
+            "dev": true,
+            "engines": {
+                "node": ">=10"
+            }
+        },
         "node_modules/ms": {
             "version": "2.1.2",
             "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -10888,6 +11041,12 @@
                 "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
             }
         },
+        "node_modules/napi-macros": {
+            "version": "2.2.2",
+            "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.2.2.tgz",
+            "integrity": "sha512-hmEVtAGYzVQpCKdbQea4skABsdXW4RUh5t5mJ2zzqowJS2OyXZTU1KhDVFhx+NlWZ4ap9mqR9TcDO3LTTttd+g==",
+            "dev": true
+        },
         "node_modules/natural-compare": {
             "version": "1.4.0",
             "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
@@ -10985,6 +11144,17 @@
                 "node": "^12.13 || ^14.13 || >=16"
             }
         },
+        "node_modules/node-gyp-build": {
+            "version": "4.6.1",
+            "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.1.tgz",
+            "integrity": "sha512-24vnklJmyRS8ViBNI8KbtK/r/DmXQMRiOMXTNz2nrTnAYUwjmEEbnnpB/+kt+yWRv73bPsSPRFddrcIbAxSiMQ==",
+            "dev": true,
+            "bin": {
+                "node-gyp-build": "bin.js",
+                "node-gyp-build-optional": "optional.js",
+                "node-gyp-build-test": "build-test.js"
+            }
+        },
         "node_modules/node-gyp/node_modules/abbrev": {
             "version": "1.1.1",
             "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@@ -16258,6 +16428,29 @@
                 "queue-microtask": "^1.2.2"
             }
         },
+        "node_modules/run-parallel-limit": {
+            "version": "1.1.0",
+            "resolved": "https://registry.npmjs.org/run-parallel-limit/-/run-parallel-limit-1.1.0.tgz",
+            "integrity": "sha512-jJA7irRNM91jaKc3Hcl1npHsFLOXOoTkPCUL1JEa1R82O2miplXXRaGdjW/KM/98YQWDhJLiSs793CnXfblJUw==",
+            "dev": true,
+            "funding": [
+                {
+                    "type": "github",
+                    "url": "https://github.com/sponsors/feross"
+                },
+                {
+                    "type": "patreon",
+                    "url": "https://www.patreon.com/feross"
+                },
+                {
+                    "type": "consulting",
+                    "url": "https://feross.org/support"
+                }
+            ],
+            "dependencies": {
+                "queue-microtask": "^1.2.2"
+            }
+        },
         "node_modules/rxjs": {
             "version": "7.8.1",
             "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz",
@@ -19421,6 +19614,27 @@
                 "url": "https://github.com/sponsors/sindresorhus"
             }
         },
+        "node_modules/ws": {
+            "version": "8.13.0",
+            "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz",
+            "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==",
+            "dev": true,
+            "engines": {
+                "node": ">=10.0.0"
+            },
+            "peerDependencies": {
+                "bufferutil": "^4.0.1",
+                "utf-8-validate": ">=5.0.2"
+            },
+            "peerDependenciesMeta": {
+                "bufferutil": {
+                    "optional": true
+                },
+                "utf-8-validate": {
+                    "optional": true
+                }
+            }
+        },
         "node_modules/xtend": {
             "version": "4.0.2",
             "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",

package.json

@@ -7,6 +7,7 @@
         "url": "gitea@git.lumeweb.com:LumeWeb/kernel.git"
     },
     "devDependencies": {
+        "@lumeweb/libs5": "^0.1.0-develop.25",
         "@lumeweb/presetter-kernel-module-preset": "^0.1.0-develop.43",
         "binconv": "^0.2.0",
         "presetter": "*",

src/coreModules.ts

@@ -0,0 +1,46 @@
+import { modules } from "#queries.js";
+import { internalModuleCall } from "./queries.js";
+import { SignedRegistryEntry } from "@lumeweb/libs5";
+import { base58btc } from "multiformats/bases/base58";
+
+const CORE_MODULES = {
+  swarm: "",
+  peerDiscoveryRegistry: "",
+  ircPeerDiscovery: "",
+  s5: "",
+};
+
+export default CORE_MODULES;
+
+export async function networkReady() {
+  for (const module of [
+    CORE_MODULES.peerDiscoveryRegistry,
+    CORE_MODULES.ircPeerDiscovery,
+    CORE_MODULES.swarm,
+  ]) {
+    if (!moduleLoaded(module)) {
+      return false;
+    }
+  }
+
+  const resolvers = await internalModuleCall(
+    CORE_MODULES.peerDiscoveryRegistry,
+    "getRelays",
+  );
+
+  return resolvers.length > 0;
+}
+
+function moduleLoaded(module: string) {
+  return module in modules;
+}
+
+export async function resolveModuleRegistryEntry(pubkey: string) {
+  const signedEntry = (await internalModuleCall(
+    CORE_MODULES.s5,
+    "getRegistryEntry",
+    { pubkey },
+  )) as SignedRegistryEntry;
+
+  return base58btc.encode(signedEntry.data);
+}

src/message.ts

@@ -93,7 +93,7 @@ export const handleIncomingMessage = function (event: any) {
     } else {
       domain = new URL(event.origin).hostname;
     }
-    handleModuleCall(event, event.source, domain, false);
+    handleModuleCall(event, event.source, domain, false, false);
     return;
   }
   if (event.data.method === "queryUpdate") {

src/queries.ts

@@ -19,6 +19,8 @@ import {
 import type { moduleQuery, presentKeyData } from "@lumeweb/libkernel/module";
 import { readableStreamToUint8Array } from "binconv";
 import { getSavedRegistryEntry } from "./registry.js";
+import { defer } from "@lumeweb/libkernel/module";
+import { networkReady, resolveModuleRegistryEntry } from "./coreModules.js";
 
 // WorkerLaunchFn is the type signature of the function that launches the
 // worker to set up for processing a query.
@@ -40,6 +42,8 @@ interface Module {
 // OpenQuery holds all of the information necessary for managing an open query.
 interface OpenQuery {
   isWorker: boolean;
+  isInternal: boolean;
+  internalHandler?: (message: OpenQueryResponse) => void;
   domain: string;
   source: any;
   dest: Worker;
@@ -47,13 +51,19 @@ interface OpenQuery {
   origin: string;
 }
 
+interface OpenQueryResponse {
+  nonce: string;
+  method: string;
+  data: any;
+  err?: any;
+}
+
 // Define the stateful variables for managing the modules. We track the set of
 // queries that are in progress, the set of skapps that are known to the
 // kernel, the set of modules that we've downloaded, and the set of modules
 // that are actively being downloaded.
 let queriesNonce = 0;
-const queries = {} as any;
+const queries = {} as { [module: string]: OpenQuery };
-const skapps = {} as any;
 const modules = {} as any;
 const modulesLoading = {} as any;
 
@@ -178,7 +188,7 @@ function handleWorkerMessage(event: MessageEvent, mod: Module, worker: Worker) {
 
   // Handle a call from the worker to another module.
   if (event.data.method === "moduleCall") {
-    handleModuleCall(event, worker, mod.domain, true);
+    handleModuleCall(event, worker, mod.domain, true, false);
     return;
   }
 
@@ -290,11 +300,12 @@ function launchWorker(mod: Module): [Worker, Err] {
 
 // handleModuleCall will handle a callModule message sent to the kernel from an
 // extension or webpage.
-function handleModuleCall(
+async function handleModuleCall(
   event: MessageEvent,
   messagePortal: any,
   callerDomain: string,
   isWorker: boolean,
+  isInternal: false | ((message: OpenQueryResponse) => void),
 ) {
   if (!("data" in event.data) || !("module" in event.data.data)) {
     logErr(
@@ -345,7 +356,7 @@ function handleModuleCall(
   if (typeof event.data.data.method !== "string") {
     logErr(
       "moduleCall",
-      "recieved moduleCall with malformed method",
+      "received moduleCall with malformed method",
       event.data,
     );
     respondErr(
@@ -365,7 +376,7 @@ function handleModuleCall(
       event,
       messagePortal,
       isWorker,
-      "presentSeed is a priviledged method, only root is allowed to use it",
+      "presentSeed is a privileged method, only root is allowed to use it",
     );
     return;
   }
@@ -384,8 +395,7 @@ function handleModuleCall(
   let finalModule = moduleDomain; // Can change with overrides.
 
   if (isRegistryEntry) {
-    finalModule = getSavedRegistryEntry(moduleDomain);
+    const registryFail = () => {
-    if (!finalModule) {
       logErr("moduleCall", "received moduleCall with no known registry entry");
       respondErr(
         event,
@@ -393,8 +403,24 @@ function handleModuleCall(
         isWorker,
         "registry entry for module is not found",
       );
+    };
+
+    finalModule = getSavedRegistryEntry(moduleDomain);
+    if (!finalModule) {
+      if (!(await networkReady())) {
+        registryFail();
         return;
       }
+      let resolvedModule;
+
+      try {
+        resolvedModule = await resolveModuleRegistryEntry(finalModule);
+      } catch (e) {
+        registryFail();
+        return;
+      }
+      finalModule = resolvedModule;
+    }
   }
   // Define a helper function to create a new query to the module. It will
   // both open a query on the module and also send an update message to the
@@ -420,6 +446,8 @@ function handleModuleCall(
     queriesNonce = queriesNonce + 1;
     queries[nonce] = {
       isWorker,
+      isInternal: !!isInternal,
+      internalHandler: isInternal ?? undefined,
       domain: callerDomain,
       source: messagePortal,
       dest: worker,
@@ -614,11 +642,14 @@ function handleModuleResponse(
 
   // We are sending either a response message or a responseUpdate message,
   // all other possibilities have been handled.
-  const sourceIsWorker = queries[event.data.nonce].isWorker;
+  const query = queries[event.data.nonce];
-  const sourceNonce = queries[event.data.nonce].nonce;
+  const sourceIsWorker = query.isWorker;
-  const source = queries[event.data.nonce].source;
+  const sourceIsInternal = query.isInternal;
-  const origin = queries[event.data.nonce].origin;
+  const internalHandler = query.internalHandler;
-  const msg: any = {
+  const sourceNonce = query.nonce;
+  const source = query.source;
+  const origin = query.origin;
+  const msg: OpenQueryResponse = {
     nonce: sourceNonce,
     method: event.data.method,
     data: event.data.data,
@@ -629,8 +660,11 @@ function handleModuleResponse(
     msg["err"] = event.data.err;
     delete queries[event.data.nonce];
   }
-  if (sourceIsWorker === true) {
+
+  if (sourceIsWorker) {
     source.postMessage(msg);
+  } else if (sourceIsInternal) {
+    internalHandler?.(msg);
   } else {
     source.postMessage(msg, origin);
   }
@@ -656,6 +690,39 @@ function handleQueryUpdate(event: MessageEvent) {
   });
 }
 
+export async function internalModuleCall(
+  module: string,
+  method: string,
+  params = {},
+): Promise<any> {
+  const callDefer = defer();
+  handleModuleCall(
+    {
+      data: {
+        data: {
+          module,
+          method,
+          data: params,
+          nonce: 0,
+        },
+      },
+      origin: "root",
+    } as any,
+    undefined,
+    "root",
+    false,
+    (message) => {
+      if (message.err) {
+        callDefer.reject(message.err);
+        return;
+      }
+      callDefer.resolve(message.data.data);
+    },
+  );
+
+  return callDefer.promise;
+}
+
 export {
   Module,
   handleModuleCall,