feat: add cached modules support with leveldb/indexeddb

src/modules.ts

@@ -1,5 +1,8 @@
 import { internalModuleCall, modules } from "./queries.js";
 import { CID, SignedRegistryEntry } from "@lumeweb/libs5";
+import { Level } from "level";
+
+let moduleStore: Level<string, Uint8Array>;
 
 const CORE_MODULES = {
   swarm: "zdiLmwHCC15afFNLYzzT2DVV7m27SrBde7oXHdSzAe95GpFZXzdpatUN6b",
@@ -39,3 +42,22 @@ export async function resolveModuleRegistryEntry(module: string) {
 
   return CID.fromRegistry(signedEntry.data).toString();
 }
+
+async function initStore() {
+  if (moduleStore) {
+    return;
+  }
+
+  const db = new Level<string, Uint8Array>("kernel-module-store");
+  await db.open();
+
+  moduleStore = db;
+}
+
+export async function store() {
+  if (!moduleStore) {
+    await initStore();
+  }
+
+  return moduleStore;
+}

src/queries.ts

@@ -6,6 +6,7 @@ import {
   addContextToErr,
   bufToB64,
   encodeU64,
+  equalBytes,
   Err,
   objAsString,
   sha512,
@@ -16,7 +17,12 @@ import type { moduleQuery, presentKeyData } from "@lumeweb/libkernel/module";
 import { defer } from "@lumeweb/libkernel/module";
 import { readableStreamToUint8Array } from "binconv";
 import { getSavedRegistryEntry } from "./registry.js";
-import { networkReady, resolveModuleRegistryEntry } from "./coreModules.js";
+import {
+  networkReady,
+  resolveModuleRegistryEntry,
+  store as moduleStore,
+} from "./modules.js";
+import { blake3 } from "@noble/hashes/blake3";
 
 // WorkerLaunchFn is the type signature of the function that launches the
 // worker to set up for processing a query.
@@ -214,17 +220,9 @@ function handleWorkerMessage(event: MessageEvent, mod: Module, worker: Worker) {
 // createModule will create a module from the provided worker code and domain.
 // This call does not launch the worker, that should be done separately.
 async function createModule(
-  workerCode: Uint8Array | ReadableStream,
+  workerCode: Uint8Array,
   domain: string,
 ): Promise<[Module | null, Err]> {
-  if (workerCode instanceof ReadableStream) {
-    try {
-      workerCode = await readableStreamToUint8Array(workerCode);
-    } catch (e) {
-      return [null, e];
-    }
-  }
-
   // Generate the URL for the worker code.
   const url = URL.createObjectURL(new Blob([workerCode]));
 
@@ -526,18 +524,33 @@ async function handleModuleCall(
   // Fetch the module in a background thread, and launch the query once the
   // module is available.
   modulesLoading[moduleDomain] = new Promise(async (resolve) => {
-    // TODO: Check localStorage for the module.
+    let cachedModule: Uint8Array | undefined;
-
-    let moduleData;
 
     try {
-      moduleData = await downloadSmallObject(finalModule);
+      cachedModule = await (await moduleStore()).get(finalModule);
-    } catch (e) {
+    } catch {}
-      const err = addContextToErr(e, "unable to load module");
+
-      respondErr(event, messagePortal, isWorker, isInternal, err);
+    let moduleData: ReadableStream<Uint8Array> | Uint8Array | undefined;
-      resolve(err);
+
-      delete modulesLoading[moduleDomain];
+    if (cachedModule) {
-      return;
+      const hash = CID.decode(finalModule).hash.hashBytes;
+      if (!equalBytes(hash, blake3(cachedModule))) {
+        logErr("corrupt module found in store: ", finalModule);
+      } else {
+        moduleData = cachedModule;
+      }
+    }
+
+    if (!moduleData) {
+      try {
+        moduleData = await downloadSmallObject(finalModule);
+      } catch (e) {
+        const err = addContextToErr(e, "unable to load module");
+        respondErr(event, messagePortal, isWorker, isInternal, err);
+        resolve(err);
+        delete modulesLoading[moduleDomain];
+        return;
+      }
     }
 
     // The call to download the skylink is async. That means it's possible that
@@ -559,9 +572,20 @@ async function handleModuleCall(
       return;
     }
 
-    // TODO: Save the result to localStorage. Can't do that until
+    if (moduleData instanceof ReadableStream) {
-    // subscriptions are in place so that localStorage can sync
+      try {
-    // with any updates from the remote module.
+        moduleData = await readableStreamToUint8Array(moduleData);
+      } catch (e) {
+        respondErr(event, messagePortal, isWorker, isInternal, e);
+        resolve(e);
+        delete modulesLoading[moduleDomain];
+        return;
+      }
+    }
+
+    if (!cachedModule) {
+      await (await moduleStore()).put(finalModule, moduleData);
+    }
 
     // Create a new module.
     const [mod, errCM] = await createModule(moduleData, moduleDomain);