Compare commits

..

No commits in common. "v0.1.0-develop.17" and "v0.1.0-develop.16" have entirely different histories.

5 changed files with 26 additions and 79 deletions

View File

@ -1,10 +1,3 @@
# [0.1.0-develop.17](https://git.lumeweb.com/LumeWeb/kernel/compare/v0.1.0-develop.16...v0.1.0-develop.17) (2023-09-13)
### Features
* add cached modules support with leveldb/indexeddb ([6e148c1](https://git.lumeweb.com/LumeWeb/kernel/commit/6e148c1e0a761ac1b253074c150b61d356285560))
# [0.1.0-develop.16](https://git.lumeweb.com/LumeWeb/kernel/compare/v0.1.0-develop.15...v0.1.0-develop.16) (2023-09-11) # [0.1.0-develop.16](https://git.lumeweb.com/LumeWeb/kernel/compare/v0.1.0-develop.15...v0.1.0-develop.16) (2023-09-11)

4
npm-shrinkwrap.json generated
View File

@ -1,12 +1,12 @@
{ {
"name": "@lumeweb/kernel", "name": "@lumeweb/kernel",
"version": "0.1.0-develop.17", "version": "0.1.0-develop.16",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@lumeweb/kernel", "name": "@lumeweb/kernel",
"version": "0.1.0-develop.17", "version": "0.1.0-develop.16",
"dependencies": { "dependencies": {
"@lumeweb/libkernel": "0.1.0-develop.53" "@lumeweb/libkernel": "0.1.0-develop.53"
}, },

View File

@ -1,6 +1,6 @@
{ {
"name": "@lumeweb/kernel", "name": "@lumeweb/kernel",
"version": "0.1.0-develop.17", "version": "0.1.0-develop.16",
"type": "module", "type": "module",
"repository": { "repository": {
"type": "git", "type": "git",

View File

@ -1,8 +1,5 @@
import { internalModuleCall, modules } from "./queries.js"; import { internalModuleCall, modules } from "./queries.js";
import { CID, SignedRegistryEntry } from "@lumeweb/libs5"; import { CID, SignedRegistryEntry } from "@lumeweb/libs5";
import { Level } from "level";
let moduleStore: Level<string, Uint8Array>;
const CORE_MODULES = { const CORE_MODULES = {
swarm: "zdiLmwHCC15afFNLYzzT2DVV7m27SrBde7oXHdSzAe95GpFZXzdpatUN6b", swarm: "zdiLmwHCC15afFNLYzzT2DVV7m27SrBde7oXHdSzAe95GpFZXzdpatUN6b",
@ -42,22 +39,3 @@ export async function resolveModuleRegistryEntry(module: string) {
return CID.fromRegistry(signedEntry.data).toString(); return CID.fromRegistry(signedEntry.data).toString();
} }
async function initStore() {
if (moduleStore) {
return;
}
const db = new Level<string, Uint8Array>("kernel-module-store");
await db.open();
moduleStore = db;
}
export async function store() {
if (!moduleStore) {
await initStore();
}
return moduleStore;
}

View File

@ -6,7 +6,6 @@ import {
addContextToErr, addContextToErr,
bufToB64, bufToB64,
encodeU64, encodeU64,
equalBytes,
Err, Err,
objAsString, objAsString,
sha512, sha512,
@ -17,12 +16,7 @@ import type { moduleQuery, presentKeyData } from "@lumeweb/libkernel/module";
import { defer } from "@lumeweb/libkernel/module"; import { defer } from "@lumeweb/libkernel/module";
import { readableStreamToUint8Array } from "binconv"; import { readableStreamToUint8Array } from "binconv";
import { getSavedRegistryEntry } from "./registry.js"; import { getSavedRegistryEntry } from "./registry.js";
import { import { networkReady, resolveModuleRegistryEntry } from "./coreModules.js";
networkReady,
resolveModuleRegistryEntry,
store as moduleStore,
} from "./modules.js";
import { blake3 } from "@noble/hashes/blake3";
// WorkerLaunchFn is the type signature of the function that launches the // WorkerLaunchFn is the type signature of the function that launches the
// worker to set up for processing a query. // worker to set up for processing a query.
@ -220,9 +214,17 @@ function handleWorkerMessage(event: MessageEvent, mod: Module, worker: Worker) {
// createModule will create a module from the provided worker code and domain. // createModule will create a module from the provided worker code and domain.
// This call does not launch the worker, that should be done separately. // This call does not launch the worker, that should be done separately.
async function createModule( async function createModule(
workerCode: Uint8Array, workerCode: Uint8Array | ReadableStream,
domain: string, domain: string,
): Promise<[Module | null, Err]> { ): Promise<[Module | null, Err]> {
if (workerCode instanceof ReadableStream) {
try {
workerCode = await readableStreamToUint8Array(workerCode);
} catch (e) {
return [null, e];
}
}
// Generate the URL for the worker code. // Generate the URL for the worker code.
const url = URL.createObjectURL(new Blob([workerCode])); const url = URL.createObjectURL(new Blob([workerCode]));
@ -524,33 +526,18 @@ async function handleModuleCall(
// Fetch the module in a background thread, and launch the query once the // Fetch the module in a background thread, and launch the query once the
// module is available. // module is available.
modulesLoading[moduleDomain] = new Promise(async (resolve) => { modulesLoading[moduleDomain] = new Promise(async (resolve) => {
let cachedModule: Uint8Array | undefined; // TODO: Check localStorage for the module.
let moduleData;
try { try {
cachedModule = await (await moduleStore()).get(finalModule); moduleData = await downloadSmallObject(finalModule);
} catch {} } catch (e) {
const err = addContextToErr(e, "unable to load module");
let moduleData: ReadableStream<Uint8Array> | Uint8Array | undefined; respondErr(event, messagePortal, isWorker, isInternal, err);
resolve(err);
if (cachedModule) { delete modulesLoading[moduleDomain];
const hash = CID.decode(finalModule).hash.hashBytes; return;
if (!equalBytes(hash, blake3(cachedModule))) {
logErr("corrupt module found in store: ", finalModule);
} else {
moduleData = cachedModule;
}
}
if (!moduleData) {
try {
moduleData = await downloadSmallObject(finalModule);
} catch (e) {
const err = addContextToErr(e, "unable to load module");
respondErr(event, messagePortal, isWorker, isInternal, err);
resolve(err);
delete modulesLoading[moduleDomain];
return;
}
} }
// The call to download the skylink is async. That means it's possible that // The call to download the skylink is async. That means it's possible that
@ -572,20 +559,9 @@ async function handleModuleCall(
return; return;
} }
if (moduleData instanceof ReadableStream) { // TODO: Save the result to localStorage. Can't do that until
try { // subscriptions are in place so that localStorage can sync
moduleData = await readableStreamToUint8Array(moduleData); // with any updates from the remote module.
} catch (e) {
respondErr(event, messagePortal, isWorker, isInternal, e);
resolve(e);
delete modulesLoading[moduleDomain];
return;
}
}
if (!cachedModule) {
await (await moduleStore()).put(finalModule, moduleData);
}
// Create a new module. // Create a new module.
const [mod, errCM] = await createModule(moduleData, moduleDomain); const [mod, errCM] = await createModule(moduleData, moduleDomain);