chore(release): 0.1.0-develop.46 [skip ci]

# [0.1.0-develop.46](https://git.lumeweb.com/LumeWeb/libkernel/compare/v0.1.0-develop.45...v0.1.0-develop.46) (2023-09-08)

### Bug Fixes

* export types since we are no longer storing them in libweb ([4abc3ed](4abc3ed9c4))
* update imports to be local ([631fde4](631fde4868))
* use Err from types ([bb1229e](bb1229eeb2))
* use types and util locally ([469f02b](469f02b69f))

CHANGELOG.md

@@ -1,3 +1,13 @@
+# [0.1.0-develop.46](https://git.lumeweb.com/LumeWeb/libkernel/compare/v0.1.0-develop.45...v0.1.0-develop.46) (2023-09-08)
+
+
+### Bug Fixes
+
+* export types since we are no longer storing them in libweb ([4abc3ed](https://git.lumeweb.com/LumeWeb/libkernel/commit/4abc3ed9c4f0c6a7662d2eda4b5c71c39e8bc0fe))
+* update imports to be local ([631fde4](https://git.lumeweb.com/LumeWeb/libkernel/commit/631fde4868e4fe5e0b8f7c779d8d079093a70c3b))
+* use Err from types ([bb1229e](https://git.lumeweb.com/LumeWeb/libkernel/commit/bb1229eeb2429d5a71e797283116d2fcfb5eb732))
+* use types and util locally ([469f02b](https://git.lumeweb.com/LumeWeb/libkernel/commit/469f02b69fc65b4caea8136587e14e095c6e6b6e))
+
 # [0.1.0-develop.45](https://git.lumeweb.com/LumeWeb/libkernel/compare/v0.1.0-develop.44...v0.1.0-develop.45) (2023-09-04)
 
 # [0.1.0-develop.44](https://git.lumeweb.com/LumeWeb/libkernel/compare/v0.1.0-develop.43...v0.1.0-develop.44) (2023-09-04)

npm-shrinkwrap.json

@@ -1,14 +1,14 @@
 {
     "name": "@lumeweb/libkernel",
-    "version": "0.1.0-develop.45",
+    "version": "0.1.0-develop.46",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "@lumeweb/libkernel",
-            "version": "0.1.0-develop.45",
+            "version": "0.1.0-develop.46",
             "dependencies": {
-                "@lumeweb/libweb": "0.2.0-develop.50",
+                "@lumeweb/libweb": "0.2.0-develop.51",
                 "emittery": "^1.0.1",
                 "p-defer": "^4.0.0"
             },
@@ -1672,11 +1672,11 @@
             }
         },
         "node_modules/@lumeweb/libportal": {
-            "version": "0.2.0-develop.24",
+            "version": "0.2.0-develop.31",
-            "resolved": "https://registry.npmjs.org/@lumeweb/libportal/-/libportal-0.2.0-develop.24.tgz",
+            "resolved": "https://registry.npmjs.org/@lumeweb/libportal/-/libportal-0.2.0-develop.31.tgz",
-            "integrity": "sha512-EEsHjgmQOkzpU16mXXB6+O4Xv/4aQ+ApcWaShlCJQnHi4EFxuPDPJP6uIoBPhIcgXMm05ZF1YSs10ognAEXlcw==",
+            "integrity": "sha512-6/8AGJTnDsnsTOaFDokmt4cDIX9cIvA61PcKWxLZ6sMgUz8LM7HV6f/FT/RgZBI9dBcXuXlRyBVcwZze47QOIw==",
             "dependencies": {
-                "@lumeweb/libs5": "^0.1.0-develop.45",
+                "@lumeweb/libs5": "^0.1.0-develop.52",
                 "@noble/curves": "^1.1.0",
                 "@noble/hashes": "^1.3.1",
                 "detect-node": "^2.1.0",
@@ -1706,9 +1706,9 @@
             }
         },
         "node_modules/@lumeweb/libs5": {
-            "version": "0.1.0-develop.45",
+            "version": "0.1.0-develop.52",
-            "resolved": "https://registry.npmjs.org/@lumeweb/libs5/-/libs5-0.1.0-develop.45.tgz",
+            "resolved": "https://registry.npmjs.org/@lumeweb/libs5/-/libs5-0.1.0-develop.52.tgz",
-            "integrity": "sha512-D2q8WhCrus/IWcvOmNE1BLPFCdwUzz0wp4hXzYggIRggui0tOBGKpLgo80ZLnjH9LUqaevAkwi9emJ6DrdDgyg==",
+            "integrity": "sha512-COQQCQJrhJRApAty1jkhhM4SIjRZviWgZrnZtjLq+7WABYiDHpSXaaiXuLTt7k5Nras1NqEAaMNOa/DHi8R67Q==",
             "dependencies": {
                 "@noble/curves": "^1.1.0",
                 "@noble/hashes": "^1.3.1",
@@ -1720,22 +1720,23 @@
             }
         },
         "node_modules/@lumeweb/libs5/node_modules/multiformats": {
-            "version": "12.1.0",
+            "version": "12.1.1",
-            "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.0.tgz",
+            "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.1.tgz",
-            "integrity": "sha512-/qTOKKnU8nwcVURjRcS+UN0QYgdS5BPZzY10Aiciu2SqncyCVMGV8KtD83EBFmsuJDsSEmT4sGvzcTkCoMw0sQ==",
+            "integrity": "sha512-GBSToTmri2vJYs8wqcZQ8kB21dCaeTOzHTIAlr8J06C1eL6UbzqURXFZ5Fl0EYm9GAFz1IlYY8SxGOs9G9NJRg==",
             "engines": {
                 "node": ">=16.0.0",
                 "npm": ">=7.0.0"
             }
         },
         "node_modules/@lumeweb/libweb": {
-            "version": "0.2.0-develop.50",
+            "version": "0.2.0-develop.51",
-            "resolved": "https://registry.npmjs.org/@lumeweb/libweb/-/libweb-0.2.0-develop.50.tgz",
+            "resolved": "https://registry.npmjs.org/@lumeweb/libweb/-/libweb-0.2.0-develop.51.tgz",
-            "integrity": "sha512-1i9/0Oob5l4Jqd/DcFhVy0afLFuUBMNtzBRVLecKOJdkJDdbuJUev9Tmo30BccnxOa9h5D0inPlkiVr0giPfeg==",
+            "integrity": "sha512-MaCPswp5oKQxjFNhDQzW9ad5kAPYY1Hesi6ZnJYYqkMfiz3UKEUARQTaQTe0Fd7ed4vNqlQeIDhOmuHoOciH/A==",
             "dependencies": {
                 "@lumeweb/community-portals": "^0.1.0-develop.6",
-                "@lumeweb/libportal": "0.2.0-develop.24",
+                "@lumeweb/libportal": "0.2.0-develop.31",
                 "@lumeweb/node-library-preset": "0.2.7",
+                "@noble/ciphers": "^0.3.0",
                 "@noble/curves": "^1.1.0",
                 "@noble/hashes": "^1.3.1",
                 "binconv": "^0.2.0"
@@ -1753,6 +1754,14 @@
                 "semantic-release": "^21.0.5"
             }
         },
+        "node_modules/@noble/ciphers": {
+            "version": "0.3.0",
+            "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-0.3.0.tgz",
+            "integrity": "sha512-ldbrnOjmNRwFdXcTM6uXDcxpMIFrbzAWNnpBPp4oTJTFF0XByGD6vf45WrehZGXRQTRVV+Zm8YP+EgEf+e4cWA==",
+            "funding": {
+                "url": "https://paulmillr.com/funding/"
+            }
+        },
         "node_modules/@noble/curves": {
             "version": "1.2.0",
             "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.2.0.tgz",
@@ -18030,9 +18039,9 @@
             }
         },
         "node_modules/ws": {
-            "version": "8.13.0",
+            "version": "8.14.0",
-            "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz",
+            "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.0.tgz",
-            "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==",
+            "integrity": "sha512-WR0RJE9Ehsio6U4TuM+LmunEsjQ5ncHlw4sn9ihD6RoJKZrVyH9FWV3dmnwu8B2aNib1OvG2X6adUCyFpQyWcg==",
             "engines": {
                 "node": ">=10.0.0"
             },

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@lumeweb/libkernel",
-    "version": "0.1.0-develop.45",
+    "version": "0.1.0-develop.46",
     "main": "lib/index.js",
     "type": "module",
     "types": "lib/index.d.ts",
@@ -34,7 +34,7 @@
         "semantic-release": "semantic-release"
     },
     "dependencies": {
-        "@lumeweb/libweb": "0.2.0-develop.50",
+        "@lumeweb/libweb": "0.2.0-develop.51",
         "emittery": "^1.0.1",
         "p-defer": "^4.0.0"
     },

src/errTracker.ts

@@ -0,0 +1,101 @@
+// errTracker.ts defines an 'ErrTracker' type which keeps track of historical
+// errors. When the number of errors gets too large, it randomly starts pruning
+// errors. It always keeps 250 of the most recent errors, and then keeps up to
+// 500 historic errors, where the first few errors after runtime are always
+// kept, and the ones in the middle are increasingly likely to be omitted from
+// the history.
+
+import { Err } from "./types.js";
+
+// MAX_ERRORS defines the maximum number of errors that will be held in the
+// HistoricErr object.
+const MAX_ERRORS = 1000;
+
+// HistoricErr is a wrapper that adds a date to the Err type.
+interface HistoricErr {
+  err: Err;
+  date: Date;
+}
+
+// ErrTracker keeps track of errors that have happened, randomly dropping
+// errors to prevent the tracker from using too much memory if there happen to
+// be a large number of errors.
+interface ErrTracker {
+  recentErrs: HistoricErr[];
+  oldErrs: HistoricErr[];
+
+  addErr: (err: Err) => void;
+  viewErrs: () => HistoricErr[];
+}
+
+// newErrTracker returns an ErrTracker object that is ready to have errors
+// added to it.
+function newErrTracker(): ErrTracker {
+  const et: ErrTracker = {
+    recentErrs: [],
+    oldErrs: [],
+
+    addErr: function (err: Err): void {
+      addHistoricErr(et, err);
+    },
+    viewErrs: function (): HistoricErr[] {
+      return viewErrs(et);
+    },
+  };
+  return et;
+}
+
+// addHistoricErr is a function that will add an error to a set of historic
+// errors. It uses randomness to prune errors once the error object is too
+// large.
+function addHistoricErr(et: ErrTracker, err: Err): void {
+  // Add this error to the set of most recent errors.
+  et.recentErrs.push({
+    err,
+    date: new Date(),
+  });
+
+  // Determine whether some of the most recent errors need to be moved into
+  // logTermErrs. If the length of the mostRecentErrs is not at least half of
+  // the MAX_ERRORS, we don't need to do anything.
+  if (et.recentErrs.length < MAX_ERRORS / 2) {
+    return;
+  }
+
+  // Iterate through the recentErrs. For the first half of the recentErrs, we
+  // will use randomness to either toss them or move them to oldErrs. The
+  // second half of the recentErrs will be kept as the new recentErrs array.
+  const newRecentErrs : HistoricErr[] = [];
+  for (let i = 0; i < et.recentErrs.length; i++) {
+    // If we are in the second half of the array, add the element to
+    // newRecentErrs.
+    if (i > et.recentErrs.length / 2) {
+      newRecentErrs.push(et.recentErrs[i]);
+      continue;
+    }
+
+    // We are in the first half of the array, use a random number to add the
+    // error oldErrs probabilistically.
+    const rand = Math.random();
+    const target = et.oldErrs.length / (MAX_ERRORS / 2);
+    if (rand > target || et.oldErrs.length < 25) {
+      et.oldErrs.push(et.recentErrs[i]);
+    }
+  }
+  et.recentErrs = newRecentErrs;
+}
+
+// viewErrs returns the list of errors that have been retained by the
+// HistoricErr object.
+function viewErrs(et: ErrTracker): HistoricErr[] {
+  const finalErrs: HistoricErr[] = [];
+  for (let i = 0; i < et.oldErrs.length; i++) {
+    finalErrs.push(et.oldErrs[i]);
+  }
+  for (let i = 0; i < et.recentErrs.length; i++) {
+    finalErrs.push(et.recentErrs[i]);
+  }
+  return finalErrs;
+}
+
+export { ErrTracker, HistoricErr, newErrTracker };

src/index.ts

@@ -1,16 +1,4 @@
 export {
-  DataFn,
-  Err,
-  ErrTuple,
-  addContextToErr,
-  objAsString,
-  bufToHex,
-  hexToBuf,
-  b64ToBuf,
-  bufToB64,
-  bufToStr,
-  decodeU64,
-  encodeU64,
   ed25519,
   sha512,
   ensureBytes,
@@ -25,3 +13,5 @@ export {
   logErr,
   getNetworkModuleStatus,
 } from "./api.js";
+export * from "./util.js";
+export * from "./types.js";

src/kernel/auth.ts

@@ -5,7 +5,7 @@ import {
   loginPromise,
   logoutPromise,
 } from "./queries.js";
-import { Err } from "@lumeweb/libweb";
+import { Err } from "#types.js";
 
 // There are 5 stages of auth.
 //

src/kernel/queries.ts

@@ -1,5 +1,6 @@
 import { log, logErr } from "./log.js";
-import { DataFn, Err, ErrTuple, bufToB64, encodeU64 } from "@lumeweb/libweb";
+import { DataFn, Err, ErrTuple } from "#types.js";
+import { bufToB64, encodeU64 } from "#util.js";
 
 // queryResolve is the 'resolve' value of a promise that returns an ErrTuple.
 // It gets called when a query sends a 'response' message.

src/kernel/query/version.ts

@@ -1,5 +1,5 @@
 import { newKernelQuery } from "../queries.js";
-import { Err } from "@lumeweb/libweb";
+import { Err } from "#types.js";
 
 // kernelVersion will fetch the version number of the kernel. If successful,
 // the returned value will be an object containing a field 'version' with a

src/module/client.ts

@@ -1,6 +1,6 @@
 import Emittery from "emittery";
 import { callModule, connectModule, log, logErr } from "../api.js";
-import { DataFn, ErrTuple } from "@lumeweb/libweb";
+import { DataFn, ErrTuple } from "#types.js";
 
 export type callModuleBound = (method: string, data?: any) => Promise<ErrTuple>;
 export type connectModuleBound = (

src/module/log.ts

@@ -1,4 +1,4 @@
-import { objAsString } from "@lumeweb/libweb";
+import { objAsString } from "#util.js";
 
 // logHelper is a helper function that runs the code for both log and logErr.
 // It takes a boolean indiciating whether the log should be an error, and then

src/module/messages.ts

@@ -9,7 +9,8 @@ import {
   handleResponseUpdate,
 } from "./queries.js";
 import { handlePresentKey } from "./key.js";
-import { DataFn, ErrFn, addContextToErr, objAsString } from "@lumeweb/libweb";
+import type { DataFn, ErrFn } from "#types.js";
+import { addContextToErr, objAsString } from "#util.js";
 
 // handlerFn takes an ActiveQuery as input and has no return value. The return
 // is expected to come in the form of calling aq.accept or aq.reject.

src/module/queries.ts

@@ -1,6 +1,6 @@
 import { log, logErr } from "./log.js";
-import { DataFn } from "./messages.js";
+import type { DataFn, ErrTuple } from "#types.js";
-import { ErrTuple, objAsString } from "@lumeweb/libweb";
+import { objAsString } from "#util.js";
 
 // queryResolve defines the function that gets called to resolve a query. It's
 // the 'resolve' field of a promise that returns a tuple containing some data

src/types.ts

@@ -0,0 +1,87 @@
+// DataFn can take any object as input and has no return value. The input is
+// allowed to be undefined.
+type DataFn = (data?: any) => void;
+
+// Err is an error type that is either a string or a null. If the value is
+// null, that indicates that there was no error. If the value is a string, it
+// indicates that there was an error and the string contains the error message.
+//
+// The skynet libraries prefer this error type to the standard Error type
+// because many times skynet libraries need to pass errors over postMessage,
+// and the 'Error' type is not able to be sent over postMessage.
+type Err = string | null;
+
+// ErrFn must take an error message as input. The input is not allowed to be
+// undefined or null, there must be an error.
+type ErrFn = (errMsg: string) => void;
+
+// ErrTuple is a type that pairs a 'data' field with an 'err' field. Skynet
+// libraries typically prefer returning ErrTuples to throwing or rejecting,
+// because it allows upstream code to avoid the try/catch/throw pattern. Though
+// the pattern is much celebrated in javascript, it encourages relaxed error
+// handling, and often makes error handling much more difficult because the try
+// and the catch are in different scopes.
+//
+// Most of the Skynet core libraries do not have any `throws` anywhere in their
+// API.
+//
+// Typically, an ErrTuple will have only one field filled out. If data is
+// returned, the err should be 'null'. If an error is returned, the data field
+// should generally be empty. Callers are expected to check the error before
+// they access any part of the data field.
+type ErrTuple<T = any> = [data: T, err: Err];
+
+// KernelAuthStatus is the structure of a message that gets sent by the kernel
+// containing its auth status. Auth occurs in 5 stages.
+//
+// Stage 0; no auth updates
+// Stage 1: bootloader is loaded, user is not yet logged in
+// Stage 2: bootloader is loaded, user is logged in
+// Stage 3: kernel is loaded, user is logged in
+// Stage 4: kernel is loaded, user is logging out (refresh iminent)
+//
+// 'kernelLoaded' is initially set to "not yet" and will be updated when the
+// kernel has loaded. If it is set to "success", it means the kernel loaded
+// without issues. If it is set to anything else, it means that there was an
+// error, and the new value is the error.
+//
+// 'kernelLoaded' will not be changed until 'loginComplete' has been set to
+// true. 'loginComplete' can be set to true immediately if the user is already
+// logged in.
+//
+// 'logoutComplete' can be set to 'true' at any point, which indicates that the
+// auth cycle needs to reset.
+interface KernelAuthStatus {
+  loginComplete: boolean;
+  kernelLoaded: "not yet" | "success" | string;
+  logoutComplete: boolean;
+}
+
+interface Portal {
+  id: string;
+  name: string;
+  url: string;
+}
+
+// RequestOverrideResponse defines the type that the kernel returns as a
+// response to a requestOverride call.
+interface RequestOverrideResponse {
+  override: boolean;
+  headers?: any; // TODO: I don't know how to do an array of types.
+  body?: Uint8Array;
+}
+
+export interface KeyPair {
+  publicKey: Uint8Array;
+  privateKey: Uint8Array;
+}
+
+export {
+  DataFn,
+  ErrFn,
+  Err,
+  ErrTuple,
+  KernelAuthStatus,
+  RequestOverrideResponse,
+  Portal,
+};

src/util.ts

@@ -0,0 +1,123 @@
+// addContextToErr is a helper function that standardizes the formatting of
+// adding context to an error.
+//
+// NOTE: To protect against accidental situations where an Error type or some
+// other type is provided instead of a string, we wrap both of the inputs with
+// objAsString before returning them. This prevents runtime failures.
+import { Err } from "#types.js";
+
+const MAX_UINT_64 = 18446744073709551615n;
+
+function addContextToErr(err: any, context: string): string {
+  if (err === null || err === undefined) {
+    err = "[no error provided]";
+  }
+  return objAsString(context) + ": " + objAsString(err);
+}
+
+// objAsString will try to return the provided object as a string. If the
+// object is already a string, it will be returned without modification. If the
+// object is an 'Error', the message of the error will be returned. If the object
+// has a toString method, the toString method will be called and the result
+// will be returned. If the object is null or undefined, a special string will
+// be returned indicating that the undefined/null object cannot be converted to
+// a string. In all other cases, JSON.stringify is used. If JSON.stringify
+// throws an exception, a message "[could not provide object as string]" will
+// be returned.
+//
+// NOTE: objAsString is intended to produce human readable output. It is lossy,
+// and it is not intended to be used for serialization.
+function objAsString(obj: any): string {
+  // Check for undefined input.
+  if (obj === undefined) {
+    return "[cannot convert undefined to string]";
+  }
+  if (obj === null) {
+    return "[cannot convert null to string]";
+  }
+
+  // Parse the error into a string.
+  if (typeof obj === "string") {
+    return obj;
+  }
+
+  // Check if the object is an error, and return the message of the error if
+  // so.
+  if (obj instanceof Error) {
+    return obj.message;
+  }
+
+  // Check if the object has a 'toString' method defined on it. To ensure
+  // that we don't crash or throw, check that the toString is a function, and
+  // also that the return value of toString is a string.
+  if (Object.prototype.hasOwnProperty.call(obj, "toString")) {
+    if (typeof obj.toString === "function") {
+      const str = obj.toString();
+      if (typeof str === "string") {
+        return str;
+      }
+    }
+  }
+
+  // If the object does not have a custom toString, attempt to perform a
+  // JSON.stringify. We use a lot of bigints in libskynet, and calling
+  // JSON.stringify on an object with a bigint will cause a throw, so we add
+  // some custom handling to allow bigint objects to still be encoded.
+  try {
+    return JSON.stringify(obj, (_, v) => {
+      if (typeof v === "bigint") {
+        return v.toString();
+      }
+      return v;
+    });
+  } catch (err: any) {
+    if (err !== undefined && typeof err.message === "string") {
+      return `[stringify failed]: ${err.message}`;
+    }
+    return "[stringify failed]";
+  }
+}
+
+// decodeU64 is the opposite of encodeU64, it takes a uint64 encoded as 8 bytes
+// and decodes them into a BigInt.
+function decodeU64(u8: Uint8Array): [bigint, Err] {
+  // Check the input.
+  if (u8.length !== 8) {
+    return [0n, "input should be 8 bytes"];
+  }
+
+  // Process the input.
+  let num = 0n;
+  for (let i = u8.length - 1; i >= 0; i--) {
+    num *= 256n;
+    num += BigInt(u8[i]);
+  }
+  return [num, null];
+}
+
+// encodeU64 will encode a bigint in the range of a uint64 to an 8 byte
+// Uint8Array.
+function encodeU64(num: bigint): [Uint8Array, Err] {
+  // Check the bounds on the bigint.
+  if (num < 0) {
+    return [new Uint8Array(0), "expected a positive integer"];
+  }
+  if (num > MAX_UINT_64) {
+    return [new Uint8Array(0), "expected a number no larger than a uint64"];
+  }
+
+  // Encode the bigint into a Uint8Array.
+  const encoded = new Uint8Array(8);
+  for (let i = 0; i < encoded.length; i++) {
+    encoded[i] = Number(num & 0xffn);
+    num = num >> 8n;
+  }
+  return [encoded, null];
+}
+
+function bufToB64(buf: Uint8Array): string {
+  const b64Str = btoa(String.fromCharCode(...buf));
+  return b64Str.replaceAll("+", "-").replaceAll("/", "_").replaceAll("=", "");
+}
+
+export { objAsString, addContextToErr, encodeU64, decodeU64, bufToB64 };