feat: implement appdb, known as hidden db in s5

src/appDb.ts

@@ -0,0 +1,178 @@
+import {
+  CID,
+  createKeyPair,
+  S5Node,
+  encryptionKeyDerivationTweak,
+  pathKeyDerivationTweak,
+  writeKeyDerivationTweak,
+} from "@lumeweb/libs5";
+import { deriveBlakeChildKey, deriveBlakeChildKeyInt } from "#keys.js";
+import { uploadObject } from "#upload.js";
+import { signRegistryEntry } from "@lumeweb/libs5/lib/service/registry.js";
+import { encodeRegistryValue } from "#cid.js";
+import { downloadSmallObject } from "#download.js";
+import { readableStreamToUint8Array } from "binconv";
+import { utf8ToBytes } from "@noble/hashes/utils";
+import { decryptMutableBytes, encryptMutableBytes } from "#encryption.js";
+
+export default class AppDb {
+  private readonly _rootKey: Uint8Array;
+  private readonly _node: S5Node;
+  private readonly _cidMap: Map<string, CID> = new Map<string, CID>();
+
+  constructor(rootKey: Uint8Array, api: S5Node) {
+    this._rootKey = rootKey;
+    this._node = api;
+  }
+
+  public async getRawData(path: string): Promise<AppDbRawDataResponse> {
+    const pathKey = this._derivePathKeyForPath(path);
+    const encryptionKey = deriveBlakeChildKeyInt(
+      pathKey,
+      encryptionKeyDerivationTweak,
+    );
+
+    const writeKey = deriveBlakeChildKeyInt(pathKey, writeKeyDerivationTweak);
+    const keyPair = await createKeyPair(writeKey);
+
+    const sre = await this._node.services.registry.get(keyPair.publicKey);
+    if (!sre) {
+      return new AppDbRawDataResponse();
+    }
+
+    const cid = CID.fromBytes(sre.data.slice(1));
+
+    const bytes = await downloadSmallObject(cid.toString());
+
+    const plaintext = await decryptMutableBytes(
+      await readableStreamToUint8Array(bytes),
+      encryptionKey,
+    );
+
+    this._cidMap.set(path, cid);
+
+    return new AppDbRawDataResponse({
+      data: plaintext,
+      cid,
+      revision: sre.revision,
+    });
+  }
+
+  public async setRawData(
+    path: string,
+    data: Uint8Array,
+    revision: number,
+  ): Promise<void> {
+    const pathKey = this._derivePathKeyForPath(path);
+    const encryptionKey = deriveBlakeChildKeyInt(
+      pathKey,
+      encryptionKeyDerivationTweak,
+    );
+
+    const cipherText = await encryptMutableBytes(data, encryptionKey);
+
+    const cid = await uploadObject(cipherText);
+    const writeKey = deriveBlakeChildKeyInt(pathKey, writeKeyDerivationTweak);
+    const keyPair = createKeyPair(writeKey);
+    const registryVal = encodeRegistryValue(cid);
+
+    const sre = await signRegistryEntry({
+      kp: keyPair,
+      data: registryVal,
+      revision,
+    });
+
+    await this._node.services.registry.set(sre);
+
+    this._cidMap.set(path, cid);
+  }
+
+  public async getJSON(path: string): Promise<AppDbJSONResponse> {
+    const res = await this.getRawData(path);
+
+    if (res.data === null) {
+      return new AppDbJSONResponse({ cid: res.cid });
+    }
+
+    const decodedData = JSON.parse(new TextDecoder().decode(res.data));
+    return new AppDbJSONResponse({
+      data: decodedData,
+      revision: res.revision,
+      cid: res.cid,
+    });
+  }
+
+  public async setJSON(
+    path: string,
+    data: any,
+    revision: number,
+  ): Promise<void> {
+    const encodedData = utf8ToBytes(JSON.stringify(data));
+    await this.setRawData(path, new Uint8Array(encodedData), revision);
+  }
+
+  private _derivePathKeyForPath(path: string): Uint8Array {
+    const pathSegments = path
+      .split("/")
+      .map((e) => e.trim())
+      .filter((element) => element.length > 0);
+
+    const key = this._deriveKeyForPathSegments(pathSegments);
+    return deriveBlakeChildKeyInt(key, pathKeyDerivationTweak);
+  }
+
+  private _deriveKeyForPathSegments(pathSegments: string[]): Uint8Array {
+    if (pathSegments.length === 0) {
+      return this._rootKey;
+    }
+
+    return deriveBlakeChildKey(
+      this._deriveKeyForPathSegments(
+        pathSegments.slice(0, pathSegments.length - 1),
+      ),
+      pathSegments[pathSegments.length - 1],
+    );
+  }
+}
+
+class AppDbRawDataResponse {
+  data: Uint8Array | undefined;
+  revision: number;
+  cid: CID | undefined;
+
+  constructor({
+    data,
+    revision = -1,
+    cid,
+  }: {
+    data?: Uint8Array;
+    revision?: number;
+    cid?: CID;
+  } = {}) {
+    this.data = data;
+    this.revision = revision || -1;
+    this.cid = cid;
+  }
+}
+
+class AppDbJSONResponse {
+  data: any;
+  revision: number;
+  cid: CID | undefined;
+
+  constructor({
+    data,
+    revision = -1,
+    cid,
+  }: {
+    data?: any;
+    revision?: number;
+    cid?: CID;
+  } = {}) {
+    this.data = data;
+    this.revision = revision || -1;
+    this.cid = cid;
+  }
+}
+
+export { AppDbRawDataResponse, AppDbJSONResponse };

src/encryption.ts

@@ -0,0 +1,109 @@
+import {
+  decodeEndian,
+  encodeEndian,
+  encryptionKeyLength,
+  encryptionNonceLength,
+  encryptionOverheadLength,
+} from "@lumeweb/libs5";
+
+import { xchacha20poly1305 } from "@noble/ciphers/chacha";
+import { randomBytes } from "@noble/ciphers/webcrypto/utils";
+
+function padFileSizeDefault(initialSize: number): number {
+  const kib = 1 << 10;
+  // Only iterate to 53 (the maximum safe power of 2).
+  for (let n = 0; n < 53; n++) {
+    if (initialSize <= (1 << n) * 80 * kib) {
+      const paddingBlock = (1 << n) * 4 * kib;
+      let finalSize = initialSize;
+      if (finalSize % paddingBlock !== 0) {
+        finalSize = initialSize - (initialSize % paddingBlock) + paddingBlock;
+      }
+      return finalSize;
+    }
+  }
+  // Prevent overflow.
+  throw new Error("Could not pad file size, overflow detected.");
+}
+
+function checkPaddedBlock(size: number): boolean {
+  const kib = 1024;
+  // Only iterate to 53 (the maximum safe power of 2).
+  for (let n = 0; n < 53; n++) {
+    if (size <= (1 << n) * 80 * kib) {
+      const paddingBlock = (1 << n) * 4 * kib;
+      return size % paddingBlock === 0;
+    }
+  }
+  throw new Error("Could not check padded file size, overflow detected.");
+}
+
+async function encryptMutableBytes(
+  data: Uint8Array,
+  key: Uint8Array,
+): Promise<Uint8Array> {
+  const lengthInBytes = encodeEndian(data.length, 4);
+
+  const totalOverhead =
+    encryptionOverheadLength + 4 + encryptionNonceLength + 2;
+
+  const finalSize =
+    padFileSizeDefault(data.length + totalOverhead) - totalOverhead;
+
+  data = new Uint8Array([
+    ...lengthInBytes,
+    ...data,
+    ...new Uint8Array(finalSize - data.length),
+  ]);
+
+  const nonce = randomBytes(encryptionNonceLength);
+
+  const header = new Uint8Array([0x8d, 0x01, ...nonce]);
+
+  const stream_x = xchacha20poly1305(key, nonce);
+
+  const encryptedBytes = stream_x.encrypt(data);
+
+  return new Uint8Array([...header, ...encryptedBytes]);
+}
+
+async function decryptMutableBytes(
+  data: Uint8Array,
+  key: Uint8Array,
+): Promise<Uint8Array> {
+  if (key.length !== encryptionKeyLength) {
+    throw `wrong encryptionKeyLength (${key.length} != ${encryptionKeyLength})`;
+  }
+
+  if (!checkPaddedBlock(data.length)) {
+    throw `Expected parameter 'data' to be padded encrypted data, length was '${
+      data.length
+    }', nearest padded block is '${padFileSizeDefault(data.length)}'`;
+  }
+
+  const version = data[1];
+  if (version !== 0x01) {
+    throw "Invalid version";
+  }
+
+  const nonce = data.subarray(2, encryptionNonceLength + 2);
+
+  const stream_x = xchacha20poly1305(key, nonce);
+
+  const decryptedBytes = stream_x.decrypt(
+    data.subarray(encryptionNonceLength + 2),
+  );
+
+  const lengthInBytes = decryptedBytes.subarray(0, 4);
+
+  const length = decodeEndian(lengthInBytes);
+
+  return decryptedBytes.subarray(4, length + 4);
+}
+
+export {
+  padFileSizeDefault,
+  checkPaddedBlock,
+  decryptMutableBytes,
+  encryptMutableBytes,
+};

src/index.ts

@@ -1,6 +1,8 @@
 import { ed25519 } from "@noble/curves/ed25519";
 import { sha512 } from "@noble/hashes/sha512";
 
+import AppDb from "#appDb.js";
+
 export {
   bytesToHex,
   hexToBytes,
@@ -30,4 +32,5 @@ export * from "./keys.js";
 export * from "./download.js";
 export * from "./upload.js";
 export * from "./portal.js";
-export { ed25519, sha512 };
+export * from "./encryption.js";
+export { ed25519, sha512, AppDb };