From 2894e63aa46bbc66d1b8cd3c59d85f083fc713f5 Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Fri, 8 Sep 2023 06:31:51 -0400 Subject: [PATCH] feat: implement appdb, known as hidden db in s5 --- src/appDb.ts | 178 ++++++++++++++++++++++++++++++++++++++++++++++ src/encryption.ts | 109 ++++++++++++++++++++++++++++ src/index.ts | 5 +- 3 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 src/appDb.ts create mode 100644 src/encryption.ts diff --git a/src/appDb.ts b/src/appDb.ts new file mode 100644 index 0000000..768ae87 --- /dev/null +++ b/src/appDb.ts @@ -0,0 +1,178 @@ +import { + CID, + createKeyPair, + S5Node, + encryptionKeyDerivationTweak, + pathKeyDerivationTweak, + writeKeyDerivationTweak, +} from "@lumeweb/libs5"; +import { deriveBlakeChildKey, deriveBlakeChildKeyInt } from "#keys.js"; +import { uploadObject } from "#upload.js"; +import { signRegistryEntry } from "@lumeweb/libs5/lib/service/registry.js"; +import { encodeRegistryValue } from "#cid.js"; +import { downloadSmallObject } from "#download.js"; +import { readableStreamToUint8Array } from "binconv"; +import { utf8ToBytes } from "@noble/hashes/utils"; +import { decryptMutableBytes, encryptMutableBytes } from "#encryption.js"; + +export default class AppDb { + private readonly _rootKey: Uint8Array; + private readonly _node: S5Node; + private readonly _cidMap: Map = new Map(); + + constructor(rootKey: Uint8Array, api: S5Node) { + this._rootKey = rootKey; + this._node = api; + } + + public async getRawData(path: string): Promise { + const pathKey = this._derivePathKeyForPath(path); + const encryptionKey = deriveBlakeChildKeyInt( + pathKey, + encryptionKeyDerivationTweak, + ); + + const writeKey = deriveBlakeChildKeyInt(pathKey, writeKeyDerivationTweak); + const keyPair = await createKeyPair(writeKey); + + const sre = await this._node.services.registry.get(keyPair.publicKey); + if (!sre) { + return new AppDbRawDataResponse(); + } + + const cid = CID.fromBytes(sre.data.slice(1)); + + const bytes = await downloadSmallObject(cid.toString()); + + const plaintext = await decryptMutableBytes( + await readableStreamToUint8Array(bytes), + encryptionKey, + ); + + this._cidMap.set(path, cid); + + return new AppDbRawDataResponse({ + data: plaintext, + cid, + revision: sre.revision, + }); + } + + public async setRawData( + path: string, + data: Uint8Array, + revision: number, + ): Promise { + const pathKey = this._derivePathKeyForPath(path); + const encryptionKey = deriveBlakeChildKeyInt( + pathKey, + encryptionKeyDerivationTweak, + ); + + const cipherText = await encryptMutableBytes(data, encryptionKey); + + const cid = await uploadObject(cipherText); + const writeKey = deriveBlakeChildKeyInt(pathKey, writeKeyDerivationTweak); + const keyPair = createKeyPair(writeKey); + const registryVal = encodeRegistryValue(cid); + + const sre = await signRegistryEntry({ + kp: keyPair, + data: registryVal, + revision, + }); + + await this._node.services.registry.set(sre); + + this._cidMap.set(path, cid); + } + + public async getJSON(path: string): Promise { + const res = await this.getRawData(path); + + if (res.data === null) { + return new AppDbJSONResponse({ cid: res.cid }); + } + + const decodedData = JSON.parse(new TextDecoder().decode(res.data)); + return new AppDbJSONResponse({ + data: decodedData, + revision: res.revision, + cid: res.cid, + }); + } + + public async setJSON( + path: string, + data: any, + revision: number, + ): Promise { + const encodedData = utf8ToBytes(JSON.stringify(data)); + await this.setRawData(path, new Uint8Array(encodedData), revision); + } + + private _derivePathKeyForPath(path: string): Uint8Array { + const pathSegments = path + .split("/") + .map((e) => e.trim()) + .filter((element) => element.length > 0); + + const key = this._deriveKeyForPathSegments(pathSegments); + return deriveBlakeChildKeyInt(key, pathKeyDerivationTweak); + } + + private _deriveKeyForPathSegments(pathSegments: string[]): Uint8Array { + if (pathSegments.length === 0) { + return this._rootKey; + } + + return deriveBlakeChildKey( + this._deriveKeyForPathSegments( + pathSegments.slice(0, pathSegments.length - 1), + ), + pathSegments[pathSegments.length - 1], + ); + } +} + +class AppDbRawDataResponse { + data: Uint8Array | undefined; + revision: number; + cid: CID | undefined; + + constructor({ + data, + revision = -1, + cid, + }: { + data?: Uint8Array; + revision?: number; + cid?: CID; + } = {}) { + this.data = data; + this.revision = revision || -1; + this.cid = cid; + } +} + +class AppDbJSONResponse { + data: any; + revision: number; + cid: CID | undefined; + + constructor({ + data, + revision = -1, + cid, + }: { + data?: any; + revision?: number; + cid?: CID; + } = {}) { + this.data = data; + this.revision = revision || -1; + this.cid = cid; + } +} + +export { AppDbRawDataResponse, AppDbJSONResponse }; diff --git a/src/encryption.ts b/src/encryption.ts new file mode 100644 index 0000000..c3048d0 --- /dev/null +++ b/src/encryption.ts @@ -0,0 +1,109 @@ +import { + decodeEndian, + encodeEndian, + encryptionKeyLength, + encryptionNonceLength, + encryptionOverheadLength, +} from "@lumeweb/libs5"; + +import { xchacha20poly1305 } from "@noble/ciphers/chacha"; +import { randomBytes } from "@noble/ciphers/webcrypto/utils"; + +function padFileSizeDefault(initialSize: number): number { + const kib = 1 << 10; + // Only iterate to 53 (the maximum safe power of 2). + for (let n = 0; n < 53; n++) { + if (initialSize <= (1 << n) * 80 * kib) { + const paddingBlock = (1 << n) * 4 * kib; + let finalSize = initialSize; + if (finalSize % paddingBlock !== 0) { + finalSize = initialSize - (initialSize % paddingBlock) + paddingBlock; + } + return finalSize; + } + } + // Prevent overflow. + throw new Error("Could not pad file size, overflow detected."); +} + +function checkPaddedBlock(size: number): boolean { + const kib = 1024; + // Only iterate to 53 (the maximum safe power of 2). + for (let n = 0; n < 53; n++) { + if (size <= (1 << n) * 80 * kib) { + const paddingBlock = (1 << n) * 4 * kib; + return size % paddingBlock === 0; + } + } + throw new Error("Could not check padded file size, overflow detected."); +} + +async function encryptMutableBytes( + data: Uint8Array, + key: Uint8Array, +): Promise { + const lengthInBytes = encodeEndian(data.length, 4); + + const totalOverhead = + encryptionOverheadLength + 4 + encryptionNonceLength + 2; + + const finalSize = + padFileSizeDefault(data.length + totalOverhead) - totalOverhead; + + data = new Uint8Array([ + ...lengthInBytes, + ...data, + ...new Uint8Array(finalSize - data.length), + ]); + + const nonce = randomBytes(encryptionNonceLength); + + const header = new Uint8Array([0x8d, 0x01, ...nonce]); + + const stream_x = xchacha20poly1305(key, nonce); + + const encryptedBytes = stream_x.encrypt(data); + + return new Uint8Array([...header, ...encryptedBytes]); +} + +async function decryptMutableBytes( + data: Uint8Array, + key: Uint8Array, +): Promise { + if (key.length !== encryptionKeyLength) { + throw `wrong encryptionKeyLength (${key.length} != ${encryptionKeyLength})`; + } + + if (!checkPaddedBlock(data.length)) { + throw `Expected parameter 'data' to be padded encrypted data, length was '${ + data.length + }', nearest padded block is '${padFileSizeDefault(data.length)}'`; + } + + const version = data[1]; + if (version !== 0x01) { + throw "Invalid version"; + } + + const nonce = data.subarray(2, encryptionNonceLength + 2); + + const stream_x = xchacha20poly1305(key, nonce); + + const decryptedBytes = stream_x.decrypt( + data.subarray(encryptionNonceLength + 2), + ); + + const lengthInBytes = decryptedBytes.subarray(0, 4); + + const length = decodeEndian(lengthInBytes); + + return decryptedBytes.subarray(4, length + 4); +} + +export { + padFileSizeDefault, + checkPaddedBlock, + decryptMutableBytes, + encryptMutableBytes, +}; diff --git a/src/index.ts b/src/index.ts index 7ad13c9..331649c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,8 @@ import { ed25519 } from "@noble/curves/ed25519"; import { sha512 } from "@noble/hashes/sha512"; +import AppDb from "#appDb.js"; + export { bytesToHex, hexToBytes, @@ -30,4 +32,5 @@ export * from "./keys.js"; export * from "./download.js"; export * from "./upload.js"; export * from "./portal.js"; -export { ed25519, sha512 }; +export * from "./encryption.js"; +export { ed25519, sha512, AppDb };