From dfe79e2fc2e9756e54bfbdec97333d768af333a7 Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Thu, 21 Mar 2024 10:19:15 -0400 Subject: [PATCH] refactor: merge all methods into a single class definition --- src/client.ts | 747 +++++++++++++++++++++++++++++++++++++--- src/index.ts | 2 +- src/methods/account.ts | 21 -- src/methods/download.ts | 195 ----------- src/methods/registry.ts | 223 ------------ src/methods/upload.ts | 415 ---------------------- src/number.ts | 21 ++ src/options/download.ts | 28 ++ src/options/registry.ts | 22 ++ src/options/upload.ts | 81 +++++ src/utils/encoding.ts | 5 + src/utils/file.ts | 14 + src/utils/options.ts | 9 +- 13 files changed, 872 insertions(+), 911 deletions(-) delete mode 100644 src/methods/account.ts delete mode 100644 src/methods/download.ts delete mode 100644 src/methods/registry.ts delete mode 100644 src/methods/upload.ts create mode 100644 src/number.ts create mode 100644 src/options/download.ts create mode 100644 src/options/registry.ts create mode 100644 src/options/upload.ts create mode 100644 src/utils/encoding.ts diff --git a/src/client.ts b/src/client.ts index ce15b5a..b00e6de 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,67 +1,79 @@ import { - uploadFile, - uploadLargeFile, - uploadDirectory, - uploadDirectoryRequest, - uploadSmallFile, - uploadSmallFileRequest, - uploadLargeFileRequest, - uploadWebapp, - uploadWebappRequest, - getTusOptions, -} from "./methods/upload.js"; -import { - downloadBlob, - downloadData, - downloadFile, - downloadProof, - getCidUrl, - getMetadata, -} from "./methods/download.js"; + CustomDownloadOptions, + CustomGetMetadataOptions, + DEFAULT_DOWNLOAD_OPTIONS, + DEFAULT_GET_METADATA_OPTIONS, + MetadataResult, +} from "#options/download.js"; -import { ensureUrl } from "./utils/url.js"; +import { addUrlQuery, ensureUrl } from "./utils/url.js"; import { - createEntry, - getEntry, - publishEntry, - subscribeToEntry, -} from "./methods/registry.js"; -import { CustomClientOptions } from "./utils/options.js"; + CustomRegistryOptions, + DEFAULT_GET_ENTRY_OPTIONS, + DEFAULT_PUBLISH_ENTRY_OPTIONS, + DEFAULT_SUBSCRIBE_ENTRY_OPTIONS, +} from "#options/registry.js"; +import { CustomClientOptions, optionsToConfig } from "./utils/options.js"; import { throwValidationError } from "./utils/validation.js"; - -import { accountPins } from "./methods/account.js"; +import { + AccountPinsResponse, + BasicUploadResponse, + getS5AccountPins, + getS5BlobCid, + getS5DownloadCid, + getS5MetadataCid, + getS5Registry, + postS5Registry, + postS5Upload, + postS5UploadDirectory, + PostS5UploadDirectoryParams, + PostS5UploadResult, +} from "./generated/index.js"; +import path from "path"; +import { customInstance } from "./axios.js"; +import { ensureBytes, equalBytes } from "@noble/curves/abstract/utils.js"; +import { concatBytes } from "@noble/hashes/utils.js"; +import { CID_HASH_TYPES } from "@lumeweb/libs5/lib/constants.js"; +import { buildRequestUrl } from "./request.js"; +import WS from "isomorphic-ws"; +import { + CID, + CID_TYPES, + createKeyPair, + KeyPairEd25519, + Packer, + SignedRegistryEntry, +} from "@lumeweb/libs5"; +import { + deserializeRegistryEntry, + signRegistryEntry, + verifyRegistryEntry, +} from "@lumeweb/libs5/lib/service/registry.js"; +import { Buffer } from "buffer"; +import { AxiosError } from "axios"; +import { + CustomUploadOptions, + DEFAULT_UPLOAD_OPTIONS, + TUS_ENDPOINT, + UploadResult, +} from "#options/upload.js"; +import { + DetailedError, + HttpRequest, + Upload, + UploadOptions, +} from "tus-js-client"; +import { ensureFileObjectConsistency } from "./utils/file.js"; +import defer from "p-defer"; +import { Multihash } from "@lumeweb/libs5/lib/multihash.js"; +import { blake3 } from "@noble/hashes/blake3"; +import { base64urlDecode, base64urlEncode } from "#utils/encoding.js"; /** * The S5 Client which can be used to access S5-net. */ export class S5Client { - // Upload - uploadFile = uploadFile; - uploadDirectory = uploadDirectory; - // Set methods (defined in other files). - uploadWebapp = uploadWebapp; - downloadFile = downloadFile; - downloadData = downloadData; - downloadBlob = downloadBlob; - downloadProof = downloadProof; - getCidUrl = getCidUrl; - getMetadata = getMetadata; - // Registry - subscribeToEntry = subscribeToEntry; - publishEntry = publishEntry; - createEntry = createEntry; - getEntry = getEntry; - accountPins = accountPins; - // Download - getTusOptions = getTusOptions; - protected uploadSmallFile = uploadSmallFile; - protected uploadSmallFileRequest = uploadSmallFileRequest; - protected uploadLargeFile = uploadLargeFile; - protected uploadLargeFileRequest = uploadLargeFileRequest; - protected uploadDirectoryRequest = uploadDirectoryRequest; - protected uploadWebappRequest = uploadWebappRequest; - /** * The S5 Client which can be used to access S5-net. * @@ -99,4 +111,631 @@ export class S5Client { ) { return new S5Client(portalUrl, customOptions); } + + public async accountPins( + customOptions: CustomClientOptions = {}, + ): Promise { + const opts = { + ...this.clientOptions, + ...customOptions, + ...{ + endpointPath: "/s5/account/pins", + baseUrl: await this.portalUrl, + }, + }; + + const config = optionsToConfig(this, opts); + + return await getS5AccountPins(config); + } + + /** + * Initiates a download of the content of the cid within the browser. + * + * @param cid - 46-character cid, or a valid cid URL. Can be followed by a path. Note that the cid will not be encoded, so if your path might contain special characters, consider using `clientOptions.path`. + * @param [customOptions] - Additional settings that can optionally be set. + * @param [customOptions.endpointDownload="/"] - The relative URL path of the portal endpoint to contact. + * @returns - The full URL that was used. + * @throws - Will throw if the cid does not contain a cid or if the path option is not a string. + */ + public async downloadFile( + cid: string, + customOptions?: CustomDownloadOptions, + ): Promise { + const url = await this.getCidUrl(cid, customOptions); + + // Download the url. + window.location.assign(url); + + return url; + } + + /** + * Constructs the full URL for the given cid. + * + * @param cid - Base64 cid, or a valid URL that contains a cid. See `downloadFile`. + * @param [customOptions] - Additional settings that can optionally be set. + * @param [customOptions.endpointDownload="/"] - The relative URL path of the portal endpoint to contact. + * @returns - The full URL for the cid. + * @throws - Will throw if the cid does not contain a cid or if the path option is not a string. + */ + public async getCidUrl( + cid: string, + customOptions: CustomDownloadOptions = {}, + ): Promise { + const opt = { ...this.clientOptions, customOptions }; + return addUrlQuery(path.join(this.portalUrl, cid), { + auth_token: opt.apiKey, + }); + } + + /** + * Gets only the metadata for the given cid without the contents. + * + * @param cid - Base64 cid. + * @param [customOptions] - Additional settings that can optionally be set. See `downloadFile` for the full list. + * @param [customOptions.endpointGetMetadata="/"] - The relative URL path of the portal endpoint to contact. + * @returns - The metadata in JSON format. Empty if no metadata was found. + * @throws - Will throw if the cid does not contain a cid . + */ + public async getMetadata( + cid: string, + customOptions: CustomGetMetadataOptions = {}, + ): Promise { + const config = optionsToConfig( + this, + DEFAULT_GET_METADATA_OPTIONS, + customOptions, + ); + + const response = await getS5MetadataCid(cid, config); + + return { metadata: response }; + } + + /** + * Downloads in-memory data from a S5 cid. + * @param cid - 46-character cid, or a valid cid URL. + * @param [customOptions] - Additional settings that can optionally be set. + * @returns - The data + */ + public async downloadData( + this: S5Client, + cid: string, + customOptions: CustomDownloadOptions = {}, + ): Promise { + const opts = { + ...DEFAULT_DOWNLOAD_OPTIONS, + ...this.clientOptions, + ...customOptions, + download: true, + }; + + const config = optionsToConfig( + this, + DEFAULT_DOWNLOAD_OPTIONS, + customOptions, + ); + + return await (await getS5DownloadCid(cid, config)).arrayBuffer(); + } + + /** + * Downloads a proof for the given cid. + * @param cid - 46-character cid, or a valid cid URL. + * @param [customOptions] - Additional settings that can optionally be set. + * @returns - The data + */ + public async downloadProof( + cid: string, + customOptions: CustomDownloadOptions = {}, + ): Promise { + return this.downloadData(`${cid}.obao`, customOptions); + } + + /** + * Downloads a blob from the given cid. This will capture a 301 redirect to the actual blob location, then download the blob. + * @param cid - 46-character cid, or a valid cid URL. + * @param [customOptions] - Additional settings that can optionally be set. + * @returns - The data + */ + + async downloadBlob( + cid: string, + customOptions: CustomDownloadOptions = {}, + ): Promise { + const config = optionsToConfig( + this, + DEFAULT_DOWNLOAD_OPTIONS, + customOptions, + ); + + let location: string | null = null; + + await getS5BlobCid(cid, { + ...config, + responseType: "arraybuffer", + beforeRedirect: (config, responseDetails) => { + location = responseDetails.headers["location"]; + }, + }); + + if (!location) { + throw new Error("Failed to download blob"); + } + + return await customInstance( + { + url: `/s5/blob/${cid}`, + method: "GET", + responseType: "arraybuffer", + }, + config, + ); + } + + public async subscribeToEntry( + publicKey: Uint8Array, + customOptions: CustomRegistryOptions = {}, + ) { + const opts = { + ...DEFAULT_SUBSCRIBE_ENTRY_OPTIONS, + ...this.clientOptions, + ...customOptions, + } satisfies CustomRegistryOptions; + + publicKey = ensureBytes("public key", publicKey, 32); + publicKey = concatBytes( + Uint8Array.from([CID_HASH_TYPES.ED25519]), + publicKey, + ); + + const url = await buildRequestUrl(this, { + baseUrl: await this.portalUrl, + endpointPath: opts.endpointSubscribeEntry, + }); + + const wsUrl = url.replace(/^http/, "ws"); + + const socket = new WS(wsUrl); + socket.binaryType = "arraybuffer"; + + socket.addEventListener("open", () => { + const packer = new Packer(); + packer.pack(2); + packer.pack(publicKey); + + socket.send(packer.takeBytes()); + }); + + return { + listen(cb: (entry: SignedRegistryEntry) => void) { + socket.addEventListener("message", (data) => { + cb(deserializeRegistryEntry(new Uint8Array(data.data as Buffer))); + }); + }, + end() { + if ( + [socket.CLOSING, socket.CLOSED].includes(socket.readyState as any) + ) { + return; + } + socket.close(); + }, + }; + } + + public async publishEntry( + this: S5Client, + signedEntry: SignedRegistryEntry, + customOptions: CustomRegistryOptions = {}, + ) { + const config = optionsToConfig( + this, + DEFAULT_PUBLISH_ENTRY_OPTIONS, + customOptions, + ); + + if (!verifyRegistryEntry(signedEntry)) { + throwValidationError( + "signedEntry", // name of the variable + signedEntry, // actual value + "parameter", // valueKind (assuming it's a function parameter) + "a valid signed registry entry", // expected description + ); + } + + return postS5Registry( + { + pk: base64urlEncode(signedEntry.pk), + revision: signedEntry.revision, + data: base64urlEncode(signedEntry.data), + signature: base64urlEncode(signedEntry.signature), + }, + config, + ); + } + + public async createEntry( + this: S5Client, + sk: Uint8Array | KeyPairEd25519, + cid: CID, + revision = 0, + ) { + if (sk instanceof Uint8Array) { + sk = createKeyPair(sk); + } + + let existing = true; + let entry = await this.getEntry(sk.publicKey); + + if (!entry) { + existing = false; + entry = { + pk: sk.publicKey, + data: cid.toRegistryEntry(), + revision, + } as unknown as SignedRegistryEntry; + } + + if (!equalBytes(sk.publicKey, entry.pk)) { + throwValidationError( + "entry.pk", // name of the variable + Buffer.from(entry.pk).toString("hex"), // actual value + "result", // valueKind (assuming it's a function parameter) + Buffer.from(sk.publicKey).toString("hex"), // expected description + ); + } + + if (existing) { + const newEntry = cid.toRegistryEntry(); + if (equalBytes(entry.data, newEntry)) { + return entry; + } + + entry.revision++; + entry.data = newEntry; + } + const signedEntry = signRegistryEntry({ + kp: sk, + data: entry.data, + revision: entry.revision, + }); + + await this.publishEntry(signedEntry); + + return signedEntry; + } + + public async getEntry( + this: S5Client, + publicKey: Uint8Array, + customOptions: CustomRegistryOptions = {}, + ) { + const config = optionsToConfig( + this, + DEFAULT_GET_ENTRY_OPTIONS, + customOptions, + ); + + try { + const ret = await getS5Registry( + { + pk: base64urlEncode(publicKey), + }, + config, + ); + + const signedEntry = { + pk: base64urlDecode(ret.pk), + revision: ret.revision, + data: base64urlDecode(ret.data), + signature: base64urlDecode(ret.signature), + } as SignedRegistryEntry; + + if (!verifyRegistryEntry(signedEntry)) { + throwValidationError( + "signedEntry", // name of the variable + signedEntry, // actual value + "result", // valueKind (assuming it's a function parameter) + "a valid signed registry entry", // expected description + ); + } + + return signedEntry; + } catch (e) { + if ((e as AxiosError).response?.status === 404) { + return undefined; + } + + throw e; + } + } + + /** + * Uploads a file to S5-net. + * + * @param file - The file to upload. + * @param [customOptions] - Additional settings that can optionally be set. + * @returns - The returned cid. + * @throws - Will throw if the request is successful but the upload response does not contain a complete response. + */ + public async uploadFile( + this: S5Client, + file: File, + customOptions: CustomUploadOptions = {}, + ): Promise { + const opts = { + ...DEFAULT_UPLOAD_OPTIONS, + ...this.clientOptions, + ...customOptions, + } as CustomUploadOptions; + + if (file.size < opts?.largeFileSize) { + return this.uploadSmallFile(file, opts); + } else { + return this.uploadLargeFile(file, opts); + } + } + + /** + * Uploads a small file to S5-net. + * + * @param file - The file to upload. + * @param [customOptions] - Additional settings that can optionally be set. + + * @returns UploadResult - The returned cid. + * @throws - Will throw if the request is successful but the upload response does not contain a complete response. + */ + public async uploadSmallFile( + this: S5Client, + file: File, + customOptions: CustomUploadOptions, + ): Promise { + const response = await this.uploadSmallFileRequest(file, customOptions); + + return { cid: CID.decode(response.cid) }; + } + + /* istanbul ignore next */ + /** + * Uploads a large file to S5-net using tus. + * + * @param file - The file to upload. + * @param [customOptions] - Additional settings that can optionally be set. + * @param [customOptions.endpointLargeUpload="/s5/upload/tus"] - The relative URL path of the portal endpoint to contact. + * @returns - The returned cid. + * @throws - Will throw if the request is successful but the upload response does not contain a complete response. + */ + public async uploadLargeFile( + this: S5Client, + file: File, + customOptions: CustomUploadOptions = {}, + ): Promise { + return await this.uploadLargeFileRequest(file, customOptions); + } + + public async getTusOptions( + this: S5Client, + file: File, + tusOptions: Partial = {}, + customOptions: CustomUploadOptions = {}, + ): Promise { + const config = optionsToConfig(this, DEFAULT_UPLOAD_OPTIONS, customOptions); + + // Validation. + const url = await buildRequestUrl(this, { + endpointPath: TUS_ENDPOINT, + }); + + file = ensureFileObjectConsistency(file); + + const hasher = blake3.create({}); + + const chunkSize = 1024 * 1024; + + let position = 0; + + while (position <= file.size) { + const chunk = file.slice(position, position + chunkSize); + ``; + hasher.update(new Uint8Array(await chunk.arrayBuffer())); + position += chunkSize; + customOptions.onHashProgress?.({ + bytes: position, + total: file.size, + }); + } + + const b3hash = hasher.digest(); + + const filename = new Multihash( + Buffer.concat([ + Buffer.alloc(1, CID_HASH_TYPES.BLAKE3), + Buffer.from(b3hash), + ]), + ).toBase64Url(); + + return { + endpoint: url, + metadata: { + hash: filename, + filename: filename, + filetype: file.type, + }, + headers: config.headers as any, + onBeforeRequest: function (req: HttpRequest) { + const xhr = req.getUnderlyingObject(); + xhr.withCredentials = true; + }, + ...tusOptions, + }; + } + + /** + * Uploads a directory to S5-net. + * + * @param directory - File objects to upload, indexed by their path strings. + * @param filename - The name of the directory. + * @param [customOptions] - Additional settings that can optionally be set. + * @param [customOptions.endpointPath="/s5/upload/directory"] - The relative URL path of the portal endpoint to contact. + * @returns - The returned cid. + * @throws - Will throw if the request is successful but the upload response does not contain a complete response. + */ + public async uploadDirectory( + this: S5Client, + directory: Record, + filename: string, + customOptions: CustomUploadOptions = {}, + ): Promise { + const response = await this.uploadDirectoryRequest( + directory, + filename, + customOptions, + ); + + return { cid: CID.decode(response.cid) }; + } + + /** + * Makes a request to upload a directory to S5-net. + * + * @param directory - File objects to upload, indexed by their path strings. + * @param filename - The name of the directory. + * @param [customOptions] - Additional settings that can optionally be set. + * @returns - The upload response. + * @throws - Will throw if the input filename is not a string. + */ + public async uploadDirectoryRequest( + this: S5Client, + directory: Record, + filename: string, + customOptions: CustomUploadOptions = {}, + ): Promise { + const config = optionsToConfig(this, DEFAULT_UPLOAD_OPTIONS, customOptions); + + const formData = new FormData(); + + for (const entry in directory) { + const file = ensureFileObjectConsistency(directory[entry]); + formData.append(entry, file, entry); + } + + const params = {} as PostS5UploadDirectoryParams; + + if (customOptions.tryFiles) { + params.tryFiles = customOptions.tryFiles; + } + if (customOptions.errorPages) { + params.errorPages = customOptions.errorPages; + } + + params.name = filename; + + /* + Hack to pass the data right since OpenAPI doesn't support variable file inputs without knowing the names ahead of time. + */ + config.data = formData; + + return postS5UploadDirectory({}, params, config); + } + + public async uploadWebapp( + this: S5Client, + directory: Record, + customOptions: CustomUploadOptions = {}, + ): Promise { + const response = await this.uploadWebappRequest(directory, customOptions); + + return { cid: CID.decode(response.cid) }; + } + + /** + * Makes a request to upload a directory to S5-net. + * @param directory - File objects to upload, indexed by their path strings. + * @param [customOptions] - Additional settings that can optionally be set. + * @param [customOptions.endpointPath] - The relative URL path of the portal endpoint to contact. + * @returns - The upload response. + * @throws - Will throw if the input filename is not a string. + */ + public async uploadWebappRequest( + this: S5Client, + directory: Record, + customOptions: CustomUploadOptions = {}, + ): Promise { + return this.uploadDirectoryRequest(directory, "webapp", customOptions); + } + + /** + * Makes a request to upload a small file to S5-net. + * + * @param file - The file to upload. + * @param [customOptions] - Additional settings that can optionally be set. + + * @returns PostS5UploadResult - The upload response. + */ + private async uploadSmallFileRequest( + this: S5Client, + file: File, + customOptions: CustomUploadOptions = {}, + ): Promise { + const config = optionsToConfig(this, DEFAULT_UPLOAD_OPTIONS, customOptions); + + file = ensureFileObjectConsistency(file); + + return postS5Upload( + { + file: file, + }, + config, + ); + } + + /* istanbul ignore next */ + /** + * Makes a request to upload a file to S5-net. + * + * @param file - The file to upload. + * @param [customOptions] - Additional settings that can optionally be set. + * @returns - The upload response. + */ + private async uploadLargeFileRequest( + this: S5Client, + file: File, + customOptions: CustomUploadOptions = {}, + ): Promise { + const p = defer(); + + const options = await this.getTusOptions( + file, + { + onSuccess: async () => { + if (!upload.url) { + p.reject(new Error("'upload.url' was not set")); + return; + } + + p.resolve({ cid }); + }, + onError: (error: Error | DetailedError) => { + // Return error body rather than entire error. + const res = (error as DetailedError).originalResponse; + const newError = res + ? new Error(res.getBody().trim()) || error + : error; + p.reject(newError); + }, + }, + customOptions, + ); + const cid = CID.fromHash( + Multihash.fromBase64Url(options.metadata?.hash).fullBytes, + file.size, + CID_TYPES.RAW, + ); + + const upload = new Upload(file, options); + + return p.promise; + } } diff --git a/src/index.ts b/src/index.ts index 6cd8cf5..543ef6e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,3 @@ export { S5Client } from "./client.js"; export type { CustomClientOptions } from "./utils/options.js"; -export type { HashProgressEvent } from "./methods/upload.js"; +export type { HashProgressEvent } from "#options/upload.js"; diff --git a/src/methods/account.ts b/src/methods/account.ts deleted file mode 100644 index 2f28652..0000000 --- a/src/methods/account.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { CustomClientOptions, optionsToConfig } from "../utils/options.js"; -import { S5Client } from "../client.js"; -import { AccountPinsResponse, getS5AccountPins } from "../generated/index.js"; - -export async function accountPins( - this: S5Client, - customOptions: CustomClientOptions = {}, -): Promise { - const opts = { - ...this.clientOptions, - ...customOptions, - ...{ - endpointPath: "/s5/account/pins", - baseUrl: await this.portalUrl, - }, - }; - - const config = optionsToConfig(this, opts); - - return await getS5AccountPins(config); -} diff --git a/src/methods/download.ts b/src/methods/download.ts deleted file mode 100644 index 75dc01c..0000000 --- a/src/methods/download.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { ResponseType } from "axios"; - -import { S5Client } from "../client.js"; -import { CustomClientOptions, optionsToConfig } from "../utils/options.js"; -import path from "path"; -import { - getS5BlobCid, - getS5DownloadCid, - getS5MetadataCid, -} from "../generated/index.js"; -import { addUrlQuery } from "../utils/url.js"; -import { customInstance } from "../axios.js"; - -/** - * Custom download options. - * - * @property [endpointDownload] - The relative URL path of the portal endpoint to contact. - * @property [download=false] - Indicates to `getCidUrl` whether the file should be downloaded (true) or opened in the browser (false). `downloadFile` and `openFile` override this value. - * @property [path] - A path to append to the cid, e.g. `dir1/dir2/file`. A Unix-style path is expected. Each path component will be URL-encoded. - * @property [range] - The Range request header to set for the download. Not applicable for in-borwser downloads. - * @property [responseType] - The response type. - * @property [subdomain=false] - Whether to return the final cid in subdomain format. - */ -export type CustomDownloadOptions = CustomClientOptions & { - path?: string; - range?: string; - responseType?: ResponseType; -}; - -export type CustomGetMetadataOptions = CustomClientOptions & {}; - -/** - * The response for a get metadata request. - * - * @property metadata - The metadata in JSON format. - * @property portalUrl - The URL of the portal. - * @property cid - 46-character cid. - */ -export type MetadataResult = { - metadata: Record; -}; - -export const DEFAULT_DOWNLOAD_OPTIONS = { - range: undefined, - responseType: undefined, -} as CustomDownloadOptions; - -const DEFAULT_GET_METADATA_OPTIONS = {}; - -/** - * Initiates a download of the content of the cid within the browser. - * - * @param this - S5Client - * @param cid - 46-character cid, or a valid cid URL. Can be followed by a path. Note that the cid will not be encoded, so if your path might contain special characters, consider using `clientOptions.path`. - * @param [customOptions] - Additional settings that can optionally be set. - * @param [customOptions.endpointDownload="/"] - The relative URL path of the portal endpoint to contact. - * @returns - The full URL that was used. - * @throws - Will throw if the cid does not contain a cid or if the path option is not a string. - */ -export async function downloadFile( - this: S5Client, - cid: string, - customOptions?: CustomDownloadOptions, -): Promise { - const url = await this.getCidUrl(cid, customOptions); - - // Download the url. - window.location.assign(url); - - return url; -} - -/** - * Constructs the full URL for the given cid. - * - * @param this - S5Client - * @param cid - Base64 cid, or a valid URL that contains a cid. See `downloadFile`. - * @param [customOptions] - Additional settings that can optionally be set. - * @param [customOptions.endpointDownload="/"] - The relative URL path of the portal endpoint to contact. - * @returns - The full URL for the cid. - * @throws - Will throw if the cid does not contain a cid or if the path option is not a string. - */ -export async function getCidUrl( - this: S5Client, - cid: string, - customOptions: CustomDownloadOptions = {}, -): Promise { - const opt = { ...this.clientOptions, customOptions }; - return addUrlQuery(path.join(this.portalUrl, cid), { - auth_token: opt.apiKey, - }); -} - -/** - * Gets only the metadata for the given cid without the contents. - * - * @param this - S5Client - * @param cid - Base64 cid. - * @param [customOptions] - Additional settings that can optionally be set. See `downloadFile` for the full list. - * @param [customOptions.endpointGetMetadata="/"] - The relative URL path of the portal endpoint to contact. - * @returns - The metadata in JSON format. Empty if no metadata was found. - * @throws - Will throw if the cid does not contain a cid . - */ -export async function getMetadata( - this: S5Client, - cid: string, - customOptions: CustomGetMetadataOptions = {}, -): Promise { - const config = optionsToConfig( - this, - DEFAULT_GET_METADATA_OPTIONS, - customOptions, - ); - - const response = await getS5MetadataCid(cid, config); - - return { metadata: response }; -} - -/** - * Downloads in-memory data from a S5 cid. - * @param this - S5Client - * @param cid - 46-character cid, or a valid cid URL. - * @param [customOptions] - Additional settings that can optionally be set. - * @returns - The data - */ -export async function downloadData( - this: S5Client, - cid: string, - customOptions: CustomDownloadOptions = {}, -): Promise { - const opts = { - ...DEFAULT_DOWNLOAD_OPTIONS, - ...this.clientOptions, - ...customOptions, - download: true, - }; - - const config = optionsToConfig(this, DEFAULT_DOWNLOAD_OPTIONS, customOptions); - - return await (await getS5DownloadCid(cid, config)).arrayBuffer(); -} - -/** - * Downloads a proof for the given cid. - * @param this - S5Client - * @param cid - 46-character cid, or a valid cid URL. - * @param [customOptions] - Additional settings that can optionally be set. - * @returns - The data - */ -export async function downloadProof( - this: S5Client, - cid: string, - customOptions: CustomDownloadOptions = {}, -): Promise { - return this.downloadData(`${cid}.obao`, customOptions); -} - -/** - * Downloads a blob from the given cid. This will capture a 301 redirect to the actual blob location, then download the blob. - * @param this - S5Client - * @param cid - 46-character cid, or a valid cid URL. - * @param [customOptions] - Additional settings that can optionally be set. - * @returns - The data - */ -export async function downloadBlob( - this: S5Client, - cid: string, - customOptions: CustomDownloadOptions = {}, -): Promise { - const config = optionsToConfig(this, DEFAULT_DOWNLOAD_OPTIONS, customOptions); - - let location: string | null = null; - - await getS5BlobCid(cid, { - ...config, - responseType: "arraybuffer", - beforeRedirect: (config, responseDetails) => { - location = responseDetails.headers["location"]; - }, - }); - - if (!location) { - throw new Error("Failed to download blob"); - } - - return await customInstance( - { - url: `/s5/blob/${cid}`, - method: "GET", - responseType: "arraybuffer", - }, - config, - ); -} diff --git a/src/methods/registry.ts b/src/methods/registry.ts deleted file mode 100644 index fa2664a..0000000 --- a/src/methods/registry.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { S5Client } from "../client.js"; -import { ensureBytes, equalBytes } from "@noble/curves/abstract/utils"; - -import WS from "isomorphic-ws"; -import { - CID, - createKeyPair, - KeyPairEd25519, - Packer, - SignedRegistryEntry, -} from "@lumeweb/libs5"; -import { - deserializeRegistryEntry, - signRegistryEntry, - verifyRegistryEntry, -} from "@lumeweb/libs5/lib/service/registry.js"; -import { Buffer } from "buffer"; -import { throwValidationError } from "../utils/validation.js"; -import { base64url } from "multiformats/bases/base64"; -import { concatBytes } from "@noble/hashes/utils"; -import { CID_HASH_TYPES } from "@lumeweb/libs5/lib/constants.js"; -import { CustomClientOptions, optionsToConfig } from "../utils/options.js"; -import { buildRequestUrl } from "../request.js"; -import { getS5Registry, postS5Registry } from "../generated/index.js"; -import { AxiosError } from "axios"; - -export const DEFAULT_GET_ENTRY_OPTIONS = {}; - -export const DEFAULT_SET_ENTRY_OPTIONS = { - endpointSetEntry: "/s5/registry", -}; - -export const DEFAULT_SUBSCRIBE_ENTRY_OPTIONS = { - endpointSubscribeEntry: "/s5/registry/subscription", -} as CustomRegistryOptions; - -export const DEFAULT_PUBLISH_ENTRY_OPTIONS = { - endpointPublishEntry: "/s5/registry", -} as CustomRegistryOptions; - -export type BaseCustomOptions = CustomClientOptions; - -export type CustomRegistryOptions = BaseCustomOptions & { - endpointSubscribeEntry?: string; - endpointPublishEntry?: string; -}; - -export async function subscribeToEntry( - this: S5Client, - publicKey: Uint8Array, - customOptions: CustomRegistryOptions = {}, -) { - const opts = { - ...DEFAULT_SUBSCRIBE_ENTRY_OPTIONS, - ...this.clientOptions, - ...customOptions, - }; - - publicKey = ensureBytes("public key", publicKey, 32); - publicKey = concatBytes(Uint8Array.from([CID_HASH_TYPES.ED25519]), publicKey); - - const url = await buildRequestUrl(this, { - baseUrl: await this.portalUrl, - endpointPath: opts.endpointSubscribeEntry, - }); - - const wsUrl = url.replace(/^http/, "ws"); - - const socket = new WS(wsUrl); - socket.binaryType = "arraybuffer"; - - socket.addEventListener("open", () => { - const packer = new Packer(); - packer.pack(2); - packer.pack(publicKey); - - socket.send(packer.takeBytes()); - }); - - return { - listen(cb: (entry: SignedRegistryEntry) => void) { - socket.addEventListener("message", (data) => { - cb(deserializeRegistryEntry(new Uint8Array(data.data as Buffer))); - }); - }, - end() { - if ([socket.CLOSING, socket.CLOSED].includes(socket.readyState as any)) { - return; - } - socket.close(); - }, - }; -} - -const base64urlEncode = (d: Uint8Array) => base64url.encode(d).substring(1); -const base64urlDecode = (d: string) => base64url.decode(`u${d}`); - -export async function publishEntry( - this: S5Client, - signedEntry: SignedRegistryEntry, - customOptions: CustomRegistryOptions = {}, -) { - const config = optionsToConfig( - this, - DEFAULT_PUBLISH_ENTRY_OPTIONS, - customOptions, - ); - - if (!verifyRegistryEntry(signedEntry)) { - throwValidationError( - "signedEntry", // name of the variable - signedEntry, // actual value - "parameter", // valueKind (assuming it's a function parameter) - "a valid signed registry entry", // expected description - ); - } - - return postS5Registry( - { - pk: base64urlEncode(signedEntry.pk), - revision: signedEntry.revision, - data: base64urlEncode(signedEntry.data), - signature: base64urlEncode(signedEntry.signature), - }, - config, - ); -} - -export async function createEntry( - this: S5Client, - sk: Uint8Array | KeyPairEd25519, - cid: CID, - revision = 0, -) { - if (sk instanceof Uint8Array) { - sk = createKeyPair(sk); - } - - let existing = true; - let entry = await this.getEntry(sk.publicKey); - - if (!entry) { - existing = false; - entry = { - pk: sk.publicKey, - data: cid.toRegistryEntry(), - revision, - } as unknown as SignedRegistryEntry; - } - - if (!equalBytes(sk.publicKey, entry.pk)) { - throwValidationError( - "entry.pk", // name of the variable - Buffer.from(entry.pk).toString("hex"), // actual value - "result", // valueKind (assuming it's a function parameter) - Buffer.from(sk.publicKey).toString("hex"), // expected description - ); - } - - if (existing) { - const newEntry = cid.toRegistryEntry(); - if (equalBytes(entry.data, newEntry)) { - return entry; - } - - entry.revision++; - entry.data = newEntry; - } - const signedEntry = signRegistryEntry({ - kp: sk, - data: entry.data, - revision: entry.revision, - }); - - await this.publishEntry(signedEntry); - - return signedEntry; -} - -export async function getEntry( - this: S5Client, - publicKey: Uint8Array, - customOptions: CustomRegistryOptions = {}, -) { - const config = optionsToConfig( - this, - DEFAULT_GET_ENTRY_OPTIONS, - customOptions, - ); - - try { - const ret = await getS5Registry( - { - pk: base64urlEncode(publicKey), - }, - config, - ); - - const signedEntry = { - pk: base64urlDecode(ret.pk), - revision: ret.revision, - data: base64urlDecode(ret.data), - signature: base64urlDecode(ret.signature), - } as SignedRegistryEntry; - - if (!verifyRegistryEntry(signedEntry)) { - throwValidationError( - "signedEntry", // name of the variable - signedEntry, // actual value - "result", // valueKind (assuming it's a function parameter) - "a valid signed registry entry", // expected description - ); - } - - return signedEntry; - } catch (e) { - if ((e as AxiosError).response?.status === 404) { - return undefined; - } - - throw e; - } -} diff --git a/src/methods/upload.ts b/src/methods/upload.ts deleted file mode 100644 index 9cd76c9..0000000 --- a/src/methods/upload.ts +++ /dev/null @@ -1,415 +0,0 @@ -import { AxiosProgressEvent } from "axios"; -import { - DetailedError, - HttpRequest, - Upload, - UploadOptions, -} from "tus-js-client"; - -import { blake3 } from "@noble/hashes/blake3"; -import { Buffer } from "buffer"; - -import { getFileMimeType } from "../utils/file.js"; -import { S5Client } from "../client.js"; -import { CID, CID_HASH_TYPES, CID_TYPES } from "@lumeweb/libs5"; -import { - type BasicUploadResponse, - postS5Upload, - postS5UploadDirectory, - PostS5UploadDirectoryParams, - PostS5UploadResult, -} from "../generated/index.js"; -import { BaseCustomOptions } from "./registry.js"; -import { optionsToConfig } from "../utils/options.js"; -import { buildRequestUrl } from "../request.js"; -import defer from "p-defer"; -import { Multihash } from "@lumeweb/libs5/lib/multihash.js"; - -/** - * The tus chunk size is (4MiB - encryptionOverhead) * dataPieces, set as default. - */ -export const TUS_CHUNK_SIZE = (1 << 22) * 8; - -/** - * The retry delays, in ms. Data is stored in for up to 20 minutes, so the - * total delays should not exceed that length of time. - */ -const DEFAULT_TUS_RETRY_DELAYS = [0, 5_000, 15_000, 60_000, 300_000, 600_000]; - -/** - * The portal file field name. - */ -const PORTAL_FILE_FIELD_NAME = "file"; - -const TUS_ENDPOINT = "/s5/upload/tus"; - -export interface HashProgressEvent { - bytes: number; - total: number; -} - -/** - * Custom upload options. - * - * @property [largeFileSize=32943040] - The size at which files are considered "large" and will be uploaded using the tus resumable upload protocol. This is the size of one chunk by default (32 mib). Note that this does not affect the actual size of chunks used by the protocol. - * @property [errorPages] - Defines a mapping of error codes and subfiles which are to be served in case we are serving the respective error code. All subfiles referred like this must be defined with absolute paths and must exist. - * @property [retryDelays=[0, 5_000, 15_000, 60_000, 300_000, 600_000]] - An array or undefined, indicating how many milliseconds should pass before the next attempt to uploading will be started after the transfer has been interrupted. The array's length indicates the maximum number of attempts. - * @property [tryFiles] - Allows us to set a list of potential subfiles to return in case the requested one does not exist or is a directory. Those subfiles might be listed with relative or absolute paths. If the path is absolute the file must exist. - */ -export type CustomUploadOptions = BaseCustomOptions & { - errorPages?: Record; - tryFiles?: string[]; - - // Large files. - largeFileSize?: number; - retryDelays?: number[]; - onUploadProgress?: (progressEvent: AxiosProgressEvent) => void; - onHashProgress?: (progressEvent: HashProgressEvent) => void; -}; - -export const DEFAULT_UPLOAD_OPTIONS = { - errorPages: { 404: "/404.html" }, - tryFiles: ["index.html"], - - // Large files. - largeFileSize: TUS_CHUNK_SIZE, - retryDelays: DEFAULT_TUS_RETRY_DELAYS, -} as CustomUploadOptions; - -export interface UploadResult { - cid: CID; -} - -/** - * Uploads a file to S5-net. - * - * @param this - S5Client - * @param file - The file to upload. - * @param [customOptions] - Additional settings that can optionally be set. - * @returns - The returned cid. - * @throws - Will throw if the request is successful but the upload response does not contain a complete response. - */ -export async function uploadFile( - this: S5Client, - file: File, - customOptions: CustomUploadOptions = {}, -): Promise { - const opts = { - ...DEFAULT_UPLOAD_OPTIONS, - ...this.clientOptions, - ...customOptions, - } as CustomUploadOptions; - - if (file.size < opts?.largeFileSize) { - return this.uploadSmallFile(file, opts); - } else { - return this.uploadLargeFile(file, opts); - } -} - -/** - * Uploads a small file to S5-net. - * - * @param this - S5Client - * @param file - The file to upload. - * @param [customOptions] - Additional settings that can optionally be set. - - * @returns UploadResult - The returned cid. - * @throws - Will throw if the request is successful but the upload response does not contain a complete response. - */ -export async function uploadSmallFile( - this: S5Client, - file: File, - customOptions: CustomUploadOptions, -): Promise { - const response = await this.uploadSmallFileRequest(file, customOptions); - - return { cid: CID.decode(response.cid) }; -} - -/** - * Makes a request to upload a small file to S5-net. - * - * @param this - S5Client - * @param file - The file to upload. - * @param [customOptions] - Additional settings that can optionally be set. - - * @returns PostS5UploadResult - The upload response. - */ -export async function uploadSmallFileRequest( - this: S5Client, - file: File, - customOptions: CustomUploadOptions = {}, -): Promise { - const config = optionsToConfig(this, DEFAULT_UPLOAD_OPTIONS, customOptions); - - file = ensureFileObjectConsistency(file); - - return postS5Upload( - { - file: file, - }, - config, - ); -} - -/* istanbul ignore next */ -/** - * Uploads a large file to S5-net using tus. - * - * @param this - S5Client - * @param file - The file to upload. - * @param [customOptions] - Additional settings that can optionally be set. - * @param [customOptions.endpointLargeUpload="/s5/upload/tus"] - The relative URL path of the portal endpoint to contact. - * @returns - The returned cid. - * @throws - Will throw if the request is successful but the upload response does not contain a complete response. - */ -export async function uploadLargeFile( - this: S5Client, - file: File, - customOptions: CustomUploadOptions = {}, -): Promise { - return await this.uploadLargeFileRequest(file, customOptions); -} - -/* istanbul ignore next */ -/** - * Makes a request to upload a file to S5-net. - * - * @param this - S5Client - * @param file - The file to upload. - * @param [customOptions] - Additional settings that can optionally be set. - * @returns - The upload response. - */ -export async function uploadLargeFileRequest( - this: S5Client, - file: File, - customOptions: CustomUploadOptions = {}, -): Promise { - const p = defer(); - - const options = await this.getTusOptions( - file, - { - onSuccess: async () => { - if (!upload.url) { - p.reject(new Error("'upload.url' was not set")); - return; - } - - p.resolve({ cid }); - }, - onError: (error: Error | DetailedError) => { - // Return error body rather than entire error. - const res = (error as DetailedError).originalResponse; - const newError = res ? new Error(res.getBody().trim()) || error : error; - p.reject(newError); - }, - }, - customOptions, - ); - const cid = CID.fromHash( - Multihash.fromBase64Url(options.metadata?.hash).fullBytes, - file.size, - CID_TYPES.RAW, - ); - - const upload = new Upload(file, options); - - return p.promise; -} - -export async function getTusOptions( - this: S5Client, - file: File, - tusOptions: Partial = {}, - customOptions: CustomUploadOptions = {}, -): Promise { - const config = optionsToConfig(this, DEFAULT_UPLOAD_OPTIONS, customOptions); - - // Validation. - const url = await buildRequestUrl(this, { - endpointPath: TUS_ENDPOINT, - }); - - file = ensureFileObjectConsistency(file); - - const hasher = blake3.create({}); - - const chunkSize = 1024 * 1024; - - let position = 0; - - while (position <= file.size) { - const chunk = file.slice(position, position + chunkSize); - hasher.update(new Uint8Array(await chunk.arrayBuffer())); - position += chunkSize; - customOptions.onHashProgress?.({ - bytes: position, - total: file.size, - }); - } - - const b3hash = hasher.digest(); - - const filename = new Multihash( - Buffer.concat([ - Buffer.alloc(1, CID_HASH_TYPES.BLAKE3), - Buffer.from(b3hash), - ]), - ).toBase64Url(); - - return { - endpoint: url, - metadata: { - hash: filename, - filename: filename, - filetype: file.type, - }, - headers: config.headers as any, - onBeforeRequest: function (req: HttpRequest) { - const xhr = req.getUnderlyingObject(); - xhr.withCredentials = true; - }, - ...tusOptions, - }; -} - -/** - * Uploads a directory to S5-net. - * - * @param this - S5Client - * @param directory - File objects to upload, indexed by their path strings. - * @param filename - The name of the directory. - * @param [customOptions] - Additional settings that can optionally be set. - * @param [customOptions.endpointPath="/s5/upload/directory"] - The relative URL path of the portal endpoint to contact. - * @returns - The returned cid. - * @throws - Will throw if the request is successful but the upload response does not contain a complete response. - */ -export async function uploadDirectory( - this: S5Client, - directory: Record, - filename: string, - customOptions: CustomUploadOptions = {}, -): Promise { - const response = await this.uploadDirectoryRequest( - directory, - filename, - customOptions, - ); - - return { cid: CID.decode(response.cid) }; -} - -/** - * Makes a request to upload a directory to S5-net. - * - * @param this - S5Client - * @param directory - File objects to upload, indexed by their path strings. - * @param filename - The name of the directory. - * @param [customOptions] - Additional settings that can optionally be set. - * @returns - The upload response. - * @throws - Will throw if the input filename is not a string. - */ -export async function uploadDirectoryRequest( - this: S5Client, - directory: Record, - filename: string, - customOptions: CustomUploadOptions = {}, -): Promise { - const config = optionsToConfig(this, DEFAULT_UPLOAD_OPTIONS, customOptions); - - const formData = new FormData(); - - for (const entry in directory) { - const file = ensureFileObjectConsistency(directory[entry]); - formData.append(entry, file, entry); - } - - const params = {} as PostS5UploadDirectoryParams; - - if (customOptions.tryFiles) { - params.tryFiles = customOptions.tryFiles; - } - if (customOptions.errorPages) { - params.errorPages = customOptions.errorPages; - } - - params.name = filename; - - /* - Hack to pass the data right since OpenAPI doesn't support variable file inputs without knowing the names ahead of time. - */ - config.data = formData; - - return postS5UploadDirectory({}, params, config); -} - -export async function uploadWebapp( - this: S5Client, - directory: Record, - customOptions: CustomUploadOptions = {}, -): Promise { - const response = await this.uploadWebappRequest(directory, customOptions); - - return { cid: CID.decode(response.cid) }; -} - -/** - * Makes a request to upload a directory to S5-net. - * @param this - S5Client - * @param directory - File objects to upload, indexed by their path strings. - * @param [customOptions] - Additional settings that can optionally be set. - * @param [customOptions.endpointPath] - The relative URL path of the portal endpoint to contact. - * @returns - The upload response. - * @throws - Will throw if the input filename is not a string. - */ -export async function uploadWebappRequest( - this: S5Client, - directory: Record, - customOptions: CustomUploadOptions = {}, -): Promise { - return this.uploadDirectoryRequest(directory, "webapp", customOptions); -} - -/** - * Sometimes file object might have had the type property defined manually with - * Object.defineProperty and some browsers (namely firefox) can have problems - * reading it after the file has been appended to form data. To overcome this, - * we recreate the file object using native File constructor with a type defined - * as a constructor argument. - * - * @param file - The input file. - * @returns - The processed file. - */ -function ensureFileObjectConsistency(file: File): File { - return new File([file], file.name, { type: getFileMimeType(file) }); -} - -/** - * convert a number to Buffer. - * - * @param value - File objects to upload, indexed by their path strings. - * @returns - The returned cid. - * @throws - Will throw if the request is successful but the upload response does not contain a complete response. - */ -function numberToBuffer(value: number) { - const view = Buffer.alloc(16); - let lastIndex = 15; - for (let index = 0; index <= 15; ++index) { - if (value % 256 !== 0) { - lastIndex = index; - } - view[index] = value % 256; - value = value >> 8; - } - return view.subarray(0, lastIndex + 1); -} - -function base64Decode(data) { - const paddedData = data.padEnd(Math.ceil(data.length / 4) * 4, "="); - - const base64 = paddedData.replace(/-/g, "+").replace(/_/g, "/"); - - return Buffer.from(base64, "base64"); -} diff --git a/src/number.ts b/src/number.ts new file mode 100644 index 0000000..ae6b593 --- /dev/null +++ b/src/number.ts @@ -0,0 +1,21 @@ +import { Buffer } from "buffer"; + +/** + * convert a number to Buffer. + * + * @param value - File objects to upload, indexed by their path strings. + * @returns - The returned cid. + * @throws - Will throw if the request is successful but the upload response does not contain a complete response. + */ +function numberToBuffer(value: number) { + const view = Buffer.alloc(16); + let lastIndex = 15; + for (let index = 0; index <= 15; ++index) { + if (value % 256 !== 0) { + lastIndex = index; + } + view[index] = value % 256; + value = value >> 8; + } + return view.subarray(0, lastIndex + 1); +} diff --git a/src/options/download.ts b/src/options/download.ts new file mode 100644 index 0000000..406c0b2 --- /dev/null +++ b/src/options/download.ts @@ -0,0 +1,28 @@ +import { ResponseType } from "axios"; +import { CustomClientOptions } from "../utils/options.js"; + +export type CustomDownloadOptions = CustomClientOptions & { + path?: string; + range?: string; + responseType?: ResponseType; +}; + +export type CustomGetMetadataOptions = CustomClientOptions & {}; + +/** + * The response for a get metadata request. + * + * @property metadata - The metadata in JSON format. + * @property portalUrl - The URL of the portal. + * @property cid - 46-character cid. + */ +export type MetadataResult = { + metadata: Record; +}; + +export const DEFAULT_DOWNLOAD_OPTIONS = { + range: undefined, + responseType: undefined, +} as CustomDownloadOptions; + +export const DEFAULT_GET_METADATA_OPTIONS = {}; diff --git a/src/options/registry.ts b/src/options/registry.ts new file mode 100644 index 0000000..b57ddab --- /dev/null +++ b/src/options/registry.ts @@ -0,0 +1,22 @@ +import { CustomClientOptions } from "../utils/options.js"; + +export const DEFAULT_GET_ENTRY_OPTIONS = {}; + +export const DEFAULT_SET_ENTRY_OPTIONS = { + endpointSetEntry: "/s5/registry", +}; + +export const DEFAULT_SUBSCRIBE_ENTRY_OPTIONS = { + endpointSubscribeEntry: "/s5/registry/subscription", +} satisfies CustomRegistryOptions; + +export const DEFAULT_PUBLISH_ENTRY_OPTIONS = { + endpointPublishEntry: "/s5/registry", +} satisfies CustomRegistryOptions; + +export type BaseCustomOptions = CustomClientOptions; + +export interface CustomRegistryOptions extends BaseCustomOptions { + endpointSubscribeEntry?: string; + endpointPublishEntry?: string; +} diff --git a/src/options/upload.ts b/src/options/upload.ts new file mode 100644 index 0000000..5090674 --- /dev/null +++ b/src/options/upload.ts @@ -0,0 +1,81 @@ +import { AxiosProgressEvent } from "axios"; +import { + DetailedError, + HttpRequest, + Upload, + UploadOptions, +} from "tus-js-client"; + +import { blake3 } from "@noble/hashes/blake3"; +import { Buffer } from "buffer"; + +import { getFileMimeType } from "../utils/file.js"; +import { S5Client } from "../client.js"; +import { CID, CID_HASH_TYPES, CID_TYPES } from "@lumeweb/libs5"; +import { + type BasicUploadResponse, + postS5Upload, + postS5UploadDirectory, + PostS5UploadDirectoryParams, + PostS5UploadResult, +} from "../generated/index.js"; +import { BaseCustomOptions } from "./registry.js"; +import { optionsToConfig } from "../utils/options.js"; +import { buildRequestUrl } from "../request.js"; +import defer from "p-defer"; +import { Multihash } from "@lumeweb/libs5/lib/multihash.js"; + +/** + * The tus chunk size is (4MiB - encryptionOverhead) * dataPieces, set as default. + */ +export const TUS_CHUNK_SIZE = (1 << 22) * 8; + +/** + * The retry delays, in ms. Data is stored in for up to 20 minutes, so the + * total delays should not exceed that length of time. + */ +const DEFAULT_TUS_RETRY_DELAYS = [0, 5_000, 15_000, 60_000, 300_000, 600_000]; + +/** + * The portal file field name. + */ +const PORTAL_FILE_FIELD_NAME = "file"; + +export const TUS_ENDPOINT = "/s5/upload/tus"; + +export interface HashProgressEvent { + bytes: number; + total: number; +} + +/** + * Custom upload options. + * + * @property [largeFileSize=32943040] - The size at which files are considered "large" and will be uploaded using the tus resumable upload protocol. This is the size of one chunk by default (32 mib). Note that this does not affect the actual size of chunks used by the protocol. + * @property [errorPages] - Defines a mapping of error codes and subfiles which are to be served in case we are serving the respective error code. All subfiles referred like this must be defined with absolute paths and must exist. + * @property [retryDelays=[0, 5_000, 15_000, 60_000, 300_000, 600_000]] - An array or undefined, indicating how many milliseconds should pass before the next attempt to uploading will be started after the transfer has been interrupted. The array's length indicates the maximum number of attempts. + * @property [tryFiles] - Allows us to set a list of potential subfiles to return in case the requested one does not exist or is a directory. Those subfiles might be listed with relative or absolute paths. If the path is absolute the file must exist. + */ +export type CustomUploadOptions = BaseCustomOptions & { + errorPages?: Record; + tryFiles?: string[]; + + // Large files. + largeFileSize?: number; + retryDelays?: number[]; + onUploadProgress?: (progressEvent: AxiosProgressEvent) => void; + onHashProgress?: (progressEvent: HashProgressEvent) => void; +}; + +export const DEFAULT_UPLOAD_OPTIONS = { + errorPages: { 404: "/404.html" }, + tryFiles: ["index.html"], + + // Large files. + largeFileSize: TUS_CHUNK_SIZE, + retryDelays: DEFAULT_TUS_RETRY_DELAYS, +} as CustomUploadOptions; + +export interface UploadResult { + cid: CID; +} diff --git a/src/utils/encoding.ts b/src/utils/encoding.ts new file mode 100644 index 0000000..6b78bba --- /dev/null +++ b/src/utils/encoding.ts @@ -0,0 +1,5 @@ +import { base64url } from "multiformats/bases/base64"; + +export const base64urlEncode = (d: Uint8Array) => + base64url.encode(d).substring(1); +export const base64urlDecode = (d: string) => base64url.decode(`u${d}`); diff --git a/src/utils/file.ts b/src/utils/file.ts index d7f3b74..3d8c5f7 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -22,3 +22,17 @@ export function getFileMimeType(file: File): string { } return ""; } + +/** + * Sometimes file object might have had the type property defined manually with + * Object.defineProperty and some browsers (namely firefox) can have problems + * reading it after the file has been appended to form data. To overcome this, + * we recreate the file object using native File constructor with a type defined + * as a constructor argument. + * + * @param file - The input file. + * @returns - The processed file. + */ +export function ensureFileObjectConsistency(file: File): File { + return new File([file], file.name, { type: getFileMimeType(file) }); +} diff --git a/src/utils/options.ts b/src/utils/options.ts index c55f36d..cf90fb8 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -1,5 +1,6 @@ import { AxiosHeaders, AxiosProgressEvent, AxiosRequestConfig } from "axios"; import { S5Client } from "../client.js"; +import { BaseCustomOptions, CustomRegistryOptions } from "#options/registry.js"; /** * Custom client options. @@ -21,8 +22,12 @@ export type CustomClientOptions = { export function optionsToConfig( client: S5Client, - def: CustomClientOptions, - ...options: CustomClientOptions[] + def: CustomClientOptions | BaseCustomOptions | CustomRegistryOptions, + ...options: ( + | CustomClientOptions + | BaseCustomOptions + | CustomRegistryOptions + )[] ): AxiosRequestConfig { const config: AxiosRequestConfig = {};