LumeWeb
/
s5-js
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: split off tus upload to getTusOptions so it can be reusable to external libraries

This commit is contained in:
Derrick Hammer 2024-03-17 14:06:10 -04:00
parent b4d1e0ec67
commit ba370250a2
Signed by: pcfreak30
GPG Key ID: C997C339BE476FF2
2 changed files with 100 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/client.ts
View File

@ -8,6 +8,7 @@ import {
uploadLargeFileRequest,
uploadWebapp,
uploadWebappRequest,
getTusOptions,
} from "./methods/upload.js";
import {
downloadBlob,
@ -53,6 +54,7 @@ export class S5Client {
getEntry = getEntry;
accountPins = accountPins;
// Download
getTusOptions = getTusOptions;
protected uploadSmallFile = uploadSmallFile;
protected uploadSmallFileRequest = uploadSmallFileRequest;
protected uploadLargeFile = uploadLargeFile;

161
src/methods/upload.ts
View File

@ -1,5 +1,10 @@
import { AxiosProgressEvent } from "axios";
import { DetailedError, HttpRequest, Upload } from "tus-js-client";
import {
DetailedError,
HttpRequest,
Upload,
UploadOptions,
} from "tus-js-client";
import { blake3 } from "@noble/hashes/blake3";
import { Buffer } from "buffer";
@ -17,6 +22,8 @@ import {
import { BaseCustomOptions } from "#methods/registry.js";
import { optionsToConfig } from "#utils/options.js";
import { buildRequestUrl } from "#request.js";
import defer from "p-defer";
import { Multihash } from "@lumeweb/libs5/lib/multihash.js";
/**
* The tus chunk size is (4MiB - encryptionOverhead) * dataPieces, set as default.
@ -173,6 +180,45 @@ export async function uploadLargeFileRequest(
file: File,
customOptions: CustomUploadOptions = {},
): Promise<UploadResult> {
const p = defer<UploadResult>();
const options = await this.getTusOptions(
file,
{
onSuccess: async () => {
if (!upload.url) {
p.reject(new Error("'upload.url' was not set"));
return;
}
p.resolve({ cid });
},
onError: (error: Error | DetailedError) => {
// Return error body rather than entire error.
const res = (error as DetailedError).originalResponse;
const newError = res ? new Error(res.getBody().trim()) || error : error;
p.reject(newError);
},
},
customOptions,
);
const cid = CID.fromHash(
Multihash.fromBase64Url(<string>options.metadata?.hash).fullBytes,
file.size,
CID_TYPES.RAW,
);
const upload = new Upload(file, options);
return p.promise;
}
export async function getTusOptions(
this: S5Client,
file: File,
tusOptions: Partial<UploadOptions> = {},
customOptions: CustomUploadOptions = {},
): Promise<UploadOptions> {
const config = optionsToConfig(this, DEFAULT_UPLOAD_OPTIONS, customOptions);
// Validation.
@ -182,104 +228,41 @@ export async function uploadLargeFileRequest(
file = ensureFileObjectConsistency(file);
const onProgress =
config.onUploadProgress &&
function (bytesSent: number, bytesTotal: number) {
const progress = bytesSent / bytesTotal;
// @ts-expect-error TS complains.
config.onUploadProgress(progress, {
loaded: bytesSent,
total: bytesTotal,
});
};
const hasher = blake3.create({});
const chunkSize = 1024 * 1024;
let position = 0;
while (position <= file.size) {
const chunk = file.slice(position, position + chunkSize);
hasher.update(new Uint8Array(await chunk.arrayBuffer()));
position += chunkSize;
}
const b3hash = hasher.digest();
const hash = Buffer.concat([
const filename = new Multihash(
Buffer.concat([
Buffer.alloc(1, CID_HASH_TYPES.BLAKE3),
Buffer.from(b3hash),
]);
const cid = Buffer.concat([
Buffer.alloc(1, CID_TYPES.RAW),
hash,
numberToBuffer(file.size),
]);
]),
).toBase64Url();
/**
* convert a number to Buffer.
*
* @param value - File objects to upload, indexed by their path strings.
* @returns - The returned cid.
* @throws - Will throw if the request is successful but the upload response does not contain a complete response.
*/
function numberToBuffer(value: number) {
const view = Buffer.alloc(16);
let lastIndex = 15;
for (let index = 0; index <= 15; ++index) {
if (value % 256 !== 0) {
lastIndex = index;
}
view[index] = value % 256;
value = value >> 8;
}
return view.subarray(0, lastIndex + 1);
}
return new Promise((resolve, reject) => {
const filename = hash
.toString("base64")
.replace(/\+/g, "-")
.replace(/\//g, "_")
.replace("=", "");
const tusOpts = {
return {
endpoint: url,
// retryDelays: opts.retryDelays,
metadata: {
hash: filename,
filename: filename,
filetype: file.type,
},
config: config.headers,
onProgress,
headers: config.headers as any,
onBeforeRequest: function (req: HttpRequest) {
const xhr = req.getUnderlyingObject();
xhr.withCredentials = true;
},
onError: (error: Error | DetailedError) => {
// Return error body rather than entire error.
const res = (error as DetailedError).originalResponse;
const newError = res ? new Error(res.getBody().trim()) || error : error;
reject(newError);
},
onSuccess: async () => {
if (!upload.url) {
reject(new Error("'upload.url' was not set"));
return;
}
const resCid =
"u" +
cid
.toString("base64")
.replace(/\+/g, "-")
.replace(/\//g, "_")
.replace("=", "");
resolve({ cid: CID.decode(resCid) });
},
...tusOptions,
};
const upload = new Upload(file, tusOpts);
upload.start();
});
}
/**
@ -392,3 +375,31 @@ export async function uploadWebappRequest(
function ensureFileObjectConsistency(file: File): File {
return new File([file], file.name, { type: getFileMimeType(file) });
}
/**
* convert a number to Buffer.
*
* @param value - File objects to upload, indexed by their path strings.
* @returns - The returned cid.
* @throws - Will throw if the request is successful but the upload response does not contain a complete response.
*/
function numberToBuffer(value: number) {
const view = Buffer.alloc(16);
let lastIndex = 15;
for (let index = 0; index <= 15; ++index) {
if (value % 256 !== 0) {
lastIndex = index;
}
view[index] = value % 256;
value = value >> 8;
}
return view.subarray(0, lastIndex + 1);
}
function base64Decode(data) {
const paddedData = data.padEnd(Math.ceil(data.length / 4) * 4, "=");
const base64 = paddedData.replace(/-/g, "+").replace(/_/g, "/");
return Buffer.from(base64, "base64");
}