Merge pull request #1029 from SkynetLabs/sequential-uploads
Sequential uploads on siasky.net website
This commit is contained in:
Ivaylo Novakov
GitHub
commit
a1ce9b4649
|
|
@ -43,6 +43,7 @@
|
|||
"http-status-codes": "^2.1.4",
|
||||
"jsonp": "^0.2.1",
|
||||
"ms": "^2.1.2",
|
||||
"nanoid": "^3.1.23",
|
||||
"normalize.css": "^8.0.1",
|
||||
"path-browserify": "^1.0.1",
|
||||
"polished": "^4.1.3",
|
||||
|
|
|
|||
|
|
@ -1,28 +1,18 @@
|
|||
import * as React from "react";
|
||||
import classnames from "classnames";
|
||||
import { Add, Cloud, ArrowUpCircle, Error, CheckCircle, Unlock, Info, ProgressRound } from "../Icons";
|
||||
import bytes from "bytes";
|
||||
import { Add, Cloud, Unlock, Info } from "../Icons";
|
||||
import classNames from "classnames";
|
||||
import { getReasonPhrase, StatusCodes } from "http-status-codes";
|
||||
import copy from "copy-text-to-clipboard";
|
||||
import path from "path-browserify";
|
||||
import { useDropzone } from "react-dropzone";
|
||||
import { SkynetClient } from "skynet-js";
|
||||
import { useTimeoutFn } from "react-use";
|
||||
import ms from "ms";
|
||||
import { nanoid } from "nanoid";
|
||||
import useAuthenticatedStatus from "../../services/useAuthenticatedStatus";
|
||||
import Link from "../Link";
|
||||
import UploaderElement from "./UploaderElement";
|
||||
|
||||
const MAX_PARALLEL_UPLOADS = 1;
|
||||
|
||||
const getFilePath = (file) => file.webkitRelativePath || file.path || file.name;
|
||||
|
||||
const getRelativeFilePath = (file) => {
|
||||
const filePath = getFilePath(file);
|
||||
const { root, dir, base } = path.parse(filePath);
|
||||
const relative = path.normalize(dir).slice(root.length).split(path.sep).slice(1);
|
||||
|
||||
return path.join(...relative, base);
|
||||
};
|
||||
|
||||
const getRootDirectory = (file) => {
|
||||
const filePath = getFilePath(file);
|
||||
const { root, dir } = path.parse(filePath);
|
||||
|
|
@ -30,43 +20,6 @@ const getRootDirectory = (file) => {
|
|||
return path.normalize(dir).slice(root.length).split(path.sep)[0];
|
||||
};
|
||||
|
||||
const createUploadErrorMessage = (error) => {
|
||||
// The request was made and the server responded with a status code that falls out of the range of 2xx
|
||||
if (error.response) {
|
||||
if (error.response.data.message) {
|
||||
return `Upload failed with error: ${error.response.data.message}`;
|
||||
}
|
||||
|
||||
const statusCode = error.response.status;
|
||||
const statusText = getReasonPhrase(error.response.status);
|
||||
|
||||
return `Upload failed, our server received your request but failed with status code: ${statusCode} ${statusText}`;
|
||||
}
|
||||
|
||||
// The request was made but no response was received. The best we can do is detect whether browser is online.
|
||||
// This will be triggered mostly if the server is offline or misconfigured and doesn't respond to valid request.
|
||||
if (error.request) {
|
||||
if (!navigator.onLine) {
|
||||
return "You are offline, please connect to the internet and try again";
|
||||
}
|
||||
|
||||
// TODO: We should add a note "our team has been notified" and have some kind of notification with this error.
|
||||
return "Server failed to respond to your request, please try again later.";
|
||||
}
|
||||
|
||||
// Match the error message to a message returned by TUS when upload exceeds max file size
|
||||
const matchTusMaxFileSizeError = error.message.match(/upload exceeds maximum size: \d+ > (?<limit>\d+)/);
|
||||
|
||||
if (matchTusMaxFileSizeError) {
|
||||
return `File exceeds size limit of ${bytes(parseInt(matchTusMaxFileSizeError.groups.limit, 10))}`;
|
||||
}
|
||||
|
||||
// TODO: We should add a note "our team has been notified" and have some kind of notification with this error.
|
||||
return `Critical error, please refresh the application and try again. ${error.message}`;
|
||||
};
|
||||
|
||||
const client = new SkynetClient(process.env.GATSBY_API_URL);
|
||||
|
||||
const RegistrationLink = () => (
|
||||
<Link
|
||||
href="https://account.siasky.net/auth/registration"
|
||||
|
|
@ -76,149 +29,46 @@ const RegistrationLink = () => (
|
|||
</Link>
|
||||
);
|
||||
|
||||
const UploadElement = ({ file, status, error, url = "", progress = 0 }) => {
|
||||
const [copied, setCopied] = React.useState(false);
|
||||
const [, , reset] = useTimeoutFn(() => setCopied(false), ms("3 seconds"));
|
||||
|
||||
const handleCopy = (url) => {
|
||||
copy(url);
|
||||
setCopied(true);
|
||||
reset();
|
||||
};
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div className="flex items-center">
|
||||
{status === "uploading" && <ArrowUpCircle className="flex-shrink-0" />}
|
||||
{status === "processing" && <ProgressRound className="flex-shrink-0 animate-spin" />}
|
||||
{status === "complete" && <CheckCircle className="flex-shrink-0" />}
|
||||
{status === "error" && <Error className="flex-shrink-0 fill-current text-error" />}
|
||||
<div className="flex flex-col flex-grow ml-3 overflow-hidden">
|
||||
<div className="text-palette-600 text-sm font-light">{file.name}</div>
|
||||
<div className="flex justify-between text-palette-400 text-xs space-x-2">
|
||||
<div className="font-content truncate">
|
||||
{status === "uploading" && (
|
||||
<span className="tabular-nums">
|
||||
Uploading {bytes(file.size * progress)} of {bytes(file.size)}
|
||||
</span>
|
||||
)}
|
||||
|
||||
{status === "processing" && <span className="text-palette-300">Processing...</span>}
|
||||
|
||||
{status === "complete" && (
|
||||
<Link href={url} className="hover:text-primary transition-colors duration-200">
|
||||
{url}
|
||||
</Link>
|
||||
)}
|
||||
|
||||
{status === "error" && error && <span className="text-error">{error}</span>}
|
||||
</div>
|
||||
<div>
|
||||
{status === "uploading" && (
|
||||
<span className="uppercase tabular-nums">
|
||||
{Math.floor(progress * 100)}%<span className="hidden desktop:inline"> completed</span>
|
||||
</span>
|
||||
)}
|
||||
{status === "processing" && <span className="uppercase text-palette-300">Wait</span>}
|
||||
{status === "complete" && (
|
||||
<button
|
||||
className="uppercase hover:text-primary transition-colors duration-200"
|
||||
onClick={() => handleCopy(url)}
|
||||
>
|
||||
<span className={classnames({ hidden: copied, "hidden desktop:inline": !copied })}>Copy link</span>
|
||||
<span className={classnames({ hidden: copied, "inline desktop:hidden": !copied })}>Copy</span>
|
||||
<span className={classnames({ hidden: !copied })}>Copied</span>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
className={classnames("flex bg-palette-200 mt-1", {
|
||||
"bg-error-dashed opacity-20": status === "error",
|
||||
"bg-primary-dashed move opacity-20": status === "processing",
|
||||
})}
|
||||
style={{ height: "5px" }}
|
||||
>
|
||||
<div
|
||||
style={{ width: `${Math.floor(progress * 100)}%` }}
|
||||
className={classnames("bg-primary", { hidden: status === "processing" || status === "error" })}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const Uploader = () => {
|
||||
const [mode, setMode] = React.useState("file");
|
||||
const [files, setFiles] = React.useState([]);
|
||||
const [uploads, setUploads] = React.useState([]);
|
||||
|
||||
const { data: authenticationStatus } = useAuthenticatedStatus();
|
||||
const authenticated = authenticationStatus?.authenticated ?? false;
|
||||
|
||||
const handleDrop = async (acceptedFiles) => {
|
||||
if (mode === "directory" && acceptedFiles.length) {
|
||||
const rootDir = getRootDirectory(acceptedFiles[0]); // get the file path from the first file
|
||||
const onUploadStateChange = React.useCallback((id, state) => {
|
||||
setUploads((uploads) => {
|
||||
const index = uploads.findIndex((upload) => upload.id === id);
|
||||
|
||||
acceptedFiles = [{ name: rootDir, directory: true, files: acceptedFiles }];
|
||||
return [...uploads.slice(0, index), { ...uploads[index], ...state }, ...uploads.slice(index + 1)];
|
||||
});
|
||||
}, []);
|
||||
|
||||
const handleDrop = async (files) => {
|
||||
if (mode === "directory" && files.length) {
|
||||
const name = getRootDirectory(files[0]); // get the file path from the first file
|
||||
|
||||
files = [{ name, files }];
|
||||
}
|
||||
|
||||
setFiles((previousFiles) => [...acceptedFiles.map((file) => ({ file, status: "uploading" })), ...previousFiles]);
|
||||
|
||||
const onFileStateChange = (file, state) => {
|
||||
setFiles((previousFiles) => {
|
||||
const index = previousFiles.findIndex((f) => f.file === file);
|
||||
|
||||
return [
|
||||
...previousFiles.slice(0, index),
|
||||
{
|
||||
...previousFiles[index],
|
||||
...state,
|
||||
},
|
||||
...previousFiles.slice(index + 1),
|
||||
];
|
||||
});
|
||||
};
|
||||
|
||||
acceptedFiles.forEach((file) => {
|
||||
const onUploadProgress = (progress) => {
|
||||
const status = progress === 1 ? "processing" : "uploading";
|
||||
|
||||
onFileStateChange(file, { status, progress });
|
||||
};
|
||||
|
||||
const upload = async () => {
|
||||
try {
|
||||
let response;
|
||||
|
||||
if (file.directory) {
|
||||
const directory = file.files.reduce((acc, file) => ({ ...acc, [getRelativeFilePath(file)]: file }), {});
|
||||
|
||||
response = await client.uploadDirectory(directory, encodeURIComponent(file.name), { onUploadProgress });
|
||||
} else {
|
||||
response = await client.uploadFile(file, { onUploadProgress });
|
||||
}
|
||||
|
||||
const url = await client.getSkylinkUrl(response.skylink, { subdomain: mode === "directory" });
|
||||
|
||||
onFileStateChange(file, { status: "complete", url });
|
||||
} catch (error) {
|
||||
if (error.response && error.response.status === StatusCodes.TOO_MANY_REQUESTS) {
|
||||
onFileStateChange(file, { progress: -1 });
|
||||
|
||||
return new Promise((resolve) => setTimeout(() => resolve(upload()), 3000));
|
||||
}
|
||||
|
||||
onFileStateChange(file, { status: "error", error: createUploadErrorMessage(error) });
|
||||
}
|
||||
};
|
||||
|
||||
upload();
|
||||
});
|
||||
setUploads((uploads) => [...files.map((file) => ({ id: nanoid(), file, mode, status: "enqueued" })), ...uploads]);
|
||||
};
|
||||
|
||||
React.useEffect(() => {
|
||||
const enqueued = uploads.filter(({ status }) => status === "enqueued");
|
||||
const uploading = uploads.filter(({ status }) => ["uploading", "processing", "retrying"].includes(status));
|
||||
const queue = enqueued.slice(0, MAX_PARALLEL_UPLOADS - uploading.length).map(({ id }) => id);
|
||||
|
||||
if (queue.length && uploading.length < MAX_PARALLEL_UPLOADS) {
|
||||
setUploads((uploads) => {
|
||||
return uploads.map((upload) => {
|
||||
if (queue.includes(upload.id)) return { ...upload, status: "uploading" };
|
||||
return upload;
|
||||
});
|
||||
});
|
||||
}
|
||||
}, [uploads]);
|
||||
|
||||
const { getRootProps, getInputProps, isDragActive, inputRef } = useDropzone({ onDrop: handleDrop });
|
||||
const inputElement = inputRef.current;
|
||||
|
||||
|
|
@ -274,7 +124,7 @@ const Uploader = () => {
|
|||
}
|
||||
)}
|
||||
>
|
||||
{files.length === 0 && <Cloud />}
|
||||
{uploads.length === 0 && <Cloud />}
|
||||
<h4 className="font-light text-palette-600 text-lg mt-2 text-center">
|
||||
{mode === "file" && <span>Add or drop your files here to pin to Skynet</span>}
|
||||
{mode === "directory" && <span>Drop any folder with an index.html file to deploy to Skynet</span>}
|
||||
|
|
@ -287,10 +137,10 @@ const Uploader = () => {
|
|||
</div>
|
||||
</div>
|
||||
|
||||
{files.length > 0 && (
|
||||
{uploads.length > 0 && (
|
||||
<div className="flex flex-col space-y-5 px-4 py-10 desktop:p-14">
|
||||
{files.map((file, index) => (
|
||||
<UploadElement key={index} {...file} />
|
||||
{uploads.map((upload) => (
|
||||
<UploaderElement key={upload.id} onUploadStateChange={onUploadStateChange} upload={upload} />
|
||||
))}
|
||||
|
||||
{!authenticated && (
|
||||
|
|
@ -315,7 +165,7 @@ const Uploader = () => {
|
|||
)}
|
||||
</div>
|
||||
|
||||
{files.length === 0 && !authenticated && (
|
||||
{uploads.length === 0 && !authenticated && (
|
||||
<div className="z-0 relative flex flex-col items-center space-y-1 mt-10">
|
||||
<Unlock />
|
||||
<p className="text-sm font-light text-palette-600">
|
||||
|
|
|
|||
|
|
@ -0,0 +1,180 @@
|
|||
import * as React from "react";
|
||||
import classnames from "classnames";
|
||||
import { ArrowUpCircle, Error, CheckCircle, ProgressRound } from "../Icons";
|
||||
import bytes from "bytes";
|
||||
import { getReasonPhrase, StatusCodes } from "http-status-codes";
|
||||
import copy from "copy-text-to-clipboard";
|
||||
import path from "path-browserify";
|
||||
import { SkynetClient } from "skynet-js";
|
||||
import { useTimeoutFn } from "react-use";
|
||||
import ms from "ms";
|
||||
import Link from "../Link";
|
||||
|
||||
const getFilePath = (file) => file.webkitRelativePath || file.path || file.name;
|
||||
|
||||
const getRelativeFilePath = (file) => {
|
||||
const filePath = getFilePath(file);
|
||||
const { root, dir, base } = path.parse(filePath);
|
||||
const relative = path.normalize(dir).slice(root.length).split(path.sep).slice(1);
|
||||
|
||||
return path.join(...relative, base);
|
||||
};
|
||||
|
||||
const createUploadErrorMessage = (error) => {
|
||||
// The request was made and the server responded with a status code that falls out of the range of 2xx
|
||||
if (error.response) {
|
||||
if (error.response.data.message) {
|
||||
return `Upload failed with error: ${error.response.data.message}`;
|
||||
}
|
||||
|
||||
const statusCode = error.response.status;
|
||||
const statusText = getReasonPhrase(error.response.status);
|
||||
|
||||
return `Upload failed, our server received your request but failed with status code: ${statusCode} ${statusText}`;
|
||||
}
|
||||
|
||||
// The request was made but no response was received. The best we can do is detect whether browser is online.
|
||||
// This will be triggered mostly if the server is offline or misconfigured and doesn't respond to valid request.
|
||||
if (error.request) {
|
||||
if (!navigator.onLine) {
|
||||
return "You are offline, please connect to the internet and try again";
|
||||
}
|
||||
|
||||
// TODO: We should add a note "our team has been notified" and have some kind of notification with this error.
|
||||
return "Server failed to respond to your request, please try again later.";
|
||||
}
|
||||
|
||||
// Match the error message to a message returned by TUS when upload exceeds max file size
|
||||
const matchTusMaxFileSizeError = error.message.match(/upload exceeds maximum size: \d+ > (?<limit>\d+)/);
|
||||
|
||||
if (matchTusMaxFileSizeError) {
|
||||
return `File exceeds size limit of ${bytes(parseInt(matchTusMaxFileSizeError.groups.limit, 10))}`;
|
||||
}
|
||||
|
||||
// TODO: We should add a note "our team has been notified" and have some kind of notification with this error.
|
||||
return `Critical error, please refresh the application and try again. ${error.message}`;
|
||||
};
|
||||
|
||||
const client = new SkynetClient(process.env.GATSBY_API_URL);
|
||||
|
||||
export default function UploaderElement({ onUploadStateChange, upload }) {
|
||||
const [copied, setCopied] = React.useState(false);
|
||||
const [, , reset] = useTimeoutFn(() => setCopied(false), ms("3 seconds"));
|
||||
const [retryTimeout, setRetryTimeout] = React.useState(ms("3 seconds")); // retry delay after "429: TOO_MANY_REQUESTS"
|
||||
|
||||
const handleCopy = (url) => {
|
||||
copy(url);
|
||||
setCopied(true);
|
||||
reset();
|
||||
};
|
||||
|
||||
React.useEffect(() => {
|
||||
if (upload.status === "uploading" && !upload.startedTime) {
|
||||
onUploadStateChange(upload.id, { startedTime: Date.now() });
|
||||
|
||||
(async () => {
|
||||
const onUploadProgress = (progress) => {
|
||||
const status = progress === 1 ? "processing" : "uploading";
|
||||
|
||||
onUploadStateChange(upload.id, { status, progress });
|
||||
};
|
||||
|
||||
try {
|
||||
let response;
|
||||
|
||||
if (upload.mode === "directory") {
|
||||
const files = upload.file.files;
|
||||
const directory = files.reduce((acc, file) => ({ ...acc, [getRelativeFilePath(file)]: file }), {});
|
||||
const name = encodeURIComponent(upload.name);
|
||||
|
||||
response = await client.uploadDirectory(directory, name, { onUploadProgress });
|
||||
} else {
|
||||
response = await client.uploadFile(upload.file, { onUploadProgress });
|
||||
}
|
||||
|
||||
const url = await client.getSkylinkUrl(response.skylink, { subdomain: upload.mode === "directory" });
|
||||
|
||||
onUploadStateChange(upload.id, { status: "complete", url });
|
||||
} catch (error) {
|
||||
if (error?.response?.status === StatusCodes.TOO_MANY_REQUESTS) {
|
||||
onUploadStateChange(upload.id, { status: "retrying", progress: 0 });
|
||||
|
||||
setTimeout(() => {
|
||||
onUploadStateChange(upload.id, { status: "enqueued", startedTime: null });
|
||||
setRetryTimeout((timeout) => timeout * 2); // increase timeout on next retry
|
||||
}, retryTimeout);
|
||||
} else {
|
||||
onUploadStateChange(upload.id, { status: "error", error: createUploadErrorMessage(error) });
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
}, [onUploadStateChange, upload, retryTimeout]);
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div className="flex items-center">
|
||||
{upload.status === "enqueued" && <ArrowUpCircle className="flex-shrink-0 fill-current text-palette-300" />}
|
||||
{upload.status === "retrying" && <ArrowUpCircle className="flex-shrink-0" />}
|
||||
{upload.status === "uploading" && <ArrowUpCircle className="flex-shrink-0" />}
|
||||
{upload.status === "processing" && <ProgressRound className="flex-shrink-0 animate-spin" />}
|
||||
{upload.status === "complete" && <CheckCircle className="flex-shrink-0" />}
|
||||
{upload.status === "error" && <Error className="flex-shrink-0 fill-current text-error" />}
|
||||
<div className="flex flex-col flex-grow ml-3 overflow-hidden">
|
||||
<div className="text-palette-600 text-sm font-light">{upload.file.name}</div>
|
||||
<div className="flex justify-between text-palette-400 text-xs space-x-2">
|
||||
<div className="font-content truncate">
|
||||
{upload.status === "uploading" && (
|
||||
<span className="tabular-nums">
|
||||
Uploading {bytes(upload.file.size * upload.progress)} of {bytes(upload.file.size)}
|
||||
</span>
|
||||
)}
|
||||
{upload.status === "enqueued" && <span className="text-palette-300">Upload in queue, please wait</span>}
|
||||
{upload.status === "processing" && <span className="text-palette-300">Processing...</span>}
|
||||
{upload.status === "complete" && (
|
||||
<Link href={upload.url} className="hover:text-primary transition-colors duration-200">
|
||||
{upload.url}
|
||||
</Link>
|
||||
)}
|
||||
{upload.status === "error" && upload.error && <span className="text-error">{upload.error}</span>}
|
||||
{upload.status === "retrying" && (
|
||||
<span>Too many parallel requests, retrying in {retryTimeout / 1000} seconds</span>
|
||||
)}
|
||||
</div>
|
||||
<div>
|
||||
{upload.status === "uploading" && (
|
||||
<span className="uppercase tabular-nums">
|
||||
{Math.floor(upload.progress * 100)}%<span className="hidden desktop:inline"> completed</span>
|
||||
</span>
|
||||
)}
|
||||
{upload.status === "processing" && <span className="uppercase text-palette-300">Wait</span>}
|
||||
{upload.status === "complete" && (
|
||||
<button
|
||||
className="uppercase hover:text-primary transition-colors duration-200"
|
||||
onClick={() => handleCopy(upload.url)}
|
||||
>
|
||||
<span className={classnames({ hidden: copied, "hidden desktop:inline": !copied })}>Copy link</span>
|
||||
<span className={classnames({ hidden: copied, "inline desktop:hidden": !copied })}>Copy</span>
|
||||
<span className={classnames({ hidden: !copied })}>Copied</span>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
className={classnames("flex bg-palette-200 mt-1", {
|
||||
"bg-error-dashed opacity-20": upload.status === "error",
|
||||
"bg-primary-dashed move opacity-20": upload.status === "processing",
|
||||
})}
|
||||
style={{ height: "5px" }}
|
||||
>
|
||||
<div
|
||||
style={{ width: `${Math.floor(upload.progress * 100)}%` }}
|
||||
className={classnames("bg-primary", { hidden: upload.status === "processing" || upload.status === "error" })}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue