Merge pull request #1029 from SkynetLabs/sequential-uploads

Sequential uploads on siasky.net website
This commit is contained in:
Ivaylo Novakov 2021-08-04 11:22:58 +02:00 committed by GitHub
commit a1ce9b4649
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 220 additions and 189 deletions

View File

@ -43,6 +43,7 @@
"http-status-codes": "^2.1.4", "http-status-codes": "^2.1.4",
"jsonp": "^0.2.1", "jsonp": "^0.2.1",
"ms": "^2.1.2", "ms": "^2.1.2",
"nanoid": "^3.1.23",
"normalize.css": "^8.0.1", "normalize.css": "^8.0.1",
"path-browserify": "^1.0.1", "path-browserify": "^1.0.1",
"polished": "^4.1.3", "polished": "^4.1.3",

View File

@ -1,28 +1,18 @@
import * as React from "react"; import * as React from "react";
import classnames from "classnames"; import classnames from "classnames";
import { Add, Cloud, ArrowUpCircle, Error, CheckCircle, Unlock, Info, ProgressRound } from "../Icons"; import { Add, Cloud, Unlock, Info } from "../Icons";
import bytes from "bytes";
import classNames from "classnames"; import classNames from "classnames";
import { getReasonPhrase, StatusCodes } from "http-status-codes";
import copy from "copy-text-to-clipboard";
import path from "path-browserify"; import path from "path-browserify";
import { useDropzone } from "react-dropzone"; import { useDropzone } from "react-dropzone";
import { SkynetClient } from "skynet-js"; import { nanoid } from "nanoid";
import { useTimeoutFn } from "react-use";
import ms from "ms";
import useAuthenticatedStatus from "../../services/useAuthenticatedStatus"; import useAuthenticatedStatus from "../../services/useAuthenticatedStatus";
import Link from "../Link"; import Link from "../Link";
import UploaderElement from "./UploaderElement";
const MAX_PARALLEL_UPLOADS = 1;
const getFilePath = (file) => file.webkitRelativePath || file.path || file.name; const getFilePath = (file) => file.webkitRelativePath || file.path || file.name;
const getRelativeFilePath = (file) => {
const filePath = getFilePath(file);
const { root, dir, base } = path.parse(filePath);
const relative = path.normalize(dir).slice(root.length).split(path.sep).slice(1);
return path.join(...relative, base);
};
const getRootDirectory = (file) => { const getRootDirectory = (file) => {
const filePath = getFilePath(file); const filePath = getFilePath(file);
const { root, dir } = path.parse(filePath); const { root, dir } = path.parse(filePath);
@ -30,43 +20,6 @@ const getRootDirectory = (file) => {
return path.normalize(dir).slice(root.length).split(path.sep)[0]; return path.normalize(dir).slice(root.length).split(path.sep)[0];
}; };
const createUploadErrorMessage = (error) => {
// The request was made and the server responded with a status code that falls out of the range of 2xx
if (error.response) {
if (error.response.data.message) {
return `Upload failed with error: ${error.response.data.message}`;
}
const statusCode = error.response.status;
const statusText = getReasonPhrase(error.response.status);
return `Upload failed, our server received your request but failed with status code: ${statusCode} ${statusText}`;
}
// The request was made but no response was received. The best we can do is detect whether browser is online.
// This will be triggered mostly if the server is offline or misconfigured and doesn't respond to valid request.
if (error.request) {
if (!navigator.onLine) {
return "You are offline, please connect to the internet and try again";
}
// TODO: We should add a note "our team has been notified" and have some kind of notification with this error.
return "Server failed to respond to your request, please try again later.";
}
// Match the error message to a message returned by TUS when upload exceeds max file size
const matchTusMaxFileSizeError = error.message.match(/upload exceeds maximum size: \d+ > (?<limit>\d+)/);
if (matchTusMaxFileSizeError) {
return `File exceeds size limit of ${bytes(parseInt(matchTusMaxFileSizeError.groups.limit, 10))}`;
}
// TODO: We should add a note "our team has been notified" and have some kind of notification with this error.
return `Critical error, please refresh the application and try again. ${error.message}`;
};
const client = new SkynetClient(process.env.GATSBY_API_URL);
const RegistrationLink = () => ( const RegistrationLink = () => (
<Link <Link
href="https://account.siasky.net/auth/registration" href="https://account.siasky.net/auth/registration"
@ -76,148 +29,45 @@ const RegistrationLink = () => (
</Link> </Link>
); );
const UploadElement = ({ file, status, error, url = "", progress = 0 }) => {
const [copied, setCopied] = React.useState(false);
const [, , reset] = useTimeoutFn(() => setCopied(false), ms("3 seconds"));
const handleCopy = (url) => {
copy(url);
setCopied(true);
reset();
};
return (
<div>
<div className="flex items-center">
{status === "uploading" && <ArrowUpCircle className="flex-shrink-0" />}
{status === "processing" && <ProgressRound className="flex-shrink-0 animate-spin" />}
{status === "complete" && <CheckCircle className="flex-shrink-0" />}
{status === "error" && <Error className="flex-shrink-0 fill-current text-error" />}
<div className="flex flex-col flex-grow ml-3 overflow-hidden">
<div className="text-palette-600 text-sm font-light">{file.name}</div>
<div className="flex justify-between text-palette-400 text-xs space-x-2">
<div className="font-content truncate">
{status === "uploading" && (
<span className="tabular-nums">
Uploading {bytes(file.size * progress)} of {bytes(file.size)}
</span>
)}
{status === "processing" && <span className="text-palette-300">Processing...</span>}
{status === "complete" && (
<Link href={url} className="hover:text-primary transition-colors duration-200">
{url}
</Link>
)}
{status === "error" && error && <span className="text-error">{error}</span>}
</div>
<div>
{status === "uploading" && (
<span className="uppercase tabular-nums">
{Math.floor(progress * 100)}%<span className="hidden desktop:inline"> completed</span>
</span>
)}
{status === "processing" && <span className="uppercase text-palette-300">Wait</span>}
{status === "complete" && (
<button
className="uppercase hover:text-primary transition-colors duration-200"
onClick={() => handleCopy(url)}
>
<span className={classnames({ hidden: copied, "hidden desktop:inline": !copied })}>Copy link</span>
<span className={classnames({ hidden: copied, "inline desktop:hidden": !copied })}>Copy</span>
<span className={classnames({ hidden: !copied })}>Copied</span>
</button>
)}
</div>
</div>
</div>
</div>
<div
className={classnames("flex bg-palette-200 mt-1", {
"bg-error-dashed opacity-20": status === "error",
"bg-primary-dashed move opacity-20": status === "processing",
})}
style={{ height: "5px" }}
>
<div
style={{ width: `${Math.floor(progress * 100)}%` }}
className={classnames("bg-primary", { hidden: status === "processing" || status === "error" })}
/>
</div>
</div>
);
};
const Uploader = () => { const Uploader = () => {
const [mode, setMode] = React.useState("file"); const [mode, setMode] = React.useState("file");
const [files, setFiles] = React.useState([]); const [uploads, setUploads] = React.useState([]);
const { data: authenticationStatus } = useAuthenticatedStatus(); const { data: authenticationStatus } = useAuthenticatedStatus();
const authenticated = authenticationStatus?.authenticated ?? false; const authenticated = authenticationStatus?.authenticated ?? false;
const handleDrop = async (acceptedFiles) => { const onUploadStateChange = React.useCallback((id, state) => {
if (mode === "directory" && acceptedFiles.length) { setUploads((uploads) => {
const rootDir = getRootDirectory(acceptedFiles[0]); // get the file path from the first file const index = uploads.findIndex((upload) => upload.id === id);
acceptedFiles = [{ name: rootDir, directory: true, files: acceptedFiles }]; return [...uploads.slice(0, index), { ...uploads[index], ...state }, ...uploads.slice(index + 1)];
}
setFiles((previousFiles) => [...acceptedFiles.map((file) => ({ file, status: "uploading" })), ...previousFiles]);
const onFileStateChange = (file, state) => {
setFiles((previousFiles) => {
const index = previousFiles.findIndex((f) => f.file === file);
return [
...previousFiles.slice(0, index),
{
...previousFiles[index],
...state,
},
...previousFiles.slice(index + 1),
];
}); });
}; }, []);
acceptedFiles.forEach((file) => { const handleDrop = async (files) => {
const onUploadProgress = (progress) => { if (mode === "directory" && files.length) {
const status = progress === 1 ? "processing" : "uploading"; const name = getRootDirectory(files[0]); // get the file path from the first file
onFileStateChange(file, { status, progress }); files = [{ name, files }];
};
const upload = async () => {
try {
let response;
if (file.directory) {
const directory = file.files.reduce((acc, file) => ({ ...acc, [getRelativeFilePath(file)]: file }), {});
response = await client.uploadDirectory(directory, encodeURIComponent(file.name), { onUploadProgress });
} else {
response = await client.uploadFile(file, { onUploadProgress });
} }
const url = await client.getSkylinkUrl(response.skylink, { subdomain: mode === "directory" }); setUploads((uploads) => [...files.map((file) => ({ id: nanoid(), file, mode, status: "enqueued" })), ...uploads]);
onFileStateChange(file, { status: "complete", url });
} catch (error) {
if (error.response && error.response.status === StatusCodes.TOO_MANY_REQUESTS) {
onFileStateChange(file, { progress: -1 });
return new Promise((resolve) => setTimeout(() => resolve(upload()), 3000));
}
onFileStateChange(file, { status: "error", error: createUploadErrorMessage(error) });
}
}; };
upload(); React.useEffect(() => {
const enqueued = uploads.filter(({ status }) => status === "enqueued");
const uploading = uploads.filter(({ status }) => ["uploading", "processing", "retrying"].includes(status));
const queue = enqueued.slice(0, MAX_PARALLEL_UPLOADS - uploading.length).map(({ id }) => id);
if (queue.length && uploading.length < MAX_PARALLEL_UPLOADS) {
setUploads((uploads) => {
return uploads.map((upload) => {
if (queue.includes(upload.id)) return { ...upload, status: "uploading" };
return upload;
}); });
}; });
}
}, [uploads]);
const { getRootProps, getInputProps, isDragActive, inputRef } = useDropzone({ onDrop: handleDrop }); const { getRootProps, getInputProps, isDragActive, inputRef } = useDropzone({ onDrop: handleDrop });
const inputElement = inputRef.current; const inputElement = inputRef.current;
@ -274,7 +124,7 @@ const Uploader = () => {
} }
)} )}
> >
{files.length === 0 && <Cloud />} {uploads.length === 0 && <Cloud />}
<h4 className="font-light text-palette-600 text-lg mt-2 text-center"> <h4 className="font-light text-palette-600 text-lg mt-2 text-center">
{mode === "file" && <span>Add or drop your files here to pin to Skynet</span>} {mode === "file" && <span>Add or drop your files here to pin to Skynet</span>}
{mode === "directory" && <span>Drop any folder with an index.html file to deploy to Skynet</span>} {mode === "directory" && <span>Drop any folder with an index.html file to deploy to Skynet</span>}
@ -287,10 +137,10 @@ const Uploader = () => {
</div> </div>
</div> </div>
{files.length > 0 && ( {uploads.length > 0 && (
<div className="flex flex-col space-y-5 px-4 py-10 desktop:p-14"> <div className="flex flex-col space-y-5 px-4 py-10 desktop:p-14">
{files.map((file, index) => ( {uploads.map((upload) => (
<UploadElement key={index} {...file} /> <UploaderElement key={upload.id} onUploadStateChange={onUploadStateChange} upload={upload} />
))} ))}
{!authenticated && ( {!authenticated && (
@ -315,7 +165,7 @@ const Uploader = () => {
)} )}
</div> </div>
{files.length === 0 && !authenticated && ( {uploads.length === 0 && !authenticated && (
<div className="z-0 relative flex flex-col items-center space-y-1 mt-10"> <div className="z-0 relative flex flex-col items-center space-y-1 mt-10">
<Unlock /> <Unlock />
<p className="text-sm font-light text-palette-600"> <p className="text-sm font-light text-palette-600">

View File

@ -0,0 +1,180 @@
import * as React from "react";
import classnames from "classnames";
import { ArrowUpCircle, Error, CheckCircle, ProgressRound } from "../Icons";
import bytes from "bytes";
import { getReasonPhrase, StatusCodes } from "http-status-codes";
import copy from "copy-text-to-clipboard";
import path from "path-browserify";
import { SkynetClient } from "skynet-js";
import { useTimeoutFn } from "react-use";
import ms from "ms";
import Link from "../Link";
const getFilePath = (file) => file.webkitRelativePath || file.path || file.name;
const getRelativeFilePath = (file) => {
const filePath = getFilePath(file);
const { root, dir, base } = path.parse(filePath);
const relative = path.normalize(dir).slice(root.length).split(path.sep).slice(1);
return path.join(...relative, base);
};
const createUploadErrorMessage = (error) => {
// The request was made and the server responded with a status code that falls out of the range of 2xx
if (error.response) {
if (error.response.data.message) {
return `Upload failed with error: ${error.response.data.message}`;
}
const statusCode = error.response.status;
const statusText = getReasonPhrase(error.response.status);
return `Upload failed, our server received your request but failed with status code: ${statusCode} ${statusText}`;
}
// The request was made but no response was received. The best we can do is detect whether browser is online.
// This will be triggered mostly if the server is offline or misconfigured and doesn't respond to valid request.
if (error.request) {
if (!navigator.onLine) {
return "You are offline, please connect to the internet and try again";
}
// TODO: We should add a note "our team has been notified" and have some kind of notification with this error.
return "Server failed to respond to your request, please try again later.";
}
// Match the error message to a message returned by TUS when upload exceeds max file size
const matchTusMaxFileSizeError = error.message.match(/upload exceeds maximum size: \d+ > (?<limit>\d+)/);
if (matchTusMaxFileSizeError) {
return `File exceeds size limit of ${bytes(parseInt(matchTusMaxFileSizeError.groups.limit, 10))}`;
}
// TODO: We should add a note "our team has been notified" and have some kind of notification with this error.
return `Critical error, please refresh the application and try again. ${error.message}`;
};
const client = new SkynetClient(process.env.GATSBY_API_URL);
export default function UploaderElement({ onUploadStateChange, upload }) {
const [copied, setCopied] = React.useState(false);
const [, , reset] = useTimeoutFn(() => setCopied(false), ms("3 seconds"));
const [retryTimeout, setRetryTimeout] = React.useState(ms("3 seconds")); // retry delay after "429: TOO_MANY_REQUESTS"
const handleCopy = (url) => {
copy(url);
setCopied(true);
reset();
};
React.useEffect(() => {
if (upload.status === "uploading" && !upload.startedTime) {
onUploadStateChange(upload.id, { startedTime: Date.now() });
(async () => {
const onUploadProgress = (progress) => {
const status = progress === 1 ? "processing" : "uploading";
onUploadStateChange(upload.id, { status, progress });
};
try {
let response;
if (upload.mode === "directory") {
const files = upload.file.files;
const directory = files.reduce((acc, file) => ({ ...acc, [getRelativeFilePath(file)]: file }), {});
const name = encodeURIComponent(upload.name);
response = await client.uploadDirectory(directory, name, { onUploadProgress });
} else {
response = await client.uploadFile(upload.file, { onUploadProgress });
}
const url = await client.getSkylinkUrl(response.skylink, { subdomain: upload.mode === "directory" });
onUploadStateChange(upload.id, { status: "complete", url });
} catch (error) {
if (error?.response?.status === StatusCodes.TOO_MANY_REQUESTS) {
onUploadStateChange(upload.id, { status: "retrying", progress: 0 });
setTimeout(() => {
onUploadStateChange(upload.id, { status: "enqueued", startedTime: null });
setRetryTimeout((timeout) => timeout * 2); // increase timeout on next retry
}, retryTimeout);
} else {
onUploadStateChange(upload.id, { status: "error", error: createUploadErrorMessage(error) });
}
}
})();
}
}, [onUploadStateChange, upload, retryTimeout]);
return (
<div>
<div className="flex items-center">
{upload.status === "enqueued" && <ArrowUpCircle className="flex-shrink-0 fill-current text-palette-300" />}
{upload.status === "retrying" && <ArrowUpCircle className="flex-shrink-0" />}
{upload.status === "uploading" && <ArrowUpCircle className="flex-shrink-0" />}
{upload.status === "processing" && <ProgressRound className="flex-shrink-0 animate-spin" />}
{upload.status === "complete" && <CheckCircle className="flex-shrink-0" />}
{upload.status === "error" && <Error className="flex-shrink-0 fill-current text-error" />}
<div className="flex flex-col flex-grow ml-3 overflow-hidden">
<div className="text-palette-600 text-sm font-light">{upload.file.name}</div>
<div className="flex justify-between text-palette-400 text-xs space-x-2">
<div className="font-content truncate">
{upload.status === "uploading" && (
<span className="tabular-nums">
Uploading {bytes(upload.file.size * upload.progress)} of {bytes(upload.file.size)}
</span>
)}
{upload.status === "enqueued" && <span className="text-palette-300">Upload in queue, please wait</span>}
{upload.status === "processing" && <span className="text-palette-300">Processing...</span>}
{upload.status === "complete" && (
<Link href={upload.url} className="hover:text-primary transition-colors duration-200">
{upload.url}
</Link>
)}
{upload.status === "error" && upload.error && <span className="text-error">{upload.error}</span>}
{upload.status === "retrying" && (
<span>Too many parallel requests, retrying in {retryTimeout / 1000} seconds</span>
)}
</div>
<div>
{upload.status === "uploading" && (
<span className="uppercase tabular-nums">
{Math.floor(upload.progress * 100)}%<span className="hidden desktop:inline"> completed</span>
</span>
)}
{upload.status === "processing" && <span className="uppercase text-palette-300">Wait</span>}
{upload.status === "complete" && (
<button
className="uppercase hover:text-primary transition-colors duration-200"
onClick={() => handleCopy(upload.url)}
>
<span className={classnames({ hidden: copied, "hidden desktop:inline": !copied })}>Copy link</span>
<span className={classnames({ hidden: copied, "inline desktop:hidden": !copied })}>Copy</span>
<span className={classnames({ hidden: !copied })}>Copied</span>
</button>
)}
</div>
</div>
</div>
</div>
<div
className={classnames("flex bg-palette-200 mt-1", {
"bg-error-dashed opacity-20": upload.status === "error",
"bg-primary-dashed move opacity-20": upload.status === "processing",
})}
style={{ height: "5px" }}
>
<div
style={{ width: `${Math.floor(upload.progress * 100)}%` }}
className={classnames("bg-primary", { hidden: upload.status === "processing" || upload.status === "error" })}
/>
</div>
</div>
);
}