From 062425167731736cab90d30a4eff5cb1a2fc207e Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 3 Aug 2021 14:14:01 +0200 Subject: [PATCH 1/4] make uploads sequential --- packages/website/package.json | 1 + .../src/components/Uploader/Uploader.js | 231 ++++-------------- .../components/Uploader/UploaderElement.js | 180 ++++++++++++++ packages/website/yarn.lock | 12 + 4 files changed, 235 insertions(+), 189 deletions(-) create mode 100644 packages/website/src/components/Uploader/UploaderElement.js diff --git a/packages/website/package.json b/packages/website/package.json index 3eceead1..4b23bc10 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -58,6 +58,7 @@ "react-svg-loader": "^3.0.3", "react-syntax-highlighter": "^15.4.4", "react-use": "^17.2.4", + "shortid": "^2.2.16", "skynet-js": "^4.0.11-beta", "stream-browserify": "^3.0.0", "swr": "^0.5.6", diff --git a/packages/website/src/components/Uploader/Uploader.js b/packages/website/src/components/Uploader/Uploader.js index bdd549f5..e3cb0677 100644 --- a/packages/website/src/components/Uploader/Uploader.js +++ b/packages/website/src/components/Uploader/Uploader.js @@ -1,28 +1,18 @@ import * as React from "react"; import classnames from "classnames"; -import { Add, Cloud, ArrowUpCircle, Error, CheckCircle, Unlock, Info, ProgressRound } from "../Icons"; -import bytes from "bytes"; +import { Add, Cloud, Unlock, Info } from "../Icons"; import classNames from "classnames"; -import { getReasonPhrase, StatusCodes } from "http-status-codes"; -import copy from "copy-text-to-clipboard"; import path from "path-browserify"; import { useDropzone } from "react-dropzone"; -import { SkynetClient } from "skynet-js"; -import { useTimeoutFn } from "react-use"; -import ms from "ms"; +import shortid from "shortid"; import useAuthenticatedStatus from "../../services/useAuthenticatedStatus"; import Link from "../Link"; +import UploaderElement from "./UploaderElement"; + +const MAX_PARALLEL_UPLOADS = 1; const getFilePath = (file) => file.webkitRelativePath || file.path || file.name; -const getRelativeFilePath = (file) => { - const filePath = getFilePath(file); - const { root, dir, base } = path.parse(filePath); - const relative = path.normalize(dir).slice(root.length).split(path.sep).slice(1); - - return path.join(...relative, base); -}; - const getRootDirectory = (file) => { const filePath = getFilePath(file); const { root, dir } = path.parse(filePath); @@ -30,43 +20,6 @@ const getRootDirectory = (file) => { return path.normalize(dir).slice(root.length).split(path.sep)[0]; }; -const createUploadErrorMessage = (error) => { - // The request was made and the server responded with a status code that falls out of the range of 2xx - if (error.response) { - if (error.response.data.message) { - return `Upload failed with error: ${error.response.data.message}`; - } - - const statusCode = error.response.status; - const statusText = getReasonPhrase(error.response.status); - - return `Upload failed, our server received your request but failed with status code: ${statusCode} ${statusText}`; - } - - // The request was made but no response was received. The best we can do is detect whether browser is online. - // This will be triggered mostly if the server is offline or misconfigured and doesn't respond to valid request. - if (error.request) { - if (!navigator.onLine) { - return "You are offline, please connect to the internet and try again"; - } - - // TODO: We should add a note "our team has been notified" and have some kind of notification with this error. - return "Server failed to respond to your request, please try again later."; - } - - // Match the error message to a message returned by TUS when upload exceeds max file size - const matchTusMaxFileSizeError = error.message.match(/upload exceeds maximum size: \d+ > (?\d+)/); - - if (matchTusMaxFileSizeError) { - return `File exceeds size limit of ${bytes(parseInt(matchTusMaxFileSizeError.groups.limit, 10))}`; - } - - // TODO: We should add a note "our team has been notified" and have some kind of notification with this error. - return `Critical error, please refresh the application and try again. ${error.message}`; -}; - -const client = new SkynetClient(process.env.GATSBY_API_URL); - const RegistrationLink = () => ( ( ); -const UploadElement = ({ file, status, error, url = "", progress = 0 }) => { - const [copied, setCopied] = React.useState(false); - const [, , reset] = useTimeoutFn(() => setCopied(false), ms("3 seconds")); - - const handleCopy = (url) => { - copy(url); - setCopied(true); - reset(); - }; - - return ( -
-
- {status === "uploading" && } - {status === "processing" && } - {status === "complete" && } - {status === "error" && } -
-
{file.name}
-
-
- {status === "uploading" && ( - - Uploading {bytes(file.size * progress)} of {bytes(file.size)} - - )} - - {status === "processing" && Processing...} - - {status === "complete" && ( - - {url} - - )} - - {status === "error" && error && {error}} -
-
- {status === "uploading" && ( - - {Math.floor(progress * 100)}% completed - - )} - {status === "processing" && Wait} - {status === "complete" && ( - - )} -
-
-
-
- -
-
-
-
- ); -}; - const Uploader = () => { const [mode, setMode] = React.useState("file"); - const [files, setFiles] = React.useState([]); + const [uploads, setUploads] = React.useState([]); const { data: authenticationStatus } = useAuthenticatedStatus(); const authenticated = authenticationStatus?.authenticated ?? false; - const handleDrop = async (acceptedFiles) => { - if (mode === "directory" && acceptedFiles.length) { - const rootDir = getRootDirectory(acceptedFiles[0]); // get the file path from the first file + const onUploadStateChange = React.useCallback((id, state) => { + setUploads((uploads) => { + const index = uploads.findIndex((upload) => upload.id === id); - acceptedFiles = [{ name: rootDir, directory: true, files: acceptedFiles }]; + return [...uploads.slice(0, index), { ...uploads[index], ...state }, ...uploads.slice(index + 1)]; + }); + }, []); + + const handleDrop = async (files) => { + if (mode === "directory" && files.length) { + const name = getRootDirectory(files[0]); // get the file path from the first file + + files = [{ name, files }]; } - setFiles((previousFiles) => [...acceptedFiles.map((file) => ({ file, status: "uploading" })), ...previousFiles]); - - const onFileStateChange = (file, state) => { - setFiles((previousFiles) => { - const index = previousFiles.findIndex((f) => f.file === file); - - return [ - ...previousFiles.slice(0, index), - { - ...previousFiles[index], - ...state, - }, - ...previousFiles.slice(index + 1), - ]; - }); - }; - - acceptedFiles.forEach((file) => { - const onUploadProgress = (progress) => { - const status = progress === 1 ? "processing" : "uploading"; - - onFileStateChange(file, { status, progress }); - }; - - const upload = async () => { - try { - let response; - - if (file.directory) { - const directory = file.files.reduce((acc, file) => ({ ...acc, [getRelativeFilePath(file)]: file }), {}); - - response = await client.uploadDirectory(directory, encodeURIComponent(file.name), { onUploadProgress }); - } else { - response = await client.uploadFile(file, { onUploadProgress }); - } - - const url = await client.getSkylinkUrl(response.skylink, { subdomain: mode === "directory" }); - - onFileStateChange(file, { status: "complete", url }); - } catch (error) { - if (error.response && error.response.status === StatusCodes.TOO_MANY_REQUESTS) { - onFileStateChange(file, { progress: -1 }); - - return new Promise((resolve) => setTimeout(() => resolve(upload()), 3000)); - } - - onFileStateChange(file, { status: "error", error: createUploadErrorMessage(error) }); - } - }; - - upload(); - }); + setUploads((uploads) => [ + ...files.map((file) => ({ id: shortid.generate(), file, mode, status: "enqueued" })), + ...uploads, + ]); }; + React.useEffect(() => { + const enqueued = uploads.filter(({ status }) => status === "enqueued"); + const uploading = uploads.filter(({ status }) => ["uploading", "processing", "retrying"].includes(status)); + const queue = enqueued.slice(0, MAX_PARALLEL_UPLOADS - uploading.length).map(({ id }) => id); + + if (queue.length && uploading.length < MAX_PARALLEL_UPLOADS) { + setUploads((uploads) => { + return uploads.map((upload) => { + if (queue.includes(upload.id)) return { ...upload, status: "uploading" }; + return upload; + }); + }); + } + }, [uploads]); + const { getRootProps, getInputProps, isDragActive, inputRef } = useDropzone({ onDrop: handleDrop }); const inputElement = inputRef.current; @@ -274,7 +127,7 @@ const Uploader = () => { } )} > - {files.length === 0 && } + {uploads.length === 0 && }

{mode === "file" && Add or drop your files here to pin to Skynet} {mode === "directory" && Drop any folder with an index.html file to deploy to Skynet} @@ -287,10 +140,10 @@ const Uploader = () => {

- {files.length > 0 && ( + {uploads.length > 0 && (
- {files.map((file, index) => ( - + {uploads.map((upload) => ( + ))} {!authenticated && ( @@ -315,7 +168,7 @@ const Uploader = () => { )}
- {files.length === 0 && !authenticated && ( + {uploads.length === 0 && !authenticated && (

diff --git a/packages/website/src/components/Uploader/UploaderElement.js b/packages/website/src/components/Uploader/UploaderElement.js new file mode 100644 index 00000000..0bfbcf8a --- /dev/null +++ b/packages/website/src/components/Uploader/UploaderElement.js @@ -0,0 +1,180 @@ +import * as React from "react"; +import classnames from "classnames"; +import { ArrowUpCircle, Error, CheckCircle, ProgressRound } from "../Icons"; +import bytes from "bytes"; +import { getReasonPhrase, StatusCodes } from "http-status-codes"; +import copy from "copy-text-to-clipboard"; +import path from "path-browserify"; +import { SkynetClient } from "skynet-js"; +import { useTimeoutFn } from "react-use"; +import ms from "ms"; +import Link from "../Link"; + +const TOO_MANY_REQUESTS_RETRY = ms("5s"); // retry delay after "492: TOO_MANY_REQUESTS" + +const getFilePath = (file) => file.webkitRelativePath || file.path || file.name; + +const getRelativeFilePath = (file) => { + const filePath = getFilePath(file); + const { root, dir, base } = path.parse(filePath); + const relative = path.normalize(dir).slice(root.length).split(path.sep).slice(1); + + return path.join(...relative, base); +}; + +const createUploadErrorMessage = (error) => { + // The request was made and the server responded with a status code that falls out of the range of 2xx + if (error.response) { + if (error.response.data.message) { + return `Upload failed with error: ${error.response.data.message}`; + } + + const statusCode = error.response.status; + const statusText = getReasonPhrase(error.response.status); + + return `Upload failed, our server received your request but failed with status code: ${statusCode} ${statusText}`; + } + + // The request was made but no response was received. The best we can do is detect whether browser is online. + // This will be triggered mostly if the server is offline or misconfigured and doesn't respond to valid request. + if (error.request) { + if (!navigator.onLine) { + return "You are offline, please connect to the internet and try again"; + } + + // TODO: We should add a note "our team has been notified" and have some kind of notification with this error. + return "Server failed to respond to your request, please try again later."; + } + + // Match the error message to a message returned by TUS when upload exceeds max file size + const matchTusMaxFileSizeError = error.message.match(/upload exceeds maximum size: \d+ > (?\d+)/); + + if (matchTusMaxFileSizeError) { + return `File exceeds size limit of ${bytes(parseInt(matchTusMaxFileSizeError.groups.limit, 10))}`; + } + + // TODO: We should add a note "our team has been notified" and have some kind of notification with this error. + return `Critical error, please refresh the application and try again. ${error.message}`; +}; + +const client = new SkynetClient(process.env.GATSBY_API_URL); + +export default function UploaderElement({ onUploadStateChange, upload }) { + const [copied, setCopied] = React.useState(false); + const [, , reset] = useTimeoutFn(() => setCopied(false), ms("3 seconds")); + + const handleCopy = (url) => { + copy(url); + setCopied(true); + reset(); + }; + + React.useEffect(() => { + if (upload.status === "uploading" && !upload.startedTime) { + onUploadStateChange(upload.id, { startedTime: Date.now() }); + + (async () => { + const onUploadProgress = (progress) => { + const status = progress === 1 ? "processing" : "uploading"; + + onUploadStateChange(upload.id, { status, progress }); + }; + + try { + let response; + + if (upload.mode === "directory") { + const files = upload.file.files; + const directory = files.reduce((acc, file) => ({ ...acc, [getRelativeFilePath(file)]: file }), {}); + const name = encodeURIComponent(upload.name); + + response = await client.uploadDirectory(directory, name, { onUploadProgress }); + } else { + response = await client.uploadFile(upload.file, { onUploadProgress }); + } + + const url = await client.getSkylinkUrl(response.skylink, { subdomain: upload.mode === "directory" }); + + onUploadStateChange(upload.id, { status: "complete", url }); + } catch (error) { + if (error?.response?.status === StatusCodes.TOO_MANY_REQUESTS) { + onUploadStateChange(upload.id, { status: "retrying", progress: 0 }); + + setTimeout(() => { + onUploadStateChange(upload.id, { status: "enqueued", startedTime: null }); + }, TOO_MANY_REQUESTS_RETRY); + } else { + onUploadStateChange(upload.id, { status: "error", error: createUploadErrorMessage(error) }); + } + } + })(); + } + }, [onUploadStateChange, upload]); + + return ( +

+
+ {upload.status === "enqueued" && } + {upload.status === "retrying" && } + {upload.status === "uploading" && } + {upload.status === "processing" && } + {upload.status === "complete" && } + {upload.status === "error" && } +
+
{upload.file.name}
+
+
+ {upload.status === "uploading" && ( + + Uploading {bytes(upload.file.size * upload.progress)} of {bytes(upload.file.size)} + + )} + {upload.status === "enqueued" && Upload in queue, please wait} + {upload.status === "processing" && Processing...} + {upload.status === "complete" && ( + + {upload.url} + + )} + {upload.status === "error" && upload.error && {upload.error}} + {upload.status === "retrying" && ( + Too many parallel requests, retrying in {TOO_MANY_REQUESTS_RETRY / 1000} seconds + )} +
+
+ {upload.status === "uploading" && ( + + {Math.floor(upload.progress * 100)}% completed + + )} + {upload.status === "processing" && Wait} + {upload.status === "complete" && ( + + )} +
+
+
+
+ +
+
+
+
+ ); +} diff --git a/packages/website/yarn.lock b/packages/website/yarn.lock index edf202c0..481befcc 100644 --- a/packages/website/yarn.lock +++ b/packages/website/yarn.lock @@ -10335,6 +10335,11 @@ nano-css@^5.3.1: stacktrace-js "^2.0.2" stylis "^4.0.6" +nanoid@^2.1.0: + version "2.1.11" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-2.1.11.tgz#ec24b8a758d591561531b4176a01e3ab4f0f0280" + integrity sha512-s/snB+WGm6uwi0WjsZdaVcuf3KJXlfGl2LcxgwkEwJF0D/BWzVWAZW/XY4bFaiR7s0Jk3FPvlnepg1H1b1UwlA== + nanoid@^3.1.23: version "3.1.23" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.23.tgz#f744086ce7c2bc47ee0a8472574d5c78e4183a81" @@ -13245,6 +13250,13 @@ short-uuid@^4.1.0: any-base "^1.1.0" uuid "^8.3.2" +shortid@^2.2.16: + version "2.2.16" + resolved "https://registry.yarnpkg.com/shortid/-/shortid-2.2.16.tgz#b742b8f0cb96406fd391c76bfc18a67a57fe5608" + integrity sha512-Ugt+GIZqvGXCIItnsL+lvFJOiN7RYqlGy7QE41O3YC1xbNSeDGIRO7xg2JJXIAj1cAGnOeC1r7/T9pgrtQbv4g== + dependencies: + nanoid "^2.1.0" + side-channel@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" From 46ed1e42d48e6c7461554b55a60eac641ed2faad Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 3 Aug 2021 14:31:52 +0200 Subject: [PATCH 2/4] use nanoid instead of shortid --- packages/website/package.json | 2 +- packages/website/src/components/Uploader/Uploader.js | 7 ++----- packages/website/yarn.lock | 12 ------------ 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/packages/website/package.json b/packages/website/package.json index 4b23bc10..8b96b08b 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -43,6 +43,7 @@ "http-status-codes": "^2.1.4", "jsonp": "^0.2.1", "ms": "^2.1.2", + "nanoid": "^3.1.23", "normalize.css": "^8.0.1", "path-browserify": "^1.0.1", "polished": "^4.1.3", @@ -58,7 +59,6 @@ "react-svg-loader": "^3.0.3", "react-syntax-highlighter": "^15.4.4", "react-use": "^17.2.4", - "shortid": "^2.2.16", "skynet-js": "^4.0.11-beta", "stream-browserify": "^3.0.0", "swr": "^0.5.6", diff --git a/packages/website/src/components/Uploader/Uploader.js b/packages/website/src/components/Uploader/Uploader.js index e3cb0677..f5241f42 100644 --- a/packages/website/src/components/Uploader/Uploader.js +++ b/packages/website/src/components/Uploader/Uploader.js @@ -4,7 +4,7 @@ import { Add, Cloud, Unlock, Info } from "../Icons"; import classNames from "classnames"; import path from "path-browserify"; import { useDropzone } from "react-dropzone"; -import shortid from "shortid"; +import { nanoid } from "nanoid"; import useAuthenticatedStatus from "../../services/useAuthenticatedStatus"; import Link from "../Link"; import UploaderElement from "./UploaderElement"; @@ -51,10 +51,7 @@ const Uploader = () => { files = [{ name, files }]; } - setUploads((uploads) => [ - ...files.map((file) => ({ id: shortid.generate(), file, mode, status: "enqueued" })), - ...uploads, - ]); + setUploads((uploads) => [...files.map((file) => ({ id: nanoid(), file, mode, status: "enqueued" })), ...uploads]); }; React.useEffect(() => { diff --git a/packages/website/yarn.lock b/packages/website/yarn.lock index 481befcc..edf202c0 100644 --- a/packages/website/yarn.lock +++ b/packages/website/yarn.lock @@ -10335,11 +10335,6 @@ nano-css@^5.3.1: stacktrace-js "^2.0.2" stylis "^4.0.6" -nanoid@^2.1.0: - version "2.1.11" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-2.1.11.tgz#ec24b8a758d591561531b4176a01e3ab4f0f0280" - integrity sha512-s/snB+WGm6uwi0WjsZdaVcuf3KJXlfGl2LcxgwkEwJF0D/BWzVWAZW/XY4bFaiR7s0Jk3FPvlnepg1H1b1UwlA== - nanoid@^3.1.23: version "3.1.23" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.23.tgz#f744086ce7c2bc47ee0a8472574d5c78e4183a81" @@ -13250,13 +13245,6 @@ short-uuid@^4.1.0: any-base "^1.1.0" uuid "^8.3.2" -shortid@^2.2.16: - version "2.2.16" - resolved "https://registry.yarnpkg.com/shortid/-/shortid-2.2.16.tgz#b742b8f0cb96406fd391c76bfc18a67a57fe5608" - integrity sha512-Ugt+GIZqvGXCIItnsL+lvFJOiN7RYqlGy7QE41O3YC1xbNSeDGIRO7xg2JJXIAj1cAGnOeC1r7/T9pgrtQbv4g== - dependencies: - nanoid "^2.1.0" - side-channel@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" From 3abd57642eef1a7684a3e8d66f7a14fcf12bd04f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Wypch=C5=82o?= Date: Tue, 3 Aug 2021 15:39:42 +0200 Subject: [PATCH 3/4] Update packages/website/src/components/Uploader/UploaderElement.js --- packages/website/src/components/Uploader/UploaderElement.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/website/src/components/Uploader/UploaderElement.js b/packages/website/src/components/Uploader/UploaderElement.js index 0bfbcf8a..f4842c04 100644 --- a/packages/website/src/components/Uploader/UploaderElement.js +++ b/packages/website/src/components/Uploader/UploaderElement.js @@ -10,7 +10,7 @@ import { useTimeoutFn } from "react-use"; import ms from "ms"; import Link from "../Link"; -const TOO_MANY_REQUESTS_RETRY = ms("5s"); // retry delay after "492: TOO_MANY_REQUESTS" +const TOO_MANY_REQUESTS_RETRY = ms("5s"); // retry delay after "429: TOO_MANY_REQUESTS" const getFilePath = (file) => file.webkitRelativePath || file.path || file.name; From 6bb7945fbfd81ba23affff680c09f7f65183491b Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 3 Aug 2021 16:01:17 +0200 Subject: [PATCH 4/4] multiply delay on every subsequent retry --- .../website/src/components/Uploader/UploaderElement.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/website/src/components/Uploader/UploaderElement.js b/packages/website/src/components/Uploader/UploaderElement.js index f4842c04..86dd2ada 100644 --- a/packages/website/src/components/Uploader/UploaderElement.js +++ b/packages/website/src/components/Uploader/UploaderElement.js @@ -10,8 +10,6 @@ import { useTimeoutFn } from "react-use"; import ms from "ms"; import Link from "../Link"; -const TOO_MANY_REQUESTS_RETRY = ms("5s"); // retry delay after "429: TOO_MANY_REQUESTS" - const getFilePath = (file) => file.webkitRelativePath || file.path || file.name; const getRelativeFilePath = (file) => { @@ -62,6 +60,7 @@ const client = new SkynetClient(process.env.GATSBY_API_URL); export default function UploaderElement({ onUploadStateChange, upload }) { const [copied, setCopied] = React.useState(false); const [, , reset] = useTimeoutFn(() => setCopied(false), ms("3 seconds")); + const [retryTimeout, setRetryTimeout] = React.useState(ms("3 seconds")); // retry delay after "429: TOO_MANY_REQUESTS" const handleCopy = (url) => { copy(url); @@ -102,14 +101,15 @@ export default function UploaderElement({ onUploadStateChange, upload }) { setTimeout(() => { onUploadStateChange(upload.id, { status: "enqueued", startedTime: null }); - }, TOO_MANY_REQUESTS_RETRY); + setRetryTimeout((timeout) => timeout * 2); // increase timeout on next retry + }, retryTimeout); } else { onUploadStateChange(upload.id, { status: "error", error: createUploadErrorMessage(error) }); } } })(); } - }, [onUploadStateChange, upload]); + }, [onUploadStateChange, upload, retryTimeout]); return (
@@ -138,7 +138,7 @@ export default function UploaderElement({ onUploadStateChange, upload }) { )} {upload.status === "error" && upload.error && {upload.error}} {upload.status === "retrying" && ( - Too many parallel requests, retrying in {TOO_MANY_REQUESTS_RETRY / 1000} seconds + Too many parallel requests, retrying in {retryTimeout / 1000} seconds )}