Merge pull request #1029 from SkynetLabs/sequential-uploads

Sequential uploads on siasky.net website

packages/website/package.json

@@ -43,6 +43,7 @@
     "http-status-codes": "^2.1.4",
     "jsonp": "^0.2.1",
     "ms": "^2.1.2",
+    "nanoid": "^3.1.23",
     "normalize.css": "^8.0.1",
     "path-browserify": "^1.0.1",
     "polished": "^4.1.3",

packages/website/src/components/Uploader/Uploader.js

@@ -1,28 +1,18 @@
 import * as React from "react";
 import classnames from "classnames";
-import { Add, Cloud, ArrowUpCircle, Error, CheckCircle, Unlock, Info, ProgressRound } from "../Icons";
+import { Add, Cloud, Unlock, Info } from "../Icons";
-import bytes from "bytes";
 import classNames from "classnames";
-import { getReasonPhrase, StatusCodes } from "http-status-codes";
-import copy from "copy-text-to-clipboard";
 import path from "path-browserify";
 import { useDropzone } from "react-dropzone";
-import { SkynetClient } from "skynet-js";
+import { nanoid } from "nanoid";
-import { useTimeoutFn } from "react-use";
-import ms from "ms";
 import useAuthenticatedStatus from "../../services/useAuthenticatedStatus";
 import Link from "../Link";
+import UploaderElement from "./UploaderElement";
+
+const MAX_PARALLEL_UPLOADS = 1;
 
 const getFilePath = (file) => file.webkitRelativePath || file.path || file.name;
 
-const getRelativeFilePath = (file) => {
-  const filePath = getFilePath(file);
-  const { root, dir, base } = path.parse(filePath);
-  const relative = path.normalize(dir).slice(root.length).split(path.sep).slice(1);
-
-  return path.join(...relative, base);
-};
-
 const getRootDirectory = (file) => {
   const filePath = getFilePath(file);
   const { root, dir } = path.parse(filePath);
@@ -30,43 +20,6 @@ const getRootDirectory = (file) => {
   return path.normalize(dir).slice(root.length).split(path.sep)[0];
 };
 
-const createUploadErrorMessage = (error) => {
-  // The request was made and the server responded with a status code that falls out of the range of 2xx
-  if (error.response) {
-    if (error.response.data.message) {
-      return `Upload failed with error: ${error.response.data.message}`;
-    }
-
-    const statusCode = error.response.status;
-    const statusText = getReasonPhrase(error.response.status);
-
-    return `Upload failed, our server received your request but failed with status code: ${statusCode} ${statusText}`;
-  }
-
-  // The request was made but no response was received. The best we can do is detect whether browser is online.
-  // This will be triggered mostly if the server is offline or misconfigured and doesn't respond to valid request.
-  if (error.request) {
-    if (!navigator.onLine) {
-      return "You are offline, please connect to the internet and try again";
-    }
-
-    // TODO: We should add a note "our team has been notified" and have some kind of notification with this error.
-    return "Server failed to respond to your request, please try again later.";
-  }
-
-  // Match the error message to a message returned by TUS when upload exceeds max file size
-  const matchTusMaxFileSizeError = error.message.match(/upload exceeds maximum size: \d+ > (?<limit>\d+)/);
-
-  if (matchTusMaxFileSizeError) {
-    return `File exceeds size limit of ${bytes(parseInt(matchTusMaxFileSizeError.groups.limit, 10))}`;
-  }
-
-  // TODO: We should add a note "our team has been notified" and have some kind of notification with this error.
-  return `Critical error, please refresh the application and try again. ${error.message}`;
-};
-
-const client = new SkynetClient(process.env.GATSBY_API_URL);
-
 const RegistrationLink = () => (
   <Link
     href="https://account.siasky.net/auth/registration"
@@ -76,148 +29,45 @@ const RegistrationLink = () => (
   </Link>
 );
 
-const UploadElement = ({ file, status, error, url = "", progress = 0 }) => {
-  const [copied, setCopied] = React.useState(false);
-  const [, , reset] = useTimeoutFn(() => setCopied(false), ms("3 seconds"));
-
-  const handleCopy = (url) => {
-    copy(url);
-    setCopied(true);
-    reset();
-  };
-
-  return (
-    <div>
-      <div className="flex items-center">
-        {status === "uploading" && <ArrowUpCircle className="flex-shrink-0" />}
-        {status === "processing" && <ProgressRound className="flex-shrink-0 animate-spin" />}
-        {status === "complete" && <CheckCircle className="flex-shrink-0" />}
-        {status === "error" && <Error className="flex-shrink-0 fill-current text-error" />}
-        <div className="flex flex-col flex-grow ml-3 overflow-hidden">
-          <div className="text-palette-600 text-sm font-light">{file.name}</div>
-          <div className="flex justify-between text-palette-400 text-xs space-x-2">
-            <div className="font-content truncate">
-              {status === "uploading" && (
-                <span className="tabular-nums">
-                  Uploading {bytes(file.size * progress)} of {bytes(file.size)}
-                </span>
-              )}
-
-              {status === "processing" && <span className="text-palette-300">Processing...</span>}
-
-              {status === "complete" && (
-                <Link href={url} className="hover:text-primary transition-colors duration-200">
-                  {url}
-                </Link>
-              )}
-
-              {status === "error" && error && <span className="text-error">{error}</span>}
-            </div>
-            <div>
-              {status === "uploading" && (
-                <span className="uppercase tabular-nums">
-                  {Math.floor(progress * 100)}%<span className="hidden desktop:inline"> completed</span>
-                </span>
-              )}
-              {status === "processing" && <span className="uppercase text-palette-300">Wait</span>}
-              {status === "complete" && (
-                <button
-                  className="uppercase hover:text-primary transition-colors duration-200"
-                  onClick={() => handleCopy(url)}
-                >
-                  <span className={classnames({ hidden: copied, "hidden desktop:inline": !copied })}>Copy link</span>
-                  <span className={classnames({ hidden: copied, "inline desktop:hidden": !copied })}>Copy</span>
-                  <span className={classnames({ hidden: !copied })}>Copied</span>
-                </button>
-              )}
-            </div>
-          </div>
-        </div>
-      </div>
-
-      <div
-        className={classnames("flex bg-palette-200 mt-1", {
-          "bg-error-dashed opacity-20": status === "error",
-          "bg-primary-dashed move opacity-20": status === "processing",
-        })}
-        style={{ height: "5px" }}
-      >
-        <div
-          style={{ width: `${Math.floor(progress * 100)}%` }}
-          className={classnames("bg-primary", { hidden: status === "processing" || status === "error" })}
-        />
-      </div>
-    </div>
-  );
-};
-
 const Uploader = () => {
   const [mode, setMode] = React.useState("file");
-  const [files, setFiles] = React.useState([]);
+  const [uploads, setUploads] = React.useState([]);
 
   const { data: authenticationStatus } = useAuthenticatedStatus();
   const authenticated = authenticationStatus?.authenticated ?? false;
 
-  const handleDrop = async (acceptedFiles) => {
+  const onUploadStateChange = React.useCallback((id, state) => {
-    if (mode === "directory" && acceptedFiles.length) {
+    setUploads((uploads) => {
-      const rootDir = getRootDirectory(acceptedFiles[0]); // get the file path from the first file
+      const index = uploads.findIndex((upload) => upload.id === id);
 
-      acceptedFiles = [{ name: rootDir, directory: true, files: acceptedFiles }];
+      return [...uploads.slice(0, index), { ...uploads[index], ...state }, ...uploads.slice(index + 1)];
-    }
-
-    setFiles((previousFiles) => [...acceptedFiles.map((file) => ({ file, status: "uploading" })), ...previousFiles]);
-
-    const onFileStateChange = (file, state) => {
-      setFiles((previousFiles) => {
-        const index = previousFiles.findIndex((f) => f.file === file);
-
-        return [
-          ...previousFiles.slice(0, index),
-          {
-            ...previousFiles[index],
-            ...state,
-          },
-          ...previousFiles.slice(index + 1),
-        ];
     });
-    };
+  }, []);
 
-    acceptedFiles.forEach((file) => {
+  const handleDrop = async (files) => {
-      const onUploadProgress = (progress) => {
+    if (mode === "directory" && files.length) {
-        const status = progress === 1 ? "processing" : "uploading";
+      const name = getRootDirectory(files[0]); // get the file path from the first file
 
-        onFileStateChange(file, { status, progress });
+      files = [{ name, files }];
-      };
-
-      const upload = async () => {
-        try {
-          let response;
-
-          if (file.directory) {
-            const directory = file.files.reduce((acc, file) => ({ ...acc, [getRelativeFilePath(file)]: file }), {});
-
-            response = await client.uploadDirectory(directory, encodeURIComponent(file.name), { onUploadProgress });
-          } else {
-            response = await client.uploadFile(file, { onUploadProgress });
     }
 
-          const url = await client.getSkylinkUrl(response.skylink, { subdomain: mode === "directory" });
+    setUploads((uploads) => [...files.map((file) => ({ id: nanoid(), file, mode, status: "enqueued" })), ...uploads]);
-
-          onFileStateChange(file, { status: "complete", url });
-        } catch (error) {
-          if (error.response && error.response.status === StatusCodes.TOO_MANY_REQUESTS) {
-            onFileStateChange(file, { progress: -1 });
-
-            return new Promise((resolve) => setTimeout(() => resolve(upload()), 3000));
-          }
-
-          onFileStateChange(file, { status: "error", error: createUploadErrorMessage(error) });
-        }
   };
 
-      upload();
+  React.useEffect(() => {
+    const enqueued = uploads.filter(({ status }) => status === "enqueued");
+    const uploading = uploads.filter(({ status }) => ["uploading", "processing", "retrying"].includes(status));
+    const queue = enqueued.slice(0, MAX_PARALLEL_UPLOADS - uploading.length).map(({ id }) => id);
+
+    if (queue.length && uploading.length < MAX_PARALLEL_UPLOADS) {
+      setUploads((uploads) => {
+        return uploads.map((upload) => {
+          if (queue.includes(upload.id)) return { ...upload, status: "uploading" };
+          return upload;
         });
-  };
+      });
+    }
+  }, [uploads]);
 
   const { getRootProps, getInputProps, isDragActive, inputRef } = useDropzone({ onDrop: handleDrop });
   const inputElement = inputRef.current;
@@ -274,7 +124,7 @@ const Uploader = () => {
               }
             )}
           >
-            {files.length === 0 && <Cloud />}
+            {uploads.length === 0 && <Cloud />}
             <h4 className="font-light text-palette-600 text-lg mt-2 text-center">
               {mode === "file" && <span>Add or drop your files here to pin to Skynet</span>}
               {mode === "directory" && <span>Drop any folder with an index.html file to deploy to Skynet</span>}
@@ -287,10 +137,10 @@ const Uploader = () => {
           </div>
         </div>
 
-        {files.length > 0 && (
+        {uploads.length > 0 && (
           <div className="flex flex-col space-y-5 px-4 py-10 desktop:p-14">
-            {files.map((file, index) => (
+            {uploads.map((upload) => (
-              <UploadElement key={index} {...file} />
+              <UploaderElement key={upload.id} onUploadStateChange={onUploadStateChange} upload={upload} />
             ))}
 
             {!authenticated && (
@@ -315,7 +165,7 @@ const Uploader = () => {
         )}
       </div>
 
-      {files.length === 0 && !authenticated && (
+      {uploads.length === 0 && !authenticated && (
         <div className="z-0 relative flex flex-col items-center space-y-1 mt-10">
           <Unlock />
           <p className="text-sm font-light text-palette-600">

packages/website/src/components/Uploader/UploaderElement.js

@@ -0,0 +1,180 @@
+import * as React from "react";
+import classnames from "classnames";
+import { ArrowUpCircle, Error, CheckCircle, ProgressRound } from "../Icons";
+import bytes from "bytes";
+import { getReasonPhrase, StatusCodes } from "http-status-codes";
+import copy from "copy-text-to-clipboard";
+import path from "path-browserify";
+import { SkynetClient } from "skynet-js";
+import { useTimeoutFn } from "react-use";
+import ms from "ms";
+import Link from "../Link";
+
+const getFilePath = (file) => file.webkitRelativePath || file.path || file.name;
+
+const getRelativeFilePath = (file) => {
+  const filePath = getFilePath(file);
+  const { root, dir, base } = path.parse(filePath);
+  const relative = path.normalize(dir).slice(root.length).split(path.sep).slice(1);
+
+  return path.join(...relative, base);
+};
+
+const createUploadErrorMessage = (error) => {
+  // The request was made and the server responded with a status code that falls out of the range of 2xx
+  if (error.response) {
+    if (error.response.data.message) {
+      return `Upload failed with error: ${error.response.data.message}`;
+    }
+
+    const statusCode = error.response.status;
+    const statusText = getReasonPhrase(error.response.status);
+
+    return `Upload failed, our server received your request but failed with status code: ${statusCode} ${statusText}`;
+  }
+
+  // The request was made but no response was received. The best we can do is detect whether browser is online.
+  // This will be triggered mostly if the server is offline or misconfigured and doesn't respond to valid request.
+  if (error.request) {
+    if (!navigator.onLine) {
+      return "You are offline, please connect to the internet and try again";
+    }
+
+    // TODO: We should add a note "our team has been notified" and have some kind of notification with this error.
+    return "Server failed to respond to your request, please try again later.";
+  }
+
+  // Match the error message to a message returned by TUS when upload exceeds max file size
+  const matchTusMaxFileSizeError = error.message.match(/upload exceeds maximum size: \d+ > (?<limit>\d+)/);
+
+  if (matchTusMaxFileSizeError) {
+    return `File exceeds size limit of ${bytes(parseInt(matchTusMaxFileSizeError.groups.limit, 10))}`;
+  }
+
+  // TODO: We should add a note "our team has been notified" and have some kind of notification with this error.
+  return `Critical error, please refresh the application and try again. ${error.message}`;
+};
+
+const client = new SkynetClient(process.env.GATSBY_API_URL);
+
+export default function UploaderElement({ onUploadStateChange, upload }) {
+  const [copied, setCopied] = React.useState(false);
+  const [, , reset] = useTimeoutFn(() => setCopied(false), ms("3 seconds"));
+  const [retryTimeout, setRetryTimeout] = React.useState(ms("3 seconds")); // retry delay after "429: TOO_MANY_REQUESTS"
+
+  const handleCopy = (url) => {
+    copy(url);
+    setCopied(true);
+    reset();
+  };
+
+  React.useEffect(() => {
+    if (upload.status === "uploading" && !upload.startedTime) {
+      onUploadStateChange(upload.id, { startedTime: Date.now() });
+
+      (async () => {
+        const onUploadProgress = (progress) => {
+          const status = progress === 1 ? "processing" : "uploading";
+
+          onUploadStateChange(upload.id, { status, progress });
+        };
+
+        try {
+          let response;
+
+          if (upload.mode === "directory") {
+            const files = upload.file.files;
+            const directory = files.reduce((acc, file) => ({ ...acc, [getRelativeFilePath(file)]: file }), {});
+            const name = encodeURIComponent(upload.name);
+
+            response = await client.uploadDirectory(directory, name, { onUploadProgress });
+          } else {
+            response = await client.uploadFile(upload.file, { onUploadProgress });
+          }
+
+          const url = await client.getSkylinkUrl(response.skylink, { subdomain: upload.mode === "directory" });
+
+          onUploadStateChange(upload.id, { status: "complete", url });
+        } catch (error) {
+          if (error?.response?.status === StatusCodes.TOO_MANY_REQUESTS) {
+            onUploadStateChange(upload.id, { status: "retrying", progress: 0 });
+
+            setTimeout(() => {
+              onUploadStateChange(upload.id, { status: "enqueued", startedTime: null });
+              setRetryTimeout((timeout) => timeout * 2); // increase timeout on next retry
+            }, retryTimeout);
+          } else {
+            onUploadStateChange(upload.id, { status: "error", error: createUploadErrorMessage(error) });
+          }
+        }
+      })();
+    }
+  }, [onUploadStateChange, upload, retryTimeout]);
+
+  return (
+    <div>
+      <div className="flex items-center">
+        {upload.status === "enqueued" && <ArrowUpCircle className="flex-shrink-0 fill-current text-palette-300" />}
+        {upload.status === "retrying" && <ArrowUpCircle className="flex-shrink-0" />}
+        {upload.status === "uploading" && <ArrowUpCircle className="flex-shrink-0" />}
+        {upload.status === "processing" && <ProgressRound className="flex-shrink-0 animate-spin" />}
+        {upload.status === "complete" && <CheckCircle className="flex-shrink-0" />}
+        {upload.status === "error" && <Error className="flex-shrink-0 fill-current text-error" />}
+        <div className="flex flex-col flex-grow ml-3 overflow-hidden">
+          <div className="text-palette-600 text-sm font-light">{upload.file.name}</div>
+          <div className="flex justify-between text-palette-400 text-xs space-x-2">
+            <div className="font-content truncate">
+              {upload.status === "uploading" && (
+                <span className="tabular-nums">
+                  Uploading {bytes(upload.file.size * upload.progress)} of {bytes(upload.file.size)}
+                </span>
+              )}
+              {upload.status === "enqueued" && <span className="text-palette-300">Upload in queue, please wait</span>}
+              {upload.status === "processing" && <span className="text-palette-300">Processing...</span>}
+              {upload.status === "complete" && (
+                <Link href={upload.url} className="hover:text-primary transition-colors duration-200">
+                  {upload.url}
+                </Link>
+              )}
+              {upload.status === "error" && upload.error && <span className="text-error">{upload.error}</span>}
+              {upload.status === "retrying" && (
+                <span>Too many parallel requests, retrying in {retryTimeout / 1000} seconds</span>
+              )}
+            </div>
+            <div>
+              {upload.status === "uploading" && (
+                <span className="uppercase tabular-nums">
+                  {Math.floor(upload.progress * 100)}%<span className="hidden desktop:inline"> completed</span>
+                </span>
+              )}
+              {upload.status === "processing" && <span className="uppercase text-palette-300">Wait</span>}
+              {upload.status === "complete" && (
+                <button
+                  className="uppercase hover:text-primary transition-colors duration-200"
+                  onClick={() => handleCopy(upload.url)}
+                >
+                  <span className={classnames({ hidden: copied, "hidden desktop:inline": !copied })}>Copy link</span>
+                  <span className={classnames({ hidden: copied, "inline desktop:hidden": !copied })}>Copy</span>
+                  <span className={classnames({ hidden: !copied })}>Copied</span>
+                </button>
+              )}
+            </div>
+          </div>
+        </div>
+      </div>
+
+      <div
+        className={classnames("flex bg-palette-200 mt-1", {
+          "bg-error-dashed opacity-20": upload.status === "error",
+          "bg-primary-dashed move opacity-20": upload.status === "processing",
+        })}
+        style={{ height: "5px" }}
+      >
+        <div
+          style={{ width: `${Math.floor(upload.progress * 100)}%` }}
+          className={classnames("bg-primary", { hidden: upload.status === "processing" || upload.status === "error" })}
+        />
+      </div>
+    </div>
+  );
+}