Merge remote-tracking branch 'origin/master' into accounts

docker/nginx/conf.d/client.conf

@@ -177,6 +177,9 @@ server {
 			-- example response: '{"skylink":"sia://XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"}'
 			local hnsres_json = json.decode(hnsres_res.body)
 
+			-- define local variable containing rest of the skylink if provided
+			local skylink_rest
+
 			if hnsres_json.skylink then
 				-- try to match the skylink with sia:// prefix
 				skylink, skylink_rest = string.match(hnsres_json.skylink, "sia://([^/?]+)(.*)")
@@ -190,7 +193,7 @@ server {
 				local datakey = hnsres_json.registry.datakey
 
 				-- make a get request to /skynet/registry endpoint with the credentials from text record
-				local registry_res = ngx.location.capture("/skynet/registry?publickey=" .. publickey .. "&datakey=" .. datakey)
+				local registry_res = ngx.location.capture("/skynet/registry/cached?publickey=" .. publickey .. "&datakey=" .. datakey)
 
 				-- we want to fail with a generic 404 when /skynet/registry returns anything but 200 OK
 				if registry_res.status ~= ngx.HTTP_OK then
@@ -213,7 +216,7 @@ server {
 			end
 
 			ngx.var.skylink = skylink
-			if request_uri_rest == "/" and skylink_rest ~= "" and skylink_rest ~= "/" then
+			if request_uri_rest == "/" and skylink_rest ~= nil and skylink_rest ~= "" and skylink_rest ~= "/" then
 				ngx.var.rest = skylink_rest
 			else
 				ngx.var.rest = request_uri_rest
@@ -248,6 +251,22 @@ server {
 		proxy_pass http://handshake-api:3100;
 	}
 
+	# internal registry endpoint that caches calls for a certain period of time
+	# it is not suitable for every registry call but some requests might be cached
+	# and we are using it currently for caching registry resolutions from /hns calls
+	location /skynet/registry/cached {
+		internal; # internal endpoint only
+		access_log off; # do not log traffic 
+
+		proxy_cache skynet;
+		proxy_cache_key publickey=$arg_publickey&datakey=$arg_datakey; # cache based on publickey and datakey
+		proxy_cache_valid 200 30s; # cache only 200 responses and only for 30 seconds
+		proxy_cache_lock on; # queue cache requests for the same resource until it is fully cached
+		proxy_cache_bypass $cookie_nocache $arg_nocache; # add cache bypass option
+
+		proxy_pass http://127.0.0.1/skynet/registry$is_args$args;
+	}
+
 	location /skynet/registry {
 		include /etc/nginx/conf.d/include/cors;
 		include /etc/nginx/conf.d/include/sia-auth;

packages/health-check/src/checks/critical.js

@@ -1,6 +1,6 @@
 const fs = require("fs");
 const superagent = require("superagent");
-const tmp = require('tmp');
+const tmp = require("tmp");
 const { StatusCodes } = require("http-status-codes");
 const { calculateElapsedTime, getResponseContent } = require("../utils");
 
@@ -8,7 +8,7 @@ const { calculateElapsedTime, getResponseContent } = require("../utils");
 async function uploadCheck(done) {
   const time = process.hrtime();
   const file = tmp.fileSync();
-  
+
   fs.writeSync(file.fd, Buffer.from(new Date())); // write current date to temp file
 
   superagent

packages/health-check/src/checks/verbose.js

@@ -874,11 +874,11 @@ function skyGalleryRedirectCheck(done) {
 // for the uncensored library skylink
 function uncensoredLibraryCheck(done) {
   const linkInfo = {
-    name: "Uncensored Library",
+    name: "The Uncensored Library V2",
     skylink: "AAC5glnZyNJ4Ieb4MhnYJGtID6qdMqEjl0or5EvEMt7bWQ",
     bodyHash: "60da6cb958699c5acd7f2a2911656ff32fca89a7",
     headers: {
-      "skynet-skylink": undefined,
+      "skynet-skylink": "AAC5glnZyNJ4Ieb4MhnYJGtID6qdMqEjl0or5EvEMt7bWQ",
       "skynet-file-metadata": {
         filename: "Unzip_The_Uncensored_Library_Map.zip",
         subfiles: {
@@ -889,8 +889,59 @@ function uncensoredLibraryCheck(done) {
           },
         },
       },
-      "content-disposition": 'attachment; filename="Unzip_The_Uncensored_Library_Map.zip"',
+      "content-disposition": 'inline; filename="Unzip_The_Uncensored_Library_Map.zip"',
-      "content-type": "application/octet-stream",
+      "content-type": "application/zip",
+    },
+  };
+
+  skylinkVerification(done, linkInfo);
+}
+
+function uncensoredLibraryPressReleaseCheck(done) {
+  const linkInfo = {
+    name: "The Uncensored Library - Press Release",
+    skylink: "AABHwuml_EhvyY8Gm7j1E2xGwodUNAJgX0A6-Cd22p9kNA",
+    bodyHash: "323217f643c3e3f1fe7532e72ac01bb0748c97be",
+    headers: {
+      "skynet-skylink": "AABHwuml_EhvyY8Gm7j1E2xGwodUNAJgX0A6-Cd22p9kNA",
+      "skynet-file-metadata": {
+        filename: "press-release-Reporters-Without-Borders-The-Uncensored-Library.zip",
+        subfiles: {
+          "press-release-Reporters-Without-Borders-The-Uncensored-Library.zip": {
+            filename: "press-release-Reporters-Without-Borders-The-Uncensored-Library.zip",
+            contenttype: "application/zip",
+            len: 383501533,
+          },
+        },
+      },
+      "content-disposition": 'inline; filename="press-release-Reporters-Without-Borders-The-Uncensored-Library.zip"',
+      "content-type": "application/zip",
+    },
+  };
+
+  // request too large, use head just to verify the headers
+  skylinkVerification(done, linkInfo, { method: "head" });
+}
+
+function uncensoredLibraryV2Check(done) {
+  const linkInfo = {
+    name: "The Uncensored Library V2",
+    skylink: "AAAs-JOsRGWgABYIo7AwTDqSX79-BxQKjDj0wiRGoRPFnw",
+    bodyHash: "1c6a885c060af8325eee82a11e9d64a13b228015",
+    headers: {
+      "skynet-skylink": "AAAs-JOsRGWgABYIo7AwTDqSX79-BxQKjDj0wiRGoRPFnw",
+      "skynet-file-metadata": {
+        filename: "The Uncensored Library V2.zip",
+        subfiles: {
+          "The Uncensored Library V2.zip": {
+            filename: "The Uncensored Library V2.zip",
+            contenttype: "application/zip",
+            len: 101262134,
+          },
+        },
+      },
+      "content-disposition": 'inline; filename="The Uncensored Library V2.zip"',
+      "content-type": "application/zip",
     },
   };
 
@@ -1024,6 +1075,20 @@ function fileEndpointCheck(done) {
   skylinkVerification(done, linkInfo);
 }
 
+// check whether hns/note-to-self would properly redirect to note-to-self/
+function hnsEndpointDirectoryRedirect(done) {
+  const expected = {
+    name: "hns endpoint directory redirect",
+    skylink: "hns/note-to-self",
+    statusCode: 307,
+    headers: {
+      location: "note-to-self/",
+    },
+  };
+
+  skylinkVerification(done, expected, { redirects: 0 });
+}
+
 function parseHeaderString(header) {
   try {
     return JSON.parse(header);
@@ -1033,26 +1098,35 @@ function parseHeaderString(header) {
 }
 
 // skylinkVerification verifies a skylink against provided information.
-function skylinkVerification(done, { name, skylink, bodyHash, headers }) {
+function skylinkVerification(done, { name, skylink, bodyHash, headers, statusCode }, { redirects, method } = {}) {
   const time = process.hrtime();
 
   // Create the query for the skylink
-  const query = `${process.env.PORTAL_URL}/${skylink}?nocache=true`;
+  const query = `${process.env.PORTAL_URL}/${skylink}`;
 
   // Get the Skylink
-  superagent
+  superagent[method || "get"](query)
-    .get(query)
+    .set("cookie", "nocache=true")
+    .redirects(redirects)
+    .ok((res) => (redirects === undefined ? res.ok : res.status < 400))
     .responseType("blob")
     .then(
       (response) => {
         const entry = { name, up: true, statusCode: response.statusCode, time: calculateElapsedTime(time) };
         const info = {};
 
-        // Check if the response body is valid by checking against the known hash
+        if (statusCode && statusCode !== response.statusCode) {
-        const currentBodyHash = hash(response.body);
-        if (currentBodyHash !== bodyHash) {
           entry.up = false;
-          info.bodyHash = { expected: bodyHash, current: currentBodyHash };
+          info.statusCode = { expected: statusCode, current: response.statusCode };
+        }
+
+        // Check if the response body is valid by checking against the known hash
+        if (bodyHash) {
+          const currentBodyHash = hash(response.body);
+          if (currentBodyHash !== bodyHash) {
+            entry.up = false;
+            info.bodyHash = { expected: bodyHash, current: currentBodyHash };
+          }
         }
 
         if (headers) {
@@ -1065,7 +1139,7 @@ function skylinkVerification(done, { name, skylink, bodyHash, headers }) {
               if (typeof currentHeader === "object") {
                 info.headers[headerName] = ensureValidJSON(detailedDiff(expectedHeader, currentHeader));
               } else {
-                info.headers[headerName] = currentHeader;
+                info.headers[headerName] = { expected: expectedHeader, current: currentHeader };
               }
             }
           });
@@ -1076,6 +1150,8 @@ function skylinkVerification(done, { name, skylink, bodyHash, headers }) {
         done(entry); // Return the entry information
       },
       (error) => {
+        console.log(error);
+
         done({
           name,
           up: false,
@@ -1109,6 +1185,8 @@ module.exports = [
   skyGalleryIndexFileCheck,
   skyGalleryRedirectCheck,
   uncensoredLibraryCheck,
+  uncensoredLibraryPressReleaseCheck,
+  uncensoredLibraryV2Check,
   fileEndpointCheck,
   bitcoinWhitepaper,
   // uniswapIndexFileCheck,
@@ -1118,4 +1196,5 @@ module.exports = [
   // uniswapHNSRedirectCheck,
   uniswapHNSResolverCheck,
   uniswapHNSResolverRedirectCheck,
+  hnsEndpointDirectoryRedirect,
 ];

packages/health-check/src/run.js

@@ -1,21 +1,25 @@
-require('yargs/yargs')(process.argv.slice(2))
+require("yargs/yargs")(process.argv.slice(2)).command(
-  .command('$0 <type>', 'Skynet portal health checks', (yargs) => {
+  "$0 <type>",
-    yargs.positional('type', {
+  "Skynet portal health checks",
-        describe: 'Type of checks to run',
+  (yargs) => {
-        type: 'string',
+    yargs
-        choices: ["critical", "verbose"]
+      .positional("type", {
-    })  
+        describe: "Type of checks to run",
-    .option("portal-url", {
+        type: "string",
+        choices: ["critical", "verbose"],
+      })
+      .option("portal-url", {
         describe: "Skynet portal url",
         default: process.env.PORTAL_URL || "https://siasky.net",
         type: "string",
-    })
+      })
-    .option("state-dir", {
+      .option("state-dir", {
         describe: "State directory",
         default: process.env.STATE_DIR || "state",
         type: "string",
-    })
+      });
-  }, async ({ type, portalUrl, stateDir }) => {
+  },
+  async ({ type, portalUrl, stateDir }) => {
     process.env.PORTAL_URL = portalUrl;
     process.env.STATE_DIR = stateDir;
 
@@ -23,11 +27,11 @@ require('yargs/yargs')(process.argv.slice(2))
     const checks = require(`../src/checks/${type}`);
 
     const entry = {
-        date: new Date().toISOString(),
+      date: new Date().toISOString(),
-        checks: await Promise.all(checks.map((check) => new Promise(check))),
+      checks: await Promise.all(checks.map((check) => new Promise(check))),
     };
 
     // read before writing to make sure no external changes are overwritten
     db.read().get(type).push(entry).write();
-  })
+  }
-  .argv
+).argv;

packages/webapp/cypress/integration/intergration.spec.js

@@ -12,12 +12,12 @@ context("Skynet", () => {
   });
 
   it("should be able to upload a file", () => {
-    cy.server();
+    cy.intercept("POST", "/skynet/skyfile").as("upload");
-    cy.route("POST", "/skynet/skyfile").as("upload");
 
     const fileName = "check.json";
 
-    cy.get('.home-upload input[type="file"]').attachFile(fileName);
+    cy.wait(1000); // delay for drag-and-drop to work properly every time
+    cy.get('.home-upload input[type="file"]').attachFile(fileName, { subjectType: "drag-n-drop" });
 
     cy.get(".home-upload").scrollIntoView();
     cy.get(".home-uploaded-files").children().should("have.length", 1);
@@ -32,7 +32,6 @@ context("Skynet", () => {
 
       cy.contains("Copy Link").click();
       cy.contains("Copied!").should("be.visible");
-      cy.contains("Copied!").should("not.be.visible");
     });
   });
 });

packages/webapp/gatsby-config.js

@@ -18,8 +18,6 @@ module.exports = {
         path: `${__dirname}/src/images`,
       },
     },
-    `gatsby-transformer-sharp`,
-    `gatsby-plugin-sharp`,
     `gatsby-plugin-robots-txt`,
     {
       resolve: `gatsby-plugin-manifest`,

packages/webapp/package.json

@@ -16,9 +16,7 @@
     "gatsby-plugin-remove-serviceworker": "1.0.0",
     "gatsby-plugin-robots-txt": "1.5.5",
     "gatsby-plugin-sass": "3.1.0",
-    "gatsby-plugin-sharp": "2.13.1",
     "gatsby-source-filesystem": "2.10.0",
-    "gatsby-transformer-sharp": "2.11.0",
     "http-status-codes": "2.1.4",
     "js-cookie": "^2.2.1",
     "jsonp": "0.2.1",
@@ -34,7 +32,7 @@
     "react-mailchimp-subscribe": "2.1.0",
     "react-syntax-highlighter": "15.4.3",
     "react-visibility-sensor": "5.1.1",
-    "skynet-js": "2.4.0"
+    "skynet-js": "3.0.0"
   },
   "devDependencies": {
     "cypress": "6.3.0",

packages/webapp/src/components/CodeExamples/Code.js

@@ -14,7 +14,7 @@ const client = new SkynetClient();
 async function example() {
   try {
     // upload
-    const skylink = await client.uploadFile(file);
+    const { skylink } = await client.uploadFile(file);
 	console.log(\`Upload successful, skylink: \${skylink}\`);
 
     // download

packages/webapp/src/components/HomeUpload/HomeUpload.js

@@ -130,7 +130,7 @@ export default function HomeUpload() {
             response = await client.uploadFile(file, { onUploadProgress });
           }
 
-          onFileStateChange(file, { status: "complete", url: client.getSkylinkUrl(response) });
+          onFileStateChange(file, { status: "complete", url: client.getSkylinkUrl(response.skylink) });
         } catch (error) {
           if (error.response && error.response.status === StatusCodes.TOO_MANY_REQUESTS) {
             onFileStateChange(file, { progress: -1 });