Merge branch 'master' into sevey/add-skylink-health-check

Co-authored-by: Karol Wypchło <kwypchlo@gmail.com>

packages/health-check/src/checks/critical.js

@@ -1,7 +1,14 @@
 const got = require("got");
-const FormData = require("form-data");
 const { isEqual } = require("lodash");
-const { calculateElapsedTime, getResponseContent, getAuthCookie, isPortalModuleEnabled } = require("../utils");
+const {
+  calculateElapsedTime,
+  getResponseContent,
+  getAuthCookie,
+  isPortalModuleEnabled,
+  uploadFunc,
+  siaDockerContainerIP,
+  defaultSiaPort,
+} = require("../utils");
 const { SkynetClient, stringToUint8ArrayUtf8, genKeyPairAndSeed } = require("skynet-js");
 
 const MODULE_BLOCKER = "b";
@@ -15,7 +22,9 @@ async function skydConfigCheck(done) {
   const data = { up: false };
 
   try {
-    const response = await got(`http://10.10.10.10:9980/renter`, { headers: { "User-Agent": "Sia-Agent" } }).json();
+    const response = await got(`http://${siaDockerContainerIP}:${defaultSiaPort}/renter`, {
+      headers: { "User-Agent": "Sia-Agent" },
+    }).json();
 
     // make sure initial funding is set to 10SC
     if (response.settings.allowance.paymentcontractinitialfunding !== "10000000000000000000000000") {
@@ -36,31 +45,9 @@ async function skydConfigCheck(done) {
 
 // uploadCheck returns the result of uploading a sample file
 async function uploadCheck(done) {
-  const authCookie = await getAuthCookie();
-  const time = process.hrtime();
-  const form = new FormData();
   const payload = Buffer.from(new Date()); // current date to ensure data uniqueness
-  const data = { up: false };
 
-  form.append("file", payload, { filename: "time.txt", contentType: "text/plain" });
-
-  try {
-    const response = await got.post(`https://${process.env.PORTAL_DOMAIN}/skynet/skyfile`, {
-      body: form,
-      headers: { cookie: authCookie },
-    });
-
-    data.statusCode = response.statusCode;
-    data.up = true;
-    data.ip = response.ip;
-  } catch (error) {
-    data.statusCode = error.response?.statusCode || error.statusCode || error.status;
-    data.errorMessage = error.message;
-    data.errorResponseContent = getResponseContent(error.response);
-    data.ip = error?.response?.ip ?? null;
-  }
-
-  done({ name: "upload_file", time: calculateElapsedTime(time), ...data });
+  return uploadFunc(done, payload, "upload_file");
 }
 
 // websiteCheck checks whether the main website is working

packages/health-check/src/checks/extended.js

@@ -1,8 +1,16 @@
+const crypto = require("crypto");
 const got = require("got");
 const hasha = require("hasha");
 const { detailedDiff } = require("deep-object-diff");
 const { isEqual } = require("lodash");
-const { calculateElapsedTime, ensureValidJSON, getResponseContent, getAuthCookie } = require("../utils");
+const {
+  calculateElapsedTime,
+  ensureValidJSON,
+  getResponseContent,
+  getAuthCookie,
+  sectorSize,
+  uploadFunc,
+} = require("../utils");
 const { parseSkylink } = require("skynet-js");
 
 // audioExampleCheck returns the result of trying to download the skylink
@@ -1206,6 +1214,13 @@ async function skylinkVerification(done, expected, { followRedirect = true, meth
   }
 }
 
+// uploadLargeFileCheck returns the result of uploading a large file
+async function uploadLargeFileCheck(done) {
+  const payload = Buffer.from(crypto.randomBytes(sectorSize));
+
+  return uploadFunc(done, payload, "upload_large_file", true);
+}
+
 module.exports = [
   audioExampleCheck,
   covid19PaperCheck,
@@ -1244,4 +1259,5 @@ module.exports = [
   skappHackerPaste,
   skappHowAboutSkapp,
   skappSkyDeploy,
+  uploadLargeFileCheck,
 ];

packages/health-check/src/utils.js

@@ -1,8 +1,30 @@
+const FormData = require("form-data");
+const got = require("got");
 const ipCheckService = "whatismyip.akamai.com";
 const ipRegex = new RegExp(
   `^(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)(?:\\.(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)){3}$`
 );
 
+// sectorSize is the skyd sector size
+const sectorSize = 1 << 22; // 40 MiB
+
+// SECOND is a helper constant for defining the number of milliseconds in a
+// second
+const SECOND = 1000;
+
+// defaultBaseSectorRedundancy is the default baseSectorRedundancy defined by
+// skyd
+const defaultBaseSectorRedundancy = 10;
+
+// defaultFanoutRedundancy is the default fanout redundancy defined by skyd
+const defaultFanoutRedundancy = 3;
+
+// siaDockerContainerIP is the local IP of the sia docker container
+const siaDockerContainerIP = "10.10.10.10";
+
+// defaultSiaPort is the default port the sia runs on.
+const defaultSiaPort = "9980";
+
 /**
  * Get the time between start and now in milliseconds
  */
@@ -132,6 +154,92 @@ function isPortalModuleEnabled(module) {
   return process.env.PORTAL_MODULES && process.env.PORTAL_MODULES.indexOf(module) !== -1;
 }
 
+// sleep is a helper method of sleeping for for given time. The input time is
+// expected in seconds
+async function sleep(seconds) {
+  return new Promise((r) => setTimeout(r, seconds * SECOND));
+}
+
+// skylinkHealthCheck checks if the skylink has reached full redundancy
+async function skylinkHealthCheck(skylink, numRetries = 30, authCookie, isLarge = false) {
+  // Get the health of the skylink
+  const response = await got(`http://${siaDockerContainerIP}:${defaultSiaPort}/skynet/health/skylink/${skylink}`, {
+    headers: { "User-Agent": "Sia-Agent", cookie: authCookie },
+  });
+  const healthData = getResponseContent(response);
+
+  // Check Basesectorredundancy first
+  if (healthData.basesectorredundancy !== defaultBaseSectorRedundancy && numRetries > 0) {
+    // Semi-smart sleep before retrying. Sleep longer if the redundancy is
+    // lower.
+    let sleepTime = defaultBaseSectorRedundancy - healthData.basesectorredundancy;
+    await sleep(sleepTime);
+    return skylinkHealthCheck(skylink, numRetries - 1, authCookie, isLarge);
+  }
+
+  // Check the Fanout redundancy if it is a large file
+  if (isLarge && healthData.fanoutredundancy != defaultFanoutRedundancy && numRetries > 0) {
+    // Semi-smart sleep before retrying. Sleep longer if the redundancy is
+    // lower.
+    let sleepTime = (defaultFanoutRedundancy - healthData.fanoutredundancy) * 10;
+    await sleep(sleepTime);
+    return skylinkHealthCheck(skylink, numRetries - 1, authCookie, isLarge);
+  }
+
+  // Throw error if the basesectorredundancy never reached 10x
+  if (healthData.basesectorredundancy !== defaultBaseSectorRedundancy && numRetries === 0) {
+    throw new Error(`File uploaded but basesector did not reach full redundancy: ${healthData.basesectorredundancy}`);
+  }
+
+  // Throw error if the fanoutredundancy never reached 3x
+  if (isLarge && healthData.fanoutredundancy !== defaultFanoutRedundancy && numRetries === 0) {
+    throw new Error(`File uploaded but fanout did not reach full redundancy: ${healthData.fanoutredundancy}`);
+  }
+
+  return response;
+}
+
+// uploadFunc handles the upload and health check for the upload checks
+async function uploadFunc(done, payload, name, isLarge = false) {
+  // Get time for calculating the elapsed time for the check
+  const time = process.hrtime();
+
+  // Initialize check params
+  const authCookie = await getAuthCookie();
+  const data = { up: false };
+  const form = new FormData();
+
+  form.append("file", payload, { filename: `${name}.txt`, contentType: "text/plain" });
+
+  let skylink;
+  try {
+    // Upload file
+    const response = await got.post(`https://${process.env.PORTAL_DOMAIN}/skynet/skyfile`, {
+      body: form,
+      headers: { cookie: authCookie },
+    });
+
+    // Check file health
+    const responseContent = getResponseContent(response);
+    skylink = responseContent.skylink;
+    await skylinkHealthCheck(skylink, 60, authCookie, isLarge);
+
+    // Update data response
+    data.statusCode = response.statusCode;
+    data.up = true;
+    data.ip = response.ip;
+    data.skylink = skylink;
+  } catch (error) {
+    data.statusCode = error.response?.statusCode || error.statusCode || error.status;
+    data.errorMessage = error.message;
+    data.errorResponseContent = getResponseContent(error.response);
+    data.ip = error?.response?.ip ?? null;
+    data.skylink = skylink;
+  }
+
+  done({ name, time: calculateElapsedTime(time), ...data });
+}
+
 module.exports = {
   calculateElapsedTime,
   getYesterdayISOString,
@@ -141,4 +249,8 @@ module.exports = {
   isPortalModuleEnabled,
   ipCheckService,
   ipRegex,
+  sectorSize,
+  siaDockerContainerIP,
+  defaultSiaPort,
+  uploadFunc,
 };