Merge pull request #781 from SkynetLabs/extend-critical-health-checks

Extend critical health checks

packages/health-check/Dockerfile

@@ -2,14 +2,23 @@ FROM node:16.1.0-alpine
 
 WORKDIR /usr/app
 
+# schedule critical checks to run every 5 minutes (any failures will disable server)
+RUN echo '*/5 * * * * /usr/app/cli/run critical > /dev/stdout' >> /etc/crontabs/root
+
+# schedule extended checks to run on every hour (optional checks, report only)
+RUN echo '0 * * * * /usr/app/cli/run extended > /dev/stdout' >> /etc/crontabs/root
+
 COPY package.json .
 RUN yarn --no-lockfile
 COPY src src
 COPY cli cli
 
-RUN echo '*/5 * * * * /usr/app/cli/run critical > /dev/stdout' >> /etc/crontabs/root
-RUN echo '0 * * * * /usr/app/cli/run extended > /dev/stdout' >> /etc/crontabs/root
-
 EXPOSE 3100
 ENV NODE_ENV production
+
+# command consists of 3 parts:
+# 1. starting crond
+# 2. aliasing siasky.net and account.siasky.net with current server ip so health checks 
+#    test portal end-to-end on prod domain (important for testing ssl certificates)
+# 3. running api service
 CMD [ "sh", "-c", "crond ; echo $(node src/whatismyip.js) siasky.net account.siasky.net >> /etc/hosts ; node --max-http-header-size=64000 src/index.js" ]

packages/health-check/src/checks/critical.js

@@ -1,7 +1,10 @@
 const got = require("got");
 const FormData = require("form-data");
-const { StatusCodes } = require("http-status-codes");
 const { calculateElapsedTime, getResponseContent } = require("../utils");
+const { SkynetClient } = require("skynet-js");
+
+const skynetClient = new SkynetClient(process.env.SKYNET_PORTAL_API);
+const exampleSkylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q";
 
 // uploadCheck returns the result of uploading a sample file
 async function uploadCheck(done) {
@@ -25,39 +28,43 @@ async function uploadCheck(done) {
     data.ip = error?.response?.ip ?? null;
   }
 
-  done({
+  done({ name: "upload_file", time: calculateElapsedTime(time), ...data });
-    name: "upload_file",
+}
-    time: calculateElapsedTime(time),
+
-    ...data,
+// websiteCheck checks whether the main website is working
-  });
+async function websiteCheck(done) {
+  return genericAccessCheck("website", process.env.SKYNET_PORTAL_API, done);
 }
 
 // downloadCheck returns the result of downloading the hard coded link
 async function downloadCheck(done) {
-  const time = process.hrtime();
+  const url = await skynetClient.getSkylinkUrl(exampleSkylink);
-  const skylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q";
-  const data = { up: false };
 
-  try {
+  return genericAccessCheck("skylink", url, done);
-    const response = await got(`${process.env.SKYNET_PORTAL_API}/${skylink}?nocache=true`);
-
-    data.statusCode = response.statusCode;
-    data.up = true;
-    data.ip = response.ip;
-  } catch (error) {
-    data.statusCode = error?.response?.statusCode || error.statusCode || error.status;
-    data.errorMessage = error.message;
-    data.errorResponseContent = getResponseContent(error.response);
-    data.ip = error?.response?.ip ?? null;
 }
 
-  done({
+// skylinkSubdomainCheck returns the result of downloading the hard coded link via subdomain
-    name: "download_file",
+async function skylinkSubdomainCheck(done) {
-    time: calculateElapsedTime(time),
+  const url = await skynetClient.getSkylinkUrl(exampleSkylink, { subdomain: true });
-    ...data,
+
-  });
+  return genericAccessCheck("skylink_via_subdomain", url, done);
 }
 
+// handshakeSubdomainCheck returns the result of downloading the skylink via handshake domain
+async function handshakeSubdomainCheck(done) {
+  const url = await skynetClient.getHnsUrl("note-to-self", { subdomain: true });
+
+  return genericAccessCheck("hns_via_subdomain", url, done);
+}
+
+// accountWebsiteCheck returns the result of accessing account dashboard website
+async function accountWebsiteCheck(done) {
+  const url = `${process.env.SKYNET_DASHBOARD_URL}/auth/login`;
+
+  return genericAccessCheck("account_website", url, done);
+}
+
+// accountHealthCheck returns the result of accounts service health checks
 async function accountHealthCheck(done) {
   const time = process.hrtime();
   const data = { up: false };
@@ -76,17 +83,33 @@ async function accountHealthCheck(done) {
     data.ip = error?.response?.ip ?? null;
   }
 
-  done({
+  done({ name: "accounts", time: calculateElapsedTime(time), ...data });
-    name: "account_health",
-    time: calculateElapsedTime(time),
-    ...data,
-  });
 }
 
-const checks = [uploadCheck, downloadCheck];
+async function genericAccessCheck(name, url, done) {
+  const time = process.hrtime();
+  const data = { up: false, url };
+
+  try {
+    const response = await got(url, { headers: { cookie: "nocache=true" } });
+
+    data.statusCode = response.statusCode;
+    data.up = true;
+    data.ip = response.ip;
+  } catch (error) {
+    data.statusCode = error?.response?.statusCode || error.statusCode || error.status;
+    data.errorMessage = error.message;
+    data.errorResponseContent = getResponseContent(error.response);
+    data.ip = error?.response?.ip ?? null;
+  }
+
+  done({ name, time: calculateElapsedTime(time), ...data });
+}
+
+const checks = [uploadCheck, websiteCheck, downloadCheck, skylinkSubdomainCheck, handshakeSubdomainCheck];
 
 if (process.env.ACCOUNTS_ENABLED) {
-  checks.push(accountHealthCheck);
+  checks.push(accountHealthCheck, accountWebsiteCheck);
 }
 
 module.exports = checks;