Merge pull request #388 from NebulousLabs/improve-health-checks

improve health checks
2020-09-09 15:09:09 +02:00 · 2020-09-09 15:09:09 +02:00 · da1cf14e5f
parent a4dd527a30 19b8caf81a
commit da1cf14e5f
15 changed files with 177 additions and 117 deletions
--- a/packages/health-check/.gitignore
+++ b/packages/health-check/.gitignore
@ -0,0 +1 @@
+state/
--- a/packages/health-check/Dockerfile
+++ b/packages/health-check/Dockerfile
@ -5,7 +5,8 @@ WORKDIR /usr/app
 COPY package.json .
 RUN yarn --no-lockfile
 COPY src/* src/
+COPY cli/* cli/

 EXPOSE 3100
 ENV NODE_ENV production
-CMD [ "node", "src/index.js" ]
+CMD [ "node", "--max-http-header-size=64000", "src/index.js" ]
--- a/packages/health-check/cli/disable
+++ b/packages/health-check/cli/disable
@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+process.env.NODE_ENV = process.env.NODE_ENV || "production";
+
+const db = require("../src/db");
+
+db.set("disabled", true).write();
--- a/packages/health-check/cli/enable
+++ b/packages/health-check/cli/enable
@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+process.env.NODE_ENV = process.env.NODE_ENV || "production";
+
+const db = require("../src/db");
+
+db.set("disabled", false).write();
--- a/packages/health-check/src/api/all.js
+++ b/packages/health-check/src/api/all.js
@ -0,0 +1,8 @@
+const db = require("../db");
+
+// returns all health check entries
+module.exports = (req, res) => {
+  const entries = db.get("entries").orderBy("date", "desc").value();
+
+  res.send(entries);
+};
--- a/packages/health-check/src/api/disabled.js
+++ b/packages/health-check/src/api/disabled.js
@ -0,0 +1,8 @@
+const db = require("../db");
+
+// returns a disabled flag status
+module.exports = (req, res) => {
+  const disabled = db.get("disabled").value();
+
+  res.send({ disabled });
+};
--- a/packages/health-check/src/endpointHealthCheck.js
+++ b/packages/health-check/src/endpointHealthCheck.js
@ -1,9 +1,15 @@
 const { StatusCodes } = require("http-status-codes");
 const { sum, sumBy } = require("lodash");
-const db = require("./db");
+const db = require("../db");

 // getStatus returns the server's current health check status
 function getStatus() {
+  const disabled = db.get("disabled").value();
+
+  if (disabled) {
+    return StatusCodes.SERVICE_UNAVAILABLE;
+  }
+
  // Grab entry element from DB
  const entry = db.get("entries").orderBy("date", "desc").head().value();

--- a/packages/health-check/src/api/recent.js
+++ b/packages/health-check/src/api/recent.js
@ -0,0 +1,16 @@
+const db = require("../db");
+
+// returns all health check entries that are not older than one day
+module.exports = (req, res) => {
+  const yesterday = new Date();
+
+  yesterday.setDate(yesterday.getDate() - 1);
+
+  const entries = db
+    .get("entries")
+    .orderBy("date", "desc")
+    .filter(({ date }) => date >= yesterday.toISOString())
+    .value();
+
+  res.send(entries);
+};
--- a/packages/health-check/src/basicChecks.js
+++ b/packages/health-check/src/basicChecks.js
@ -1,51 +0,0 @@
-const superagent = require("superagent");
-const { StatusCodes } = require("http-status-codes");
-
-// uploadCheck returns the result of uploading a sample file
-async function uploadCheck(done) {
-  const time = process.hrtime();
-
-  superagent
-    .post(`http://${process.env.PORTAL_URL}/skynet/skyfile`)
-    .attach("file", "package.json", "package.json")
-    .end((err, res) => {
-      const statusCode = (res && res.statusCode) || (err && err.statusCode) || null;
-
-      done({
-        name: "upload_file",
-        up: statusCode === StatusCodes.OK,
-        statusCode,
-        time: catchRequestTime(time),
-        critical: true,
-      });
-    });
-}
-
-// downloadCheck returns the result of downloading the hard coded link
-function downloadCheck(done) {
-  const time = process.hrtime();
-  const skylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q";
-
-  superagent.get(`http://${process.env.PORTAL_URL}/${skylink}?nocache=true`).end((err, res) => {
-    const statusCode = (res && res.statusCode) || (err && err.statusCode) || null;
-
-    done({
-      name: "download_file",
-      up: statusCode === StatusCodes.OK,
-      statusCode,
-      time: catchRequestTime(time),
-      critical: true,
-    });
-  });
-}
-
-// catchRequestTime records the time it took to resolve the request in
-// milliseconds
-function catchRequestTime(start) {
-  const diff = process.hrtime(start);
-
-  return Math.round((diff[0] * 1e9 + diff[1]) / 1e6); // msec
-}
-
-module.exports.basicChecks = [uploadCheck, downloadCheck];
-module.exports.catchRequestTime = catchRequestTime;
--- a/packages/health-check/src/checks/critical.js
+++ b/packages/health-check/src/checks/critical.js
@ -0,0 +1,48 @@
+const superagent = require("superagent");
+const { StatusCodes } = require("http-status-codes");
+const { calculateElapsedTime } = require("../utils");
+
+// uploadCheck returns the result of uploading a sample file
+async function uploadCheck(done) {
+  const time = process.hrtime();
+
+  superagent
+    .post(`http://${process.env.PORTAL_URL}/skynet/skyfile`)
+    .attach("file", "package.json", "package.json")
+    .end((error, response) => {
+      const statusCode = (response && response.statusCode) || (error && error.statusCode) || null;
+
+      done({
+        name: "upload_file",
+        up: statusCode === StatusCodes.OK,
+        statusCode,
+        time: calculateElapsedTime(time),
+        critical: true,
+      });
+    });
+}
+
+// downloadCheck returns the result of downloading the hard coded link
+async function downloadCheck(done) {
+  const time = process.hrtime();
+  const skylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q";
+  let statusCode;
+
+  try {
+    const response = await superagent.get(`http://${process.env.PORTAL_URL}/${skylink}?nocache=true`);
+
+    statusCode = response.statusCode;
+  } catch (error) {
+    statusCode = error.statusCode || error.status;
+  }
+
+  done({
+    name: "download_file",
+    up: statusCode === StatusCodes.OK,
+    statusCode,
+    time: calculateElapsedTime(time),
+    critical: true,
+  });
+}
+
+module.exports.criticalChecks = [uploadCheck, downloadCheck];
--- a/packages/health-check/src/checks/verbose.js
+++ b/packages/health-check/src/checks/verbose.js
@ -2,13 +2,13 @@ const superagent = require("superagent");
 const hash = require("object-hash");
 const { detailedDiff } = require("deep-object-diff");
 const { isEqual } = require("lodash");
-const checks = require("./basicChecks");
+const { calculateElapsedTime } = require("../utils");

 // audioExampleCheck returns the result of trying to download the skylink
 // for the Example audio file on siasky.net
 function audioExampleCheck(done) {
  const linkInfo = {
-    description: "Audio Example",
+    name: "Audio Example",
    skylink: "_A2zt5SKoqwnnZU4cBF8uBycSKULXMyeg1c5ZISBr2Q3dA",
    bodyHash: "be335f5ad9bc357248f3d35c7e49df491afb6b12",
    metadata: { filename: "feel-good.mp3" },
@ -21,7 +21,7 @@ function audioExampleCheck(done) {
 // for a known Covid19 paper
 function covid19PaperCheck(done) {
  const linkInfo = {
-    description: "Covid-19 Paper",
+    name: "Covid-19 Paper",
    skylink: "PAMZVmfutxWoG6Wnl5BRKuWLkDNZR42k_okRRvksJekA3A",
    bodyHash: "81b9fb74829a96ceafa429840d1ef0ce44376ddd",
    metadata: {
@ -43,7 +43,7 @@ function covid19PaperCheck(done) {
 // for another known Covid19 paper
 function covid19CoroNopePaperCheck(done) {
  const linkInfo = {
-    description: "Covid-19 CoroNope Paper",
+    name: "Covid-19 CoroNope Paper",
    skylink: "bACLKGmcmX4NCp47WwOOJf0lU666VLeT5HRWpWVtqZPjEA",
    bodyHash: "901f6fd65ef595f70b6bfebbb2d05942351ef2b3",
    metadata: { filename: "coronope.pdf" },
@ -56,8 +56,8 @@ function covid19CoroNopePaperCheck(done) {
 // for the Example Dapp on siasky.net
 function dappExampleCheck(done) {
  const linkInfo = {
-    description: "Dapp Example (UniSwap)",
-    skylink: "EAC5HJr5Pu086EAZG4fP_r6Pnd7Ft366vt6t2AnjkoFb9Q/index.html",
+    name: "Dapp Example (UniSwap)",
+    skylink: "EADWpKD0myqH2tZa6xtKebg6kNnwYnI94fl4R8UKgNrmOA",
    bodyHash: "d6ad2506590bb45b5acc6a8a964a3da4d657354f",
    metadata: {
      filename: "/index.html",
@ -73,7 +73,7 @@ function dappExampleCheck(done) {
 // for the Develop Momentum Application with a trailing /index.html
 function developMomentumIndexFileCheck(done) {
  const linkInfo = {
-    description: "Develop Momentum Index File",
+    name: "Develop Momentum Index File",
    skylink: "EAA1fG_ip4C1Vi1Ijvsr1oyr8jpH0Bo9HXya0T3kw-elGw/index.html",
    bodyHash: "53b44a9d3cfa9b3d66ce5c29976f4383725d3652",
    metadata: {
@ -90,7 +90,7 @@ function developMomentumIndexFileCheck(done) {
 // for the Example HTML file on siasky.net
 function htmlExampleCheck(done) {
  const linkInfo = {
-    description: "HTML Example",
+    name: "HTML Example",
    skylink: "PAL0w4SdA5rFCDGEutgpeQ50Om-YkBabtXVOJAkmedslKw",
    bodyHash: "c932fd56f98b6db589e56be8018817f13bb29f72",
    metadata: { filename: "introduction â Sia API Documentation.html" },
@ -103,7 +103,7 @@ function htmlExampleCheck(done) {
 // for the Example image on siasky.net
 function imageExampleCheck(done) {
  const linkInfo = {
-    description: "Image Example",
+    name: "Image Example",
    skylink: "IADUs8d9CQjUO34LmdaaNPK_STuZo24rpKVfYW3wPPM2uQ",
    bodyHash: "313207978d0a88bf2b961f098804e9ab0f82837f",
    metadata: { filename: "sia-lm.png" },
@ -116,7 +116,7 @@ function imageExampleCheck(done) {
 // for the Example JSON file on siasky.net
 function jsonExampleCheck(done) {
  const linkInfo = {
-    description: "JSON Example",
+    name: "JSON Example",
    skylink: "AAC0uO43g64ULpyrW0zO3bjEknSFbAhm8c-RFP21EQlmSQ",
    bodyHash: "198771c3d07d5c7302aadcc0697a7298e5e8ccc3",
    metadata: { filename: "consensus.json" },
@ -129,7 +129,7 @@ function jsonExampleCheck(done) {
 // for the Example PDF file on siasky.net
 function pdfExampleCheck(done) {
  const linkInfo = {
-    description: "PDF Example",
+    name: "PDF Example",
    skylink: "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg",
    bodyHash: "9bd8162e1575569a9041972f7f62d65887063dc3",
    metadata: { filename: "sia.pdf" },
@ -142,7 +142,7 @@ function pdfExampleCheck(done) {
 // a Random Image.
 function randomImageCheck(done) {
  const linkInfo = {
-    description: "Random Image",
+    name: "Random Image",
    skylink: "PAHx7JmsU9EFGbqm5q0LNKT2wKfoJ_mhPI8zWlNEXZ8uOQ/",
    bodyHash: "4c73c5a0eddd5823be677d7f93bf80cc9338ee9f",
    metadata: {
@ -159,7 +159,7 @@ function randomImageCheck(done) {
 // a Random Image with no trailing slash.
 function randomImageRedirectCheck(done) {
  const linkInfo = {
-    description: "Random Image Redirect",
+    name: "Random Image Redirect",
    skylink: "PAHx7JmsU9EFGbqm5q0LNKT2wKfoJ_mhPI8zWlNEXZ8uOQ",
    bodyHash: "4c73c5a0eddd5823be677d7f93bf80cc9338ee9f",
    metadata: {
@ -175,7 +175,7 @@ function randomImageRedirectCheck(done) {
 // skyBayCheck returns the result of trying to download the skylink for the SkyBay Application.
 function skyBayCheck(done) {
  const linkInfo = {
-    description: "SkyBay",
+    name: "SkyBay",
    skylink: "EABkMjXzxJRpPz0eO0Or5fy2eo-rz3prdigGwRlyNd9mwA/",
    bodyHash: "25d63937c9734fb08d2749c6517d1b3de8ecb856",
    metadata: {
@ -191,7 +191,7 @@ function skyBayCheck(done) {
 // for the SkyBay Application with no trailing slash.
 function skyBayRedirectCheck(done) {
  const linkInfo = {
-    description: "SkyBay Redirect",
+    name: "SkyBay Redirect",
    skylink: "EABkMjXzxJRpPz0eO0Or5fy2eo-rz3prdigGwRlyNd9mwA",
    bodyHash: "25d63937c9734fb08d2749c6517d1b3de8ecb856",
    metadata: {
@ -206,7 +206,7 @@ function skyBayRedirectCheck(done) {
 // skyBinCheck returns the result of trying to download the skylink for the SkyBin Application.
 function skyBinCheck(done) {
  const linkInfo = {
-    description: "SkyBin",
+    name: "SkyBin",
    skylink: "CAAVU14pB9GRIqCrejD7rlS27HltGGiiCLICzmrBV0wVtA/",
    bodyHash: "767ec67c417e11b97c5db7dad9ea3b6b27cb0d39",
    metadata: { filename: "skybin.html" },
@ -219,7 +219,7 @@ function skyBinCheck(done) {
 // for the SkyBin Application with no trailing slash.
 function skyBinRedirectCheck(done) {
  const linkInfo = {
-    description: "SkyBin Redirect",
+    name: "SkyBin Redirect",
    skylink: "CAAVU14pB9GRIqCrejD7rlS27HltGGiiCLICzmrBV0wVtA",
    bodyHash: "767ec67c417e11b97c5db7dad9ea3b6b27cb0d39",
    metadata: { filename: "skybin.html" },
@ -231,7 +231,7 @@ function skyBinRedirectCheck(done) {
 // skyGalleryCheck returns the result of trying to download the skylink for the SkyGallery Application.
 function skyGalleryCheck(done) {
  const linkInfo = {
-    description: "SkyGallery",
+    name: "SkyGallery",
    skylink: "AADW6GsQcetwDBaDYnGCSTbYjSKY743NtY1A5VRx5sj3Dg/",
    bodyHash: "077e54054748d278114f1870f8045a162eb73641",
    metadata: {
@ -365,7 +365,7 @@ function skyGalleryCheck(done) {
 // for the SkyGallery Application with a trailing /index.html
 function skyGalleryIndexFileCheck(done) {
  const linkInfo = {
-    description: "SkyGallery Index File",
+    name: "SkyGallery Index File",
    skylink: "AADW6GsQcetwDBaDYnGCSTbYjSKY743NtY1A5VRx5sj3Dg/index.html",
    bodyHash: "077e54054748d278114f1870f8045a162eb73641",
    metadata: {
@ -382,7 +382,7 @@ function skyGalleryIndexFileCheck(done) {
 // for the SkyGallery Application with no trailing slash.
 function skyGalleryRedirectCheck(done) {
  const linkInfo = {
-    description: "SkyGallery Redirect",
+    name: "SkyGallery Redirect",
    skylink: "AADW6GsQcetwDBaDYnGCSTbYjSKY743NtY1A5VRx5sj3Dg",
    bodyHash: "077e54054748d278114f1870f8045a162eb73641",
    metadata: {
@ -516,7 +516,7 @@ function skyGalleryRedirectCheck(done) {
 // for the uncensored library skylink
 function uncensoredLibraryCheck(done) {
  const linkInfo = {
-    description: "Uncensored Library",
+    name: "Uncensored Library",
    skylink: "AAC5glnZyNJ4Ieb4MhnYJGtID6qdMqEjl0or5EvEMt7bWQ",
    bodyHash: "60da6cb958699c5acd7f2a2911656ff32fca89a7",
    metadata: {
@ -538,7 +538,7 @@ function uncensoredLibraryCheck(done) {
 // for the Uniswap Application with a trailing /index.html
 function uniswapIndexFileCheck(done) {
  const linkInfo = {
-    description: "Uniswap Skylink Index File",
+    name: "Uniswap Skylink Index File",
    skylink: "IAC6CkhNYuWZqMVr1gob1B6tPg4MrBGRzTaDvAIAeu9A9w/index.html",
    bodyHash: "3965f9a7def085b3a764ddc76a528eda38d72359",
    metadata: {
@ -551,53 +551,48 @@ function uniswapIndexFileCheck(done) {
  skylinkVerification(done, linkInfo);
 }

-// skylinkVerification verifies a skylink against known information provided in
-// the linkInfo.
-function skylinkVerification(done, linkInfo) {
+// verifies a skylink against provided information
+function skylinkVerification(done, { name, skylink, bodyHash, metadata }) {
  const time = process.hrtime();

  // Create the query for the skylink
-  const query = `http://${process.env.PORTAL_URL}/${linkInfo.skylink}?nocache=true`;
+  const query = `http://${process.env.PORTAL_URL}/${skylink}?nocache=true`;

  // Get the Skylink
  superagent
    .get(query)
    .responseType("blob")
-    .end((err, res) => {
-      // Record the statusCode
-      const statusCode = (res && res.statusCode) || (err && err.statusCode) || null;
-      let info = null;
+    .then(
+      (response) => {
+        const entry = { name, up: true, statusCode: response.statusCode, time: calculateElapsedTime(time) };
+        const info = {};

-      // Determine if the skylink is up. Start with checking if there was an
-      // error in the request.
-      let up = err === null;
-      if (up) {
-        // Check if the response body is valid by checking against the known
-        // hash
-        const validBody = hash(res.body) === linkInfo.bodyHash;
-        // Check if the metadata is valid
-        const metadata = res.header["skynet-file-metadata"] ? JSON.parse(res.header["skynet-file-metadata"]) : null;
-        const validMetadata = isEqual(metadata, linkInfo.metadata);
-        // Redetermine if the Skylink is up based on the results from the body
-        // and metadata hash checks
-        up = up && validBody && validMetadata;
-
-        info = {
-          body: { valid: validBody },
-          metadata: { valid: validMetadata, diff: detailedDiff(metadata, linkInfo.metadata) },
-        };
+        // Check if the response body is valid by checking against the known hash
+        const currentBodyHash = hash(response.body);
+        if (currentBodyHash !== bodyHash) {
+          entry.up = false;
+          info.bodyHash = { expected: bodyHash, current: currentBodyHash };
        }

-      // Return the entry information
-      done({
-        name: linkInfo.description,
-        up,
-        info,
-        statusCode,
-        time: checks.catchRequestTime(time),
-        critical: true,
-      });
-    });
+        // Check if the metadata is valid by deep comparing expected value with response
+        const currentMetadata =
+          response.header["skynet-file-metadata"] && JSON.parse(response.header["skynet-file-metadata"]);
+        if (!isEqual(currentMetadata, metadata)) {
+          entry.up = false;
+          info.metadata = detailedDiff(currentMetadata, metadata);
+        }
+
+        if (Object.keys(info).length) entry.info = info; // add info only if it exists
+
+        done(entry); // Return the entry information
+      },
+      (error) => {
+        const statusCode = error.statusCode || error.status;
+        const entry = { name, up: false, statusCode, time: calculateElapsedTime(time) };
+
+        done(entry); // Return the entry information
+      }
+    );
 }

 module.exports.verboseChecks = [
--- a/packages/health-check/src/db.js
+++ b/packages/health-check/src/db.js
@ -1,10 +1,13 @@
+const fs = require("fs");
 const low = require("lowdb");
 const FileSync = require("lowdb/adapters/FileSync");
 const Memory = require("lowdb/adapters/Memory");

+if (!fs.existsSync("state")) fs.mkdirSync("state");
+
 const adapter = process.env.NODE_ENV === "production" ? new FileSync("state/state.json") : new Memory();
 const db = low(adapter);

-db.defaults({ entries: [] }).write();
+db.defaults({ disabled: false, entries: [] }).write();

 module.exports = db;
--- a/packages/health-check/src/index.js
+++ b/packages/health-check/src/index.js
@ -17,7 +17,10 @@ const server = express();
 server.use(bodyparser.urlencoded({ extended: false }));
 server.use(bodyparser.json());

-server.get("/health-check", require("./endpointHealthCheck"));
+server.get("/health-check", require("./api/index"));
+server.get("/health-check/recent", require("./api/recent"));
+server.get("/health-check/all", require("./api/all"));
+server.get("/health-check/disabled", require("./api/disabled"));

 server.listen(port, host, (error) => {
  if (error) throw error;
--- a/packages/health-check/src/schedule.js
+++ b/packages/health-check/src/schedule.js
@ -1,13 +1,13 @@
 const schedule = require("node-schedule");
 const db = require("./db");
-const { basicChecks } = require("./basicChecks");
-const { verboseChecks } = require("./verboseChecks");
+const { criticalChecks } = require("./checks/critical");
+const { verboseChecks } = require("./checks/verbose");

-// execute the basic health-check script every 5 minutes
+// execute the critical health-check script every 5 minutes
 const basicJob = schedule.scheduleJob("*/5 * * * *", async () => {
  const entry = { date: new Date().toISOString(), checks: [] };

-  entry.checks = await Promise.all(basicChecks.map((check) => new Promise(check)));
+  entry.checks = await Promise.all(criticalChecks.map((check) => new Promise(check)));

  db.get("entries").push(entry).write();
 });
--- a/packages/health-check/src/utils.js
+++ b/packages/health-check/src/utils.js
@ -0,0 +1,8 @@
+// return the time between start and now in milliseconds
+function calculateElapsedTime(start) {
+  const diff = process.hrtime(start);
+
+  return Math.round((diff[0] * 1e9 + diff[1]) / 1e6); // msec
+}
+
+module.exports = { calculateElapsedTime };