Merge pull request #699 from SkynetLabs/improve-health-checks

improve health checks
2021-04-29 16:59:21 +02:00 · 2021-04-29 16:59:21 +02:00 · 7648e22cd7
parent b0641fc334 7eb6d15d4b
commit 7648e22cd7
15 changed files with 1477 additions and 4476 deletions
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -14,135 +14,6 @@ networks:
        - subnet: 10.10.10.0/24

 services:
-  sia:
-    build:
-      context: ./docker/sia
-      dockerfile: Dockerfile
-      args:
-        branch: v1.5.5
-    container_name: sia
-    restart: unless-stopped
-    logging: *default-logging
-    environment:
-      - SIA_MODULES=gctwr
-    env_file:
-      - .env
-    volumes:
-      - ./docker/data/sia:/sia-data
-    networks:
-      shared:
-        ipv4_address: 10.10.10.10
-    expose:
-      - 9980
-
-  caddy:
-    build:
-      context: ./docker/caddy
-      dockerfile: Dockerfile
-    container_name: caddy
-    restart: unless-stopped
-    logging: *default-logging
-    env_file:
-      - .env
-    volumes:
-      - ./docker/data/caddy/data:/data
-      - ./docker/data/caddy/config:/config
-      - ./docker/caddy/Caddyfile:/etc/caddy/Caddyfile
-    networks:
-      shared:
-        ipv4_address: 10.10.10.20
-    ports:
-      - "80:80"
-      - "443:443"
-    depends_on:
-      - nginx
-
-  nginx:
-    build:
-      context: ./docker/nginx
-      dockerfile: Dockerfile
-    container_name: nginx
-    restart: unless-stopped
-    logging: *default-logging
-    env_file:
-      - .env
-    volumes:
-      - ./docker/nginx/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf:ro
-      - ./docker/nginx/conf.d:/etc/nginx/conf.d:ro
-      - ./docker/data/nginx/cache:/data/nginx/cache
-      - ./docker/data/nginx/logs:/usr/local/openresty/nginx/logs
-      - ./docker/data/nginx/skynet:/data/nginx/skynet:ro
-      - ./docker/data/sia/apipassword:/data/sia/apipassword:ro
-    networks:
-      shared:
-        ipv4_address: 10.10.10.30
-    expose:
-      - 80
-    depends_on:
-      - sia
-      - health-check
-      - handshake-api
-      - website
-
-  website:
-    build:
-      context: ./packages/website
-      dockerfile: Dockerfile
-    container_name: website
-    restart: unless-stopped
-    logging: *default-logging
-    env_file:
-      - .env
-    networks:
-      shared:
-        ipv4_address: 10.10.10.35
-    expose:
-      - 9000
-
-  handshake:
-    build:
-      context: ./docker/handshake
-      dockerfile: Dockerfile
-    container_name: handshake
-    restart: unless-stopped
-    logging: *default-logging
-    environment:
-      - HSD_LOG_CONSOLE=false
-      - HSD_HTTP_HOST=0.0.0.0
-      - HSD_NETWORK=main
-      - HSD_PORT=12037
-    env_file:
-      - .env
-    volumes:
-      - ./docker/data/handshake/.hsd:/root/.hsd
-    networks:
-      shared:
-        ipv4_address: 10.10.10.40
-    expose:
-      - 12037
-
-  handshake-api:
-    build:
-      context: ./packages/handshake-api
-      dockerfile: Dockerfile
-    container_name: handshake-api
-    restart: unless-stopped
-    logging: *default-logging
-    environment:
-      - HOSTNAME=0.0.0.0
-      - HSD_HOST=handshake
-      - HSD_NETWORK=main
-      - HSD_PORT=12037
-    env_file:
-      - .env
-    networks:
-      shared:
-        ipv4_address: 10.10.10.50
-    expose:
-      - 3100
-    depends_on:
-      - handshake
-
  health-check:
    build:
      context: ./packages/health-check
@ -157,10 +28,7 @@ services:
        ipv4_address: 10.10.10.60
    environment:
      - HOSTNAME=0.0.0.0
-      - PORTAL_URL=http://nginx
+      - PORTAL_URL=https://siasky.net
      - STATE_DIR=/usr/app/state
    expose:
      - 3100
-    depends_on:
-      - handshake
-      - handshake-api
--- a/docker/nginx/conf.d/include/cors
+++ b/docker/nginx/conf.d/include/cors
@ -1,8 +1,8 @@
 if ($request_method = 'OPTIONS') {
    more_set_headers 'Access-Control-Allow-Origin: $http_origin';
    more_set_headers 'Access-Control-Allow-Credentials: true';
-    more_set_headers 'Access-Control-Allow-Methods: GET, POST, OPTIONS, PUT, DELETE';
-    more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
+    more_set_headers 'Access-Control-Allow-Methods: GET, POST, HEAD, OPTIONS, PUT, PATH, DELETE';
+    more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,X-HTTP-Method-Override,upload-offset,upload-length,tus-version,tus-resumable,tus-extension,tus-max-size';
    more_set_headers 'Access-Control-Max-Age: 1728000';
    more_set_headers 'Content-Type: text/plain; charset=utf-8';
    more_set_headers 'Content-Length: 0';
@ -11,6 +11,6 @@ if ($request_method = 'OPTIONS') {

 more_set_headers 'Access-Control-Allow-Origin: $http_origin';
 more_set_headers 'Access-Control-Allow-Credentials: true';
-more_set_headers 'Access-Control-Allow-Methods: GET, POST, OPTIONS, PUT, DELETE';
-more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
-more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range,Skynet-File-Metadata,Skynet-Skylink,Skynet-Portal-Api';
+more_set_headers 'Access-Control-Allow-Methods: GET, POST, HEAD, OPTIONS, PUT, PATH, DELETE';
+more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,X-HTTP-Method-Override,upload-offset,upload-length,tus-version,tus-resumable,tus-extension,tus-max-size';
+more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range,Skynet-File-Metadata,Skynet-Skylink,Skynet-Portal-Api,upload-offset,upload-length,tus-version,tus-resumable,tus-extension,tus-max-size';
--- a/packages/dashboard/src/services/authServerSideProps.js
+++ b/packages/dashboard/src/services/authServerSideProps.js
@ -1,6 +1,6 @@
 import ky from "ky/umd";

-const isProduction = process.env.NODE_ENV === "production";
+const isProduction = false; // process.env.NODE_ENV === "production";

 export default function authServerSideProps(getServerSideProps) {
  return function authenticate(context) {
--- a/packages/health-check/Dockerfile
+++ b/packages/health-check/Dockerfile
@ -7,9 +7,9 @@ RUN yarn --no-lockfile
 COPY src src
 COPY cli cli

-RUN echo '*/5 * * * * /usr/app/cli/run critical' >> /etc/crontabs/root
-RUN echo '0 * * * * /usr/app/cli/run verbose' >> /etc/crontabs/root
+RUN echo '* * * * * /usr/app/cli/run critical > /dev/stdout' >> /etc/crontabs/root
+RUN echo '* * * * * /usr/app/cli/run extended > /dev/stdout' >> /etc/crontabs/root

 EXPOSE 3100
 ENV NODE_ENV production
-CMD [ "sh", "-c", "crond ; node --max-http-header-size=64000 src/index.js" ]
+CMD [ "sh", "-c", "crond -l 2 -f ; node --max-http-header-size=64000 src/index.js" ]
--- a/packages/health-check/package.json
+++ b/packages/health-check/package.json
@ -6,12 +6,14 @@
  "dependencies": {
    "deep-object-diff": "^1.1.0",
    "express": "^4.17.1",
+    "form-data": "^3.0.1",
+    "got": "^11.8.2",
+    "graceful-fs": "^4.2.6",
+    "hasha": "^5.2.2",
    "http-status-codes": "^2.1.2",
    "lodash": "^4.17.21",
    "lowdb": "^1.0.0",
-    "object-hash": "^2.1.1",
-    "superagent": "^6.0.0",
-    "tmp": "^0.2.1",
+    "write-file-atomic": "^3.0.3",
    "yargs": "^16.2.0"
  },
  "devDependencies": {
--- a/packages/health-check/src/adapters/FileSyncAtomic.js
+++ b/packages/health-check/src/adapters/FileSyncAtomic.js
@ -0,0 +1,28 @@
+const fs = require("graceful-fs");
+const Base = require("lowdb/adapters/Base");
+const { sync: writeFileAtomicSync } = require("write-file-atomic");
+
+class FileSyncAtomic extends Base {
+  read() {
+    if (fs.existsSync(this.source)) {
+      try {
+        const data = fs.readFileSync(this.source, "utf-8").trim();
+        return data ? this.deserialize(data) : this.defaultValue;
+      } catch (e) {
+        if (e instanceof SyntaxError) {
+          e.message = `Malformed JSON in file: ${this.source}\n${e.message}`;
+        }
+        throw e;
+      }
+    } else {
+      writeFileAtomicSync(this.source, this.serialize(this.defaultValue));
+      return this.defaultValue;
+    }
+  }
+
+  write(data) {
+    return writeFileAtomicSync(this.source, this.serialize(data));
+  }
+}
+
+module.exports = FileSyncAtomic;
--- a/packages/health-check/src/api/extended.js
+++ b/packages/health-check/src/api/extended.js
@ -1,11 +1,11 @@
 const db = require("../db");
 const { getYesterdayISOString } = require("../utils");

-// returns all verbose health check entries
+// returns all extended health check entries
 module.exports = (req, res) => {
  const yesterday = getYesterdayISOString();
  const entries = db
-    .get("verbose")
+    .get("extended")
    .orderBy("date", "desc")
    .filter(({ date }) => date > yesterday)
    .value();
--- a/packages/health-check/src/checks/critical.js
+++ b/packages/health-check/src/checks/critical.js
@ -1,51 +1,28 @@
-const fs = require("fs");
-const superagent = require("superagent");
-const tmp = require("tmp");
+const got = require("got");
+const FormData = require("form-data");
 const { StatusCodes } = require("http-status-codes");
 const { calculateElapsedTime, getResponseContent } = require("../utils");

 // uploadCheck returns the result of uploading a sample file
 async function uploadCheck(done) {
  const time = process.hrtime();
-  const file = tmp.fileSync();
+  const form = new FormData();
+  const data = Buffer.from(new Date()); // current date to ensure data uniqueness

-  fs.writeSync(file.fd, Buffer.from(new Date())); // write current date to temp file
-
-  superagent
-    .post(`${process.env.PORTAL_URL}/skynet/skyfile`)
-    .attach("file", file.name, file.name)
-    .end((error, response) => {
-      file.removeCallback();
-
-      const statusCode = (response && response.statusCode) || (error && error.statusCode) || null;
-
-      done({
-        name: "upload_file",
-        up: statusCode === StatusCodes.OK,
-        statusCode,
-        errorResponseContent: getResponseContent(error?.response),
-        time: calculateElapsedTime(time),
-      });
-    });
-}
-
-// downloadCheck returns the result of downloading the hard coded link
-async function downloadCheck(done) {
-  const time = process.hrtime();
-  const skylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q";
+  form.append("file", data, { filename: "time.txt", contentType: "text/plain" });
  let statusCode, errorResponseContent;

  try {
-    const response = await superagent.get(`${process.env.PORTAL_URL}/${skylink}?nocache=true`);
+    const response = await got.post(`${process.env.PORTAL_URL}/skynet/skyfile`, { body: form });

    statusCode = response.statusCode;
  } catch (error) {
-    statusCode = error.statusCode || error.status;
-    errorResponseContent = getResponseContent(error.response);
+    statusCode = error?.response?.statusCode || error.statusCode || error.status;
+    errorResponseContent = getResponseContent(error?.response);
  }

  done({
-    name: "download_file",
+    name: "upload_file",
    up: statusCode === StatusCodes.OK,
    statusCode,
    errorResponseContent,
@ -53,4 +30,30 @@ async function downloadCheck(done) {
  });
 }

+// downloadCheck returns the result of downloading the hard coded link
+async function downloadCheck(done) {
+  const time = process.hrtime();
+  const skylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q";
+  let statusCode, errorMessage, errorResponseContent;
+
+  try {
+    const response = await got(`${process.env.PORTAL_URL}/${skylink}?nocache=true`);
+
+    statusCode = response.statusCode;
+  } catch (error) {
+    statusCode = error?.response?.statusCode || error.statusCode || error.status;
+    errorMessage = error.message;
+    errorResponseContent = getResponseContent(error.response);
+  }
+
+  done({
+    name: "download_file",
+    up: statusCode === StatusCodes.OK,
+    statusCode,
+    errorMessage,
+    errorResponseContent,
+    time: calculateElapsedTime(time),
+  });
+}
+
 module.exports = [uploadCheck, downloadCheck];
--- a/packages/health-check/src/checks/extended.js
+++ b/packages/health-check/src/checks/extended.js
@ -1,5 +1,5 @@
-const superagent = require("superagent");
-const hash = require("object-hash");
+const got = require("got");
+const hasha = require("hasha");
 const { detailedDiff } = require("deep-object-diff");
 const { isEqual } = require("lodash");
 const { calculateElapsedTime, ensureValidJSON, getResponseContent } = require("../utils");
@ -10,7 +10,7 @@ function audioExampleCheck(done) {
  const linkInfo = {
    name: "Audio Example",
    skylink: "_A2zt5SKoqwnnZU4cBF8uBycSKULXMyeg1c5ZISBr2Q3dA",
-    bodyHash: "be335f5ad9bc357248f3d35c7e49df491afb6b12",
+    bodyHash: "1bea1f570043f20149ae4cb4d30089d90897b15b",
    headers: {
      "skynet-skylink": "_A2zt5SKoqwnnZU4cBF8uBycSKULXMyeg1c5ZISBr2Q3dA",
      "skynet-file-metadata": { filename: "feel-good.mp3" },
@ -28,7 +28,7 @@ function covid19PaperCheck(done) {
  const linkInfo = {
    name: "Covid-19 Paper",
    skylink: "PAMZVmfutxWoG6Wnl5BRKuWLkDNZR42k_okRRvksJekA3A",
-    bodyHash: "81b9fb74829a96ceafa429840d1ef0ce44376ddd",
+    bodyHash: "7ce20bfc4221503fd0bf909ad20c422eca125c7d",
    headers: {
      "skynet-skylink": "PAMZVmfutxWoG6Wnl5BRKuWLkDNZR42k_okRRvksJekA3A",
      "skynet-file-metadata": {
@ -55,7 +55,7 @@ function covid19CoroNopePaperCheck(done) {
  const linkInfo = {
    name: "Covid-19 CoroNope Paper",
    skylink: "bACLKGmcmX4NCp47WwOOJf0lU666VLeT5HRWpWVtqZPjEA",
-    bodyHash: "901f6fd65ef595f70b6bfebbb2d05942351ef2b3",
+    bodyHash: "0db705da1b1232f8344ed74fd38245d35a49a965",
    headers: {
      "skynet-skylink": "bACLKGmcmX4NCp47WwOOJf0lU666VLeT5HRWpWVtqZPjEA",
      "skynet-file-metadata": { filename: "coronope.pdf" },
@ -73,7 +73,7 @@ function dappExampleCheck(done) {
  const linkInfo = {
    name: "Dapp Example (UniSwap)",
    skylink: "EADWpKD0myqH2tZa6xtKebg6kNnwYnI94fl4R8UKgNrmOA",
-    bodyHash: "d6ad2506590bb45b5acc6a8a964a3da4d657354f",
+    bodyHash: "7b74cbb5927e964db493b82cc1f8a532f1ff72f5",
    headers: {
      "skynet-skylink": "EADWpKD0myqH2tZa6xtKebg6kNnwYnI94fl4R8UKgNrmOA",
      "skynet-file-metadata": {
@ -557,7 +557,7 @@ function dappExampleCheck(done) {
  skylinkVerification(done, linkInfo);
 }

-const developMomentumBodyHash = "53b44a9d3cfa9b3d66ce5c29976f4383725d3652";
+const developMomentumBodyHash = "08e96877dd6c99c3e1d98105f2fd9df377b53d65";
 const developMomentumMetadata = require("../fixtures/developMomentumMetadata.json");

 // developMomentumCheck returns the result of trying to download the skylink
@ -624,7 +624,7 @@ function htmlExampleCheck(done) {
  const linkInfo = {
    name: "HTML Example",
    skylink: "PAL0w4SdA5rFCDGEutgpeQ50Om-YkBabtXVOJAkmedslKw",
-    bodyHash: "c932fd56f98b6db589e56be8018817f13bb29f72",
+    bodyHash: "ecffcfbb74e017698cad30a91a74b9ba0b046413",
    headers: {
      "skynet-skylink": "PAL0w4SdA5rFCDGEutgpeQ50Om-YkBabtXVOJAkmedslKw",
      "skynet-file-metadata": { filename: "introduction â\x80\x93 Sia API Documentation.html" },
@ -642,7 +642,7 @@ function imageExampleCheck(done) {
  const linkInfo = {
    name: "Image Example",
    skylink: "IADUs8d9CQjUO34LmdaaNPK_STuZo24rpKVfYW3wPPM2uQ",
-    bodyHash: "313207978d0a88bf2b961f098804e9ab0f82837f",
+    bodyHash: "e318667a9d53a45a9d010ac4e0d120ad064279ac",
    headers: {
      "skynet-skylink": "IADUs8d9CQjUO34LmdaaNPK_STuZo24rpKVfYW3wPPM2uQ",
      "skynet-file-metadata": { filename: "sia-lm.png" },
@ -660,7 +660,7 @@ function jsonExampleCheck(done) {
  const linkInfo = {
    name: "JSON Example",
    skylink: "AAC0uO43g64ULpyrW0zO3bjEknSFbAhm8c-RFP21EQlmSQ",
-    bodyHash: "198771c3d07d5c7302aadcc0697a7298e5e8ccc3",
+    bodyHash: "b514603ce8acd937197712700e21259f18a857d6",
    headers: {
      "skynet-skylink": "AAC0uO43g64ULpyrW0zO3bjEknSFbAhm8c-RFP21EQlmSQ",
      "skynet-file-metadata": { filename: "consensus.json" },
@ -678,7 +678,7 @@ function pdfExampleCheck(done) {
  const linkInfo = {
    name: "PDF Example",
    skylink: "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg",
-    bodyHash: "9bd8162e1575569a9041972f7f62d65887063dc3",
+    bodyHash: "7e079f7afc9e5bc0c1be04543e22ac552a14a8da",
    headers: {
      "skynet-skylink": "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg",
      "skynet-file-metadata": { filename: "sia.pdf" },
@ -739,7 +739,7 @@ function skyBayCheck(done) {
  const linkInfo = {
    name: "SkyBay",
    skylink: "EABkMjXzxJRpPz0eO0Or5fy2eo-rz3prdigGwRlyNd9mwA/",
-    bodyHash: "25d63937c9734fb08d2749c6517d1b3de8ecb856",
+    bodyHash: "dfc0b1d3d1113254d7545d19f6118855ed9c778b",
    headers: {
      "skynet-skylink": "EABkMjXzxJRpPz0eO0Or5fy2eo-rz3prdigGwRlyNd9mwA",
      "skynet-file-metadata": {
@ -760,7 +760,7 @@ function skyBayRedirectCheck(done) {
  const linkInfo = {
    name: "SkyBay Redirect",
    skylink: "EABkMjXzxJRpPz0eO0Or5fy2eo-rz3prdigGwRlyNd9mwA",
-    bodyHash: "25d63937c9734fb08d2749c6517d1b3de8ecb856",
+    bodyHash: "dfc0b1d3d1113254d7545d19f6118855ed9c778b",
    headers: {
      "skynet-skylink": "EABkMjXzxJRpPz0eO0Or5fy2eo-rz3prdigGwRlyNd9mwA",
      "skynet-file-metadata": {
@ -780,7 +780,7 @@ function skyBinCheck(done) {
  const linkInfo = {
    name: "SkyBin",
    skylink: "CAAVU14pB9GRIqCrejD7rlS27HltGGiiCLICzmrBV0wVtA/",
-    bodyHash: "767ec67c417e11b97c5db7dad9ea3b6b27cb0d39",
+    bodyHash: "858ff733c4cb06a80060b8a62cf303fd5a051651",
    headers: {
      "skynet-skylink": "CAAVU14pB9GRIqCrejD7rlS27HltGGiiCLICzmrBV0wVtA",
      "skynet-file-metadata": { filename: "skybin.html" },
@ -798,7 +798,7 @@ function skyBinRedirectCheck(done) {
  const linkInfo = {
    name: "SkyBin Redirect",
    skylink: "CAAVU14pB9GRIqCrejD7rlS27HltGGiiCLICzmrBV0wVtA",
-    bodyHash: "767ec67c417e11b97c5db7dad9ea3b6b27cb0d39",
+    bodyHash: "858ff733c4cb06a80060b8a62cf303fd5a051651",
    headers: {
      "skynet-skylink": "CAAVU14pB9GRIqCrejD7rlS27HltGGiiCLICzmrBV0wVtA",
      "skynet-file-metadata": { filename: "skybin.html" },
@ -810,7 +810,7 @@ function skyBinRedirectCheck(done) {
  skylinkVerification(done, linkInfo);
 }

-const skyGalleryBodyHash = "077e54054748d278114f1870f8045a162eb73641";
+const skyGalleryBodyHash = "cb5905023a29bdd60d58817f26503345c9a1bd09";
 const skyGalleryMetadata = require("../fixtures/skygalleryMetadata.json");

 // skyGalleryCheck returns the result of trying to download the skylink for the SkyGallery Application.
@ -874,9 +874,9 @@ function skyGalleryRedirectCheck(done) {
 // for the uncensored library skylink
 function uncensoredLibraryCheck(done) {
  const linkInfo = {
-    name: "The Uncensored Library V2",
+    name: "Unzip The Uncensored Library Map",
    skylink: "AAC5glnZyNJ4Ieb4MhnYJGtID6qdMqEjl0or5EvEMt7bWQ",
-    bodyHash: "60da6cb958699c5acd7f2a2911656ff32fca89a7",
+    bodyHash: "cd0377661eefd656c8b46c497aa03112393ba893",
    headers: {
      "skynet-skylink": "AAC5glnZyNJ4Ieb4MhnYJGtID6qdMqEjl0or5EvEMt7bWQ",
      "skynet-file-metadata": {
@ -901,7 +901,7 @@ function uncensoredLibraryPressReleaseCheck(done) {
  const linkInfo = {
    name: "The Uncensored Library - Press Release",
    skylink: "AABHwuml_EhvyY8Gm7j1E2xGwodUNAJgX0A6-Cd22p9kNA",
-    bodyHash: "323217f643c3e3f1fe7532e72ac01bb0748c97be",
+    bodyHash: "da39a3ee5e6b4b0d3255bfef95601890afd80709",
    headers: {
      "skynet-skylink": "AABHwuml_EhvyY8Gm7j1E2xGwodUNAJgX0A6-Cd22p9kNA",
      "skynet-file-metadata": {
@ -927,7 +927,7 @@ function uncensoredLibraryV2Check(done) {
  const linkInfo = {
    name: "The Uncensored Library V2",
    skylink: "AAAs-JOsRGWgABYIo7AwTDqSX79-BxQKjDj0wiRGoRPFnw",
-    bodyHash: "1c6a885c060af8325eee82a11e9d64a13b228015",
+    bodyHash: "f2a802c2b7482825613a08853538203a53c96bd1",
    headers: {
      "skynet-skylink": "AAAs-JOsRGWgABYIo7AwTDqSX79-BxQKjDj0wiRGoRPFnw",
      "skynet-file-metadata": {
@ -952,7 +952,7 @@ function bitcoinWhitepaper(done) {
  skylinkVerification(done, {
    name: "Bitcoin Whitepaper",
    skylink: "3ACpC9Umme41zlWUgMQh1fw0sNwgWwyfDDhRQ9Sppz9hjQ",
-    bodyHash: "5d1fd2c37c1a3409cfc41861f4206472559670f3",
+    bodyHash: "8de2fdb04edce612738eb51e14ecc426381f8ed8",
    headers: {
      "skynet-skylink": "3ACpC9Umme41zlWUgMQh1fw0sNwgWwyfDDhRQ9Sppz9hjQ",
      "content-disposition": 'inline; filename="bitcoin.pdf"',
@ -1039,7 +1039,7 @@ function uniswapHNSResolverCheck(done) {
  const linkInfo = {
    name: "Uniswap HNS Resolver",
    skylink: "hnsres/uniswap-dex/",
-    bodyHash: "44a3f0f56861ae841a6cb19cb0b3edf98ad610f8",
+    bodyHash: "3634496800c254b93f9dcbca2aeb53e644f706c0",
  };

  skylinkVerification(done, linkInfo);
@ -1052,7 +1052,7 @@ function uniswapHNSResolverRedirectCheck(done) {
  const linkInfo = {
    name: "Uniswap HNS Resolver Redirect",
    skylink: "hnsres/uniswap-dex",
-    bodyHash: "44a3f0f56861ae841a6cb19cb0b3edf98ad610f8",
+    bodyHash: "3634496800c254b93f9dcbca2aeb53e644f706c0",
  };

  skylinkVerification(done, linkInfo);
@ -1063,7 +1063,7 @@ function fileEndpointCheck(done) {
  const linkInfo = {
    name: "File endpoint check",
    skylink: "file/XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg",
-    bodyHash: "9bd8162e1575569a9041972f7f62d65887063dc3",
+    bodyHash: "7e079f7afc9e5bc0c1be04543e22ac552a14a8da",
    headers: {
      "skynet-skylink": "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg",
      "skynet-file-metadata": { filename: "sia.pdf" },
@ -1086,7 +1086,7 @@ function hnsEndpointDirectoryRedirect(done) {
    },
  };

-  skylinkVerification(done, expected, { redirects: 0 });
+  skylinkVerification(done, expected, { followRedirect: false });
 }

 function parseHeaderString(header) {
@ -1098,43 +1098,35 @@ function parseHeaderString(header) {
 }

 // skylinkVerification verifies a skylink against provided information.
-function skylinkVerification(done, { name, skylink, bodyHash, headers, statusCode }, { redirects, method } = {}) {
+async function skylinkVerification(done, expected, { followRedirect = true, method = "get" } = {}) {
  const time = process.hrtime();
+  const details = { name: expected.name, skylink: expected.skylink };

-  // Create the query for the skylink
-  const query = `${process.env.PORTAL_URL}/${skylink}`;
-
-  // Get the Skylink
-  superagent[method || "get"](query)
-    .set("cookie", "nocache=true")
-    .redirects(redirects)
-    .ok((res) => (redirects === undefined ? res.ok : res.status < 400))
-    .responseType("blob")
-    .then(
-      (response) => {
-        const entry = { name, up: true, statusCode: response.statusCode, time: calculateElapsedTime(time) };
+  try {
+    const query = `${process.env.PORTAL_URL}/${expected.skylink}`;
+    const response = await got[method](query, { followRedirect, headers: { cookie: "nocache=true" } });
+    const entry = { ...details, up: true, statusCode: response.statusCode, time: calculateElapsedTime(time) };
    const info = {};

-        if (statusCode && statusCode !== response.statusCode) {
+    if (expected.statusCode && expected.statusCode !== response.statusCode) {
      entry.up = false;
-          info.statusCode = { expected: statusCode, current: response.statusCode };
+      info.statusCode = { expected: expected.statusCode, current: response.statusCode };
    }

    // Check if the response body is valid by checking against the known hash
-        if (bodyHash) {
-          const currentBodyHash = hash(response.body);
-          if (currentBodyHash !== bodyHash) {
+    if (expected.bodyHash) {
+      const currentBodyHash = hasha(response.rawBody, { algorithm: "sha1" });
+      if (currentBodyHash !== expected.bodyHash) {
        entry.up = false;
-            info.bodyHash = { expected: bodyHash, current: currentBodyHash };
+        info.bodyHash = { expected: expected.bodyHash, current: currentBodyHash };
      }
    }

-        if (headers) {
-          Object.entries(headers).forEach(([headerName, expectedHeader]) => {
-            const currentHeader = parseHeaderString(response.header[headerName]);
+    if (expected.headers) {
+      Object.entries(expected.headers).forEach(([headerName, expectedHeader]) => {
+        const currentHeader = parseHeaderString(response.headers[headerName]);
        if (!isEqual(currentHeader, expectedHeader)) {
          entry.up = false;
-
          info.headers = info.headers ?? {};
          if (typeof currentHeader === "object") {
            info.headers[headerName] = ensureValidJSON(detailedDiff(expectedHeader, currentHeader));
@ -1148,19 +1140,16 @@ function skylinkVerification(done, { name, skylink, bodyHash, headers, statusCod
    if (Object.keys(info).length) entry.info = info; // add info only if it exists

    done(entry); // Return the entry information
-      },
-      (error) => {
-        console.log(error);
-
+  } catch (error) {
    done({
-          name,
+      ...details,
      up: false,
-          statusCode: error.statusCode || error.status,
+      statusCode: error?.response?.statusCode || error.statusCode || error.status,
+      errorMessage: error.message,
      errorResponseContent: getResponseContent(error.response),
      time: calculateElapsedTime(time),
    });
  }
-    );
 }

 module.exports = [
--- a/packages/health-check/src/db.js
+++ b/packages/health-check/src/db.js
@ -1,12 +1,12 @@
-const fs = require("fs");
+const fs = require("graceful-fs");
 const low = require("lowdb");
-const FileSync = require("lowdb/adapters/FileSync");
+const FileSyncAtomic = require("./adapters/FileSyncAtomic");

 if (!fs.existsSync(process.env.STATE_DIR)) fs.mkdirSync(process.env.STATE_DIR);

-const adapter = new FileSync(`${process.env.STATE_DIR}/state.json`);
+const adapter = new FileSyncAtomic(`${process.env.STATE_DIR}/state.json`);
 const db = low(adapter);

-db.defaults({ disabled: false, critical: [], verbose: [] }).write();
+db.defaults({ disabled: false, critical: [], extended: [] }).write();

 module.exports = db;
--- a/packages/health-check/src/index.js
+++ b/packages/health-check/src/index.js
@ -21,7 +21,7 @@ server.use((req, res, next) => {

 server.get("/health-check", require("./api/index"));
 server.get("/health-check/critical", require("./api/critical"));
-server.get("/health-check/verbose", require("./api/verbose"));
+server.get("/health-check/extended", require("./api/extended"));
 server.get("/health-check/disabled", require("./api/disabled"));

 server.listen(port, host, (error) => {
--- a/packages/health-check/src/run.js
+++ b/packages/health-check/src/run.js
@ -8,7 +8,7 @@ require("yargs/yargs")(process.argv.slice(2)).command(
      .positional("type", {
        describe: "Type of checks to run",
        type: "string",
-        choices: ["critical", "verbose"],
+        choices: ["critical", "extended"],
      })
      .option("portal-url", {
        describe: "Skynet portal url",
--- a/packages/health-check/src/utils.js
+++ b/packages/health-check/src/utils.js
@ -23,9 +23,9 @@ function getYesterdayISOString() {
 */
 function getResponseContent(response) {
  try {
-    return JSON.parse(response?.text);
+    return JSON.parse(response?.body || response?.text);
  } catch {
-    return response?.text;
+    return response?.body || response?.text;
  }
 }

--- a/setup-scripts/health-checker.py
+++ b/setup-scripts/health-checker.py
@ -144,8 +144,8 @@ async def check_health():
        json_critical = requests.get(
            "http://localhost/health-check/critical", verify=False
        ).json()
-        json_verbose = requests.get(
-            "http://localhost/health-check/verbose", verify=False
+        json_extended = requests.get(
+            "http://localhost/health-check/extended", verify=False
        ).json()
    except:
        trace = traceback.format_exc()
@ -157,8 +157,8 @@ async def check_health():
    critical_checks_total = 0
    critical_checks_failed = 0

-    verbose_checks_total = 0
-    verbose_checks_failed = 0
+    extended_checks_total = 0
+    extended_checks_failed = 0

    failed_records = []
    failed_records_file = None
@ -178,18 +178,18 @@ async def check_health():
        if bad:
            failed_records.append(critical)

-    for verbose in json_verbose:
-        time = datetime.strptime(verbose["date"], "%Y-%m-%dT%H:%M:%S.%fZ")
+    for extended in json_extended:
+        time = datetime.strptime(extended["date"], "%Y-%m-%dT%H:%M:%S.%fZ")
        if time < time_limit:
            continue
        bad = False
-        for check in verbose["checks"]:
-            verbose_checks_total += 1
+        for check in extended["checks"]:
+            extended_checks_total += 1
            if check["up"] == False:
-                verbose_checks_failed += 1
+                extended_checks_failed += 1
                bad = True
        if bad:
-            failed_records.append(verbose)
+            failed_records.append(extended)

    ################################################################################
    # create a message
@ -213,14 +213,14 @@ async def check_health():
        message += "All {} critical checks passed. ".format(
            critical_checks_total)

-    if verbose_checks_failed:
-        message += "{}/{} verbose checks failed over the last {} hours! ".format(
-            verbose_checks_failed, verbose_checks_total, CHECK_HOURS
+    if extended_checks_failed:
+        message += "{}/{} extended checks failed over the last {} hours! ".format(
+            extended_checks_failed, extended_checks_total, CHECK_HOURS
        )
        force_notify = True
    else:
-        message += "All {} verbose checks passed. ".format(
-            verbose_checks_total)
+        message += "All {} extended checks passed. ".format(
+            extended_checks_total)

    if len(failed_records):
        failed_records_file = json.dumps(failed_records, indent=2)
--- a/yarn.lock
+++ b/yarn.lock