Merge pull request #903 from SkynetLabs/dnsmasq-debug

resolve ip mismatch on some of the health checks

packages/health-check/Dockerfile

@@ -18,15 +18,15 @@ COPY cli cli
 EXPOSE 3100
 ENV NODE_ENV production
 
-# 1. alias siasky.net with current server ip to ommit load balancer
-# 2. prepend dnsmasq nameserver so it tries to resolve first
-# 3. start dnsmasq in the background
-# 4. start crond in the background
-# 5. start the health-check api service
+# 1. start dnsmasq in the background with:
+#    - alias to siasky.net with current server ip so it overrides load balancer request
+#    - default docker nameserver 127.0.0.11 for any other request
+# 2. replace docker nameserver with dnsmasq nameserver in /etc/resolv.conf
+# 3. start crond in the background to schedule periodic health checks
+# 4. start the health-check api service
 CMD [ "sh", "-c", \
-      "echo address=/siasky.net/$(node src/whatismyip.js) > /etc/dnsmasq.d/siasky.net.conf ; \
-       echo -e \"nameserver 127.0.0.1\n$(cat /etc/resolv.conf)\" > /etc/resolv.conf ; \
-       dnsmasq ; \
+      "dnsmasq --no-resolv --log-facility=/var/log/dnsmasq.log --address=/siasky.net/$(node src/whatismyip.js) --server=127.0.0.11 ; \
+       echo \"$(sed 's/127.0.0.11/127.0.0.1/' /etc/resolv.conf)\" > /etc/resolv.conf ; \
        crond ; \
-       node --max-http-header-size=64000 src/index.js" \
+       node src/index.js" \
     ]

packages/health-check/src/checks/critical.js

@@ -67,7 +67,7 @@ async function accountWebsiteCheck(done) {
 // directServerApiAccessCheck returns the basic server api check on direct server address
 async function directServerApiAccessCheck(done) {
   if (!process.env.SKYNET_SERVER_API) {
-    return done({ up: false, info: { message: "SKYNET_SERVER_API env variable not configured" } });
+    return done({ up: false, errors: [{ message: "SKYNET_SERVER_API env variable not configured" }] });
   }
 
   const [portalAccessCheck, serverAccessCheck] = await Promise.all([
@@ -77,13 +77,14 @@ async function directServerApiAccessCheck(done) {
 
   if (portalAccessCheck.ip !== serverAccessCheck.ip) {
     serverAccessCheck.up = false;
-    serverAccessCheck.info = {
+    serverAccessCheck.errors = serverAccessCheck.errors ?? [];
+    serverAccessCheck.errors.push({
       message: "Access ip mismatch between portal and server access",
       response: {
         portal: { name: process.env.SKYNET_PORTAL_API, ip: portalAccessCheck.ip },
         server: { name: process.env.SKYNET_SERVER_API, ip: serverAccessCheck.ip },
       },
-    };
+    });
   }
 
   return done(serverAccessCheck);

packages/health-check/src/checks/middleware.js

@@ -0,0 +1,29 @@
+const got = require("got");
+
+const getCurrentAddress = async () => {
+  try {
+    const { body } = await got("http://whatismyip.akamai.com");
+    if (body) return body;
+    throw new Error("whatismyip.akamai.com responded with empty body");
+  } catch (error) {
+    console.log(error.message);
+    return "-- error fetching ip address from whatismyip.akamai.com --";
+  }
+};
+
+module.exports = async function middleware() {
+  const ip = await getCurrentAddress();
+
+  return (check) => {
+    if (check.ip && check.ip !== ip) {
+      check.up = false;
+      check.errors = check.errors ?? [];
+      check.errors.push({
+        message: "Response ip was different than current server ip - possibly there was an error with routing request",
+        data: { response: check.ip, server: ip },
+      });
+    }
+
+    return check;
+  };
+};

packages/health-check/src/run.js

@@ -1,4 +1,5 @@
 const { getYesterdayISOString } = require("./utils");
+const createMiddleware = require("./checks/middleware");
 
 require("yargs/yargs")(process.argv.slice(2)).command(
   "$0 <type>",
@@ -27,10 +28,11 @@ require("yargs/yargs")(process.argv.slice(2)).command(
 
     const db = require("../src/db");
     const checks = require(`../src/checks/${type}`);
+    const middleware = await createMiddleware();
 
     const entry = {
       date: new Date().toISOString(),
-      checks: await Promise.all(checks.map((check) => new Promise(check))),
+      checks: (await Promise.all(checks.map((check) => new Promise(check)))).map(middleware),
     };
 
     db.read() // read before writing to make sure no external changes are overwritten