Merge pull request #1668 from SkynetLabs/make-ip-check-in-healthcheck-more-reliable

make ip check in health checks more reliable

.github/workflows/test-packages-health-check.yml

@@ -0,0 +1,23 @@
+name: Test - packages/health-check
+
+on:
+  pull_request:
+    paths:
+      - packages/health-check/**
+
+defaults:
+  run:
+    working-directory: packages/health-check
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16.x
+
+      - run: yarn
+      - run: yarn jest

packages/health-check/Dockerfile

@@ -7,10 +7,10 @@ WORKDIR /usr/app
 ENV PATH="/usr/app/bin:${PATH}"
 
 # schedule critical checks to run every 5 minutes (any failures will disable server)
-RUN echo '*/5 * * * * /usr/app/bin/cli run critical > /dev/stdout' >> /etc/crontabs/root
+RUN echo '*/5 * * * * source /etc/environment ; /usr/app/bin/cli run critical >> /proc/1/fd/1' >> /etc/crontabs/root
 
 # schedule extended checks to run on every hour (optional checks, report only)
-RUN echo '0 * * * * /usr/app/bin/cli run extended > /dev/stdout' >> /etc/crontabs/root
+RUN echo '0 * * * * source /etc/environment ; /usr/app/bin/cli run extended >> /proc/1/fd/1' >> /etc/crontabs/root
 
 COPY package.json yarn.lock ./
 
@@ -23,16 +23,18 @@ COPY bin bin
 EXPOSE 3100
 ENV NODE_ENV production
 
-# 1. start dnsmasq in the background with:
+# 1. get public server ip and save it in /etc/environment (passed to cron tasks as env variable)
+# 2. start dnsmasq in the background with:
 #    - alias PORTAL_DOMAIN with current server ip so it overrides potential load balancer request
 #    - default docker nameserver 127.0.0.11 for any other request
-# 2. replace docker nameserver with dnsmasq nameserver in /etc/resolv.conf
+# 3. replace docker nameserver with dnsmasq nameserver in /etc/resolv.conf
-# 3. start crond in the background to schedule periodic health checks
+# 4. start crond in the background to schedule periodic health checks
-# 4. start the health-check api service
+# 5. start the health-check api service
 CMD [ "sh", "-c", \
-      "serverip=$(node src/whatismyip.js) ; \
+      "export serverip=$(node src/whatismyip.js) && \
-       dnsmasq --no-resolv --log-facility=/var/log/dnsmasq.log --address=/$PORTAL_DOMAIN/$serverip --server=127.0.0.11 ; \
+       echo \"export serverip=${serverip}\" >> /etc/environment && \
-       echo \"$(sed 's/127.0.0.11/127.0.0.1/' /etc/resolv.conf)\" > /etc/resolv.conf ; \
+       dnsmasq --no-resolv --log-facility=/var/log/dnsmasq.log --address=/$PORTAL_DOMAIN/$serverip --server=127.0.0.11 && \
-       crond ; \
+       echo \"$(sed 's/127.0.0.11/127.0.0.1/' /etc/resolv.conf)\" > /etc/resolv.conf && \
+       crond && \
        node src/index.js" \
     ]

packages/health-check/package.json

@@ -18,6 +18,7 @@
     "yargs": "^17.3.1"
   },
   "devDependencies": {
+    "jest": "^27.5.0",
     "prettier": "^2.5.1"
   }
 }

packages/health-check/src/checks/middleware.js

@@ -1,13 +1,28 @@
 const got = require("got");
+const { ipCheckService, ipRegex } = require("../utils");
 
 const getCurrentAddress = async () => {
+  // use serverip env variable when available (set via Dockerfile)
+  if (process.env.serverip) {
+    if (ipRegex.test(process.env.serverip)) return process.env.serverip;
+
+    // log error to console for future reference but do not break
+    console.log(`Environment variable serverip contains invalid ip: "${process.env.serverip}"`);
+  }
+
   try {
-    const { body } = await got("http://whatismyip.akamai.com");
+    const { body } = await got(`http://${ipCheckService}`);
-    if (body) return body;
+    if (ipRegex.test(body)) {
-    throw new Error("whatismyip.akamai.com responded with empty body");
+      console.info(`Server public ip: ${body} (source: ${ipCheckService})`);
+
+      return body;
+    }
+
+    throw new Error(`${ipCheckService} responded with invalid ip: "${body}"`);
   } catch (error) {
-    console.log(error.message);
+    console.log(error.message); // log error to console for future reference
-    return "-- error fetching ip address from whatismyip.akamai.com --";
+
+    return null;
   }
 };
 
@@ -15,7 +30,8 @@ module.exports = async function middleware() {
   const ip = await getCurrentAddress();
 
   return (check) => {
-    if (check.ip && check.ip !== ip) {
+    // check only if current ip and check ip are provided
+    if (ip && check.ip && check.ip !== ip) {
       check.up = false;
       check.errors = check.errors ?? [];
       check.errors.push({

packages/health-check/src/index.js

@@ -42,4 +42,10 @@ server.listen(port, host, (error) => {
   if (error) throw error;
 
   console.info(`Server listening at http://${host}:${port} (NODE_ENV: ${process.env.NODE_ENV})`);
+
+  const { ipRegex } = require("./utils");
+
+  if (ipRegex.test(process.env.serverip)) {
+    console.info(`Server public ip: ${process.env.serverip}`);
+  }
 });

packages/health-check/src/utils.js

@@ -1,4 +1,7 @@
-const got = require("got");
+const ipCheckService = "whatismyip.akamai.com";
+const ipRegex = new RegExp(
+  `^(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)(?:\\.(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)){3}$`
+);
 
 /**
  * Get the time between start and now in milliseconds
@@ -60,6 +63,8 @@ function getAuthCookie() {
   if (!password) throw new Error("ACCOUNTS_TEST_USER_PASSWORD cannot be empty");
 
   async function authenticate() {
+    const got = require("got");
+
     try {
       // authenticate with given test user credentials
       const response = await got.post(`${process.env.SKYNET_DASHBOARD_URL}/api/login`, {
@@ -114,4 +119,6 @@ module.exports = {
   ensureValidJSON,
   getAuthCookie,
   isPortalModuleEnabled,
+  ipCheckService,
+  ipRegex,
 };

packages/health-check/src/utils.test.js

@@ -0,0 +1,19 @@
+describe("ipRegex", () => {
+  const { ipRegex } = require("./utils");
+
+  test("should test true for valid ip", () => {
+    expect(ipRegex.test("8.8.8.8")).toEqual(true);
+    expect(ipRegex.test("127.0.0.1")).toEqual(true);
+    expect(ipRegex.test("192.168.0.1")).toEqual(true);
+    expect(ipRegex.test("10.10.10.10")).toEqual(true);
+    expect(ipRegex.test("135.124.12.47")).toEqual(true);
+  });
+
+  test("should test false for invalid ip", () => {
+    expect(ipRegex.test("888.8.8.8")).toEqual(false);
+    expect(ipRegex.test("....")).toEqual(false);
+    expect(ipRegex.test(null)).toEqual(false);
+    expect(ipRegex.test("foo")).toEqual(false);
+    expect(ipRegex.test("")).toEqual(false);
+  });
+});

packages/health-check/src/whatismyip.js

@@ -1,13 +1,18 @@
 const http = require("http");
+const { ipCheckService, ipRegex } = require("./utils");
 
-const request = http.request({ host: "whatismyip.akamai.com" }, (response) => {
+const request = http.request({ host: ipCheckService }, (response) => {
   response.on("data", (data) => {
-    process.stdout.write(data);
+    if (ipRegex.test(data)) {
+      process.stdout.write(data);
+    } else {
+      throw new Error(`${ipCheckService} responded with invalid ip: "${data}"`);
+    }
   });
 });
 
 request.on("error", (error) => {
-  console.error(error);
+  throw error; // throw error to exit with code 1
 });
 
 request.end();