Merge pull request #781 from SkynetLabs/extend-critical-health-checks

Extend critical health checks
This commit is contained in:
Matthew Sevey 2021-05-24 11:35:56 -04:00 committed by GitHub
commit 13e867b5aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 69 additions and 37 deletions

View File

@ -2,14 +2,23 @@ FROM node:16.1.0-alpine
WORKDIR /usr/app WORKDIR /usr/app
# schedule critical checks to run every 5 minutes (any failures will disable server)
RUN echo '*/5 * * * * /usr/app/cli/run critical > /dev/stdout' >> /etc/crontabs/root
# schedule extended checks to run on every hour (optional checks, report only)
RUN echo '0 * * * * /usr/app/cli/run extended > /dev/stdout' >> /etc/crontabs/root
COPY package.json . COPY package.json .
RUN yarn --no-lockfile RUN yarn --no-lockfile
COPY src src COPY src src
COPY cli cli COPY cli cli
RUN echo '*/5 * * * * /usr/app/cli/run critical > /dev/stdout' >> /etc/crontabs/root
RUN echo '0 * * * * /usr/app/cli/run extended > /dev/stdout' >> /etc/crontabs/root
EXPOSE 3100 EXPOSE 3100
ENV NODE_ENV production ENV NODE_ENV production
# command consists of 3 parts:
# 1. starting crond
# 2. aliasing siasky.net and account.siasky.net with current server ip so health checks
# test portal end-to-end on prod domain (important for testing ssl certificates)
# 3. running api service
CMD [ "sh", "-c", "crond ; echo $(node src/whatismyip.js) siasky.net account.siasky.net >> /etc/hosts ; node --max-http-header-size=64000 src/index.js" ] CMD [ "sh", "-c", "crond ; echo $(node src/whatismyip.js) siasky.net account.siasky.net >> /etc/hosts ; node --max-http-header-size=64000 src/index.js" ]

View File

@ -1,7 +1,10 @@
const got = require("got"); const got = require("got");
const FormData = require("form-data"); const FormData = require("form-data");
const { StatusCodes } = require("http-status-codes");
const { calculateElapsedTime, getResponseContent } = require("../utils"); const { calculateElapsedTime, getResponseContent } = require("../utils");
const { SkynetClient } = require("skynet-js");
const skynetClient = new SkynetClient(process.env.SKYNET_PORTAL_API);
const exampleSkylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q";
// uploadCheck returns the result of uploading a sample file // uploadCheck returns the result of uploading a sample file
async function uploadCheck(done) { async function uploadCheck(done) {
@ -25,39 +28,43 @@ async function uploadCheck(done) {
data.ip = error?.response?.ip ?? null; data.ip = error?.response?.ip ?? null;
} }
done({ done({ name: "upload_file", time: calculateElapsedTime(time), ...data });
name: "upload_file", }
time: calculateElapsedTime(time),
...data, // websiteCheck checks whether the main website is working
}); async function websiteCheck(done) {
return genericAccessCheck("website", process.env.SKYNET_PORTAL_API, done);
} }
// downloadCheck returns the result of downloading the hard coded link // downloadCheck returns the result of downloading the hard coded link
async function downloadCheck(done) { async function downloadCheck(done) {
const time = process.hrtime(); const url = await skynetClient.getSkylinkUrl(exampleSkylink);
const skylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q";
const data = { up: false };
try { return genericAccessCheck("skylink", url, done);
const response = await got(`${process.env.SKYNET_PORTAL_API}/${skylink}?nocache=true`);
data.statusCode = response.statusCode;
data.up = true;
data.ip = response.ip;
} catch (error) {
data.statusCode = error?.response?.statusCode || error.statusCode || error.status;
data.errorMessage = error.message;
data.errorResponseContent = getResponseContent(error.response);
data.ip = error?.response?.ip ?? null;
} }
done({ // skylinkSubdomainCheck returns the result of downloading the hard coded link via subdomain
name: "download_file", async function skylinkSubdomainCheck(done) {
time: calculateElapsedTime(time), const url = await skynetClient.getSkylinkUrl(exampleSkylink, { subdomain: true });
...data,
}); return genericAccessCheck("skylink_via_subdomain", url, done);
} }
// handshakeSubdomainCheck returns the result of downloading the skylink via handshake domain
async function handshakeSubdomainCheck(done) {
const url = await skynetClient.getHnsUrl("note-to-self", { subdomain: true });
return genericAccessCheck("hns_via_subdomain", url, done);
}
// accountWebsiteCheck returns the result of accessing account dashboard website
async function accountWebsiteCheck(done) {
const url = `${process.env.SKYNET_DASHBOARD_URL}/auth/login`;
return genericAccessCheck("account_website", url, done);
}
// accountHealthCheck returns the result of accounts service health checks
async function accountHealthCheck(done) { async function accountHealthCheck(done) {
const time = process.hrtime(); const time = process.hrtime();
const data = { up: false }; const data = { up: false };
@ -76,17 +83,33 @@ async function accountHealthCheck(done) {
data.ip = error?.response?.ip ?? null; data.ip = error?.response?.ip ?? null;
} }
done({ done({ name: "accounts", time: calculateElapsedTime(time), ...data });
name: "account_health",
time: calculateElapsedTime(time),
...data,
});
} }
const checks = [uploadCheck, downloadCheck]; async function genericAccessCheck(name, url, done) {
const time = process.hrtime();
const data = { up: false, url };
try {
const response = await got(url, { headers: { cookie: "nocache=true" } });
data.statusCode = response.statusCode;
data.up = true;
data.ip = response.ip;
} catch (error) {
data.statusCode = error?.response?.statusCode || error.statusCode || error.status;
data.errorMessage = error.message;
data.errorResponseContent = getResponseContent(error.response);
data.ip = error?.response?.ip ?? null;
}
done({ name, time: calculateElapsedTime(time), ...data });
}
const checks = [uploadCheck, websiteCheck, downloadCheck, skylinkSubdomainCheck, handshakeSubdomainCheck];
if (process.env.ACCOUNTS_ENABLED) { if (process.env.ACCOUNTS_ENABLED) {
checks.push(accountHealthCheck); checks.push(accountHealthCheck, accountWebsiteCheck);
} }
module.exports = checks; module.exports = checks;