LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into sevey/add-skapps-to-health-checl

This commit is contained in:
Matthew Sevey 2020-09-11 11:35:48 -04:00
parent 9874eb29d8 cd051c45f2
commit 781e71464d
34 changed files with 2204 additions and 1289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
docker-compose.yml
View File

@ -2,7 +2,10 @@ version: "3.7"
networks:
shared:
driver: bridge
ipam:
driver: default
config:
- subnet: 192.168.0.0/24
volumes:
webapp:
@ -23,7 +26,8 @@ services:
volumes:
- ./docker/data/sia:/sia-data
networks:
- shared
shared:
ipv4_address: 192.168.0.10
expose:
- 9980
@ -38,7 +42,8 @@ services:
volumes:
- ./docker/data/sia-upload:/sia-data
networks:
- shared
shared:
ipv4_address: 192.168.0.11
expose:
- 9980
@ -55,7 +60,8 @@ services:
- ./docker/data/caddy/config:/config
- ./docker/caddy/Caddyfile:/etc/caddy/Caddyfile
networks:
- shared
shared:
ipv4_address: 192.168.0.20
ports:
- "80:80"
- "443:443"
@ -76,7 +82,8 @@ services:
- ./docker/data/sia/apipassword:/data/sia/apipassword:ro
- webapp:/var/www/webportal:ro
networks:
- shared
shared:
ipv4_address: 192.168.0.30
expose:
- 80
depends_on:
@ -110,7 +117,8 @@ services:
volumes:
- ./docker/data/handshake/.hsd:/root/.hsd
networks:
- shared
shared:
ipv4_address: 192.168.0.40
expose:
- 12037
@ -128,7 +136,8 @@ services:
env_file:
- .env
networks:
- shared
shared:
ipv4_address: 192.168.0.50
expose:
- 3100
depends_on:
@ -143,7 +152,8 @@ services:
volumes:
- ./docker/data/health-check/state:/usr/app/state
networks:
- shared
shared:
ipv4_address: 192.168.0.60
environment:
- HOSTNAME=0.0.0.0
- PORTAL_URL=nginx

2
docker/handshake/Dockerfile
View File

@ -1,5 +1,5 @@
# we can't use alpine image since it fails on node-gyp
FROM node:14.8.0
FROM node:14.9.0
WORKDIR /opt/hsd

64
docker/nginx/conf.d/client.conf
View File

@ -21,6 +21,10 @@ server {
listen 80 default_server;
listen [::]:80 default_server;
# parse subdomain (a base32 encoded Skylink) into custom variable
server_name "~^([a-z0-9]{55})\..*$";
set $subdomain $1;
# ddos protection: closing slow connections
client_body_timeout 5s;
client_header_timeout 5s;
@ -31,6 +35,17 @@ server {
client_max_body_size 128k;
location / {
# The only safe thing to do inside an if in a location block is return
# or rewrite, since we need to proxy_pass we have to work our way around
# using a custom error code.
#
# See https://www.nginx.com/resources/wiki/start/topics/depth/ifisevil/
error_page 418 = @subdomain;
recursive_error_pages on;
if ($subdomain != "") {
return 418;
}
include /etc/nginx/conf.d/include/cors;
root /var/www/webportal;
@ -151,29 +166,11 @@ server {
# proxy this call to siad endpoint (make sure the ip is correct)
proxy_pass http://siad/skynet/skyfile/$dir1/$dir2/$dir3/$dir4$is_args$args;
}
location ~ "/skynet/skyfile/(.+)" {
include /etc/nginx/conf.d/include/cors;
include /etc/nginx/conf.d/include/sia-auth;
limit_conn uploads_by_ip 10; # ddos protection: max 10 uploads at a time
client_max_body_size 1000M; # make sure to limit the size of upload to a sane value
proxy_read_timeout 600;
proxy_request_buffering off; # stream uploaded files through the proxy as it comes in
proxy_set_header Expect $http_expect;
proxy_set_header User-Agent: Sia-Agent;
# we need to explicitly use set directive here because $1 will contain the siapath with
# decoded whitespaces and set will re-encode it for us before passing it to proxy_pass
set $siapath $1;
# proxy this call to siad endpoint (make sure the ip is correct)
proxy_pass http://siad/skynet/skyfile/$siapath$is_args$args;
}
location ~ "^/([a-zA-Z0-9-_]{46}(/.*)?)$" {
include /etc/nginx/conf.d/include/cors;
include /etc/nginx/conf.d/include/proxy-buffer;
include /etc/nginx/conf.d/include/proxy-cache-downloads;
limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
add_header Cache-Control "public, max-age=86400"; # allow consumer to cache response
@ -186,19 +183,26 @@ server {
proxy_set_header User-Agent: Sia-Agent;
# proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct)
proxy_pass http://siad/skynet/skylink/$skylink$is_args$args;
}
# cache frequent (> 10) downloads for 24 hours
proxy_cache skynet;
proxy_cache_key $uri;
proxy_cache_min_uses 10;
proxy_cache_valid 200 1440m;
proxy_cache_bypass $cookie_nocache $arg_nocache; # add cache bypass option
add_header X-Proxy-Cache $upstream_cache_status;
location @subdomain {
include /etc/nginx/conf.d/include/cors;
include /etc/nginx/conf.d/include/proxy-buffer;
include /etc/nginx/conf.d/include/proxy-cache-downloads;
limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
add_header Cache-Control "public, max-age=86400"; # allow consumer to cache response
proxy_read_timeout 600;
proxy_set_header User-Agent: Sia-Agent;
# proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct)
proxy_pass http://siad/skynet/skylink/$subdomain/$request_uri;
}
location ~ "^/file/([a-zA-Z0-9-_]{46}(/.*)?)$" {
include /etc/nginx/conf.d/include/cors;
include /etc/nginx/conf.d/include/proxy-buffer;
include /etc/nginx/conf.d/include/proxy-cache-downloads;
limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
add_header Cache-Control "public, max-age=86400"; # allow consumer to cache response
@ -212,13 +216,5 @@ server {
# proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct)
# this alias also adds attachment=true url param to force download the file
proxy_pass http://siad/skynet/skylink/$skylink?attachment=true&$args;
# cache frequent (> 10) downloads for 24 hours
proxy_cache skynet;
proxy_cache_key $uri;
proxy_cache_min_uses 10;
proxy_cache_valid 200 1440m;
proxy_cache_bypass $cookie_nocache $arg_nocache; # add cache bypass option
add_header X-Proxy-Cache $upstream_cache_status;
}
}

9
docker/nginx/conf.d/include/proxy-cache-downloads Normal file
View File

@ -0,0 +1,9 @@
proxy_cache skynet;
slice 1m;
proxy_http_version 1.1; # upgrade if necessary because 1.0 does not support byte-range requests
proxy_set_header Range $slice_range; # pass slice range to proxy
proxy_cache_key $uri$slice_range; # include $slice_range in the cache key
proxy_cache_min_uses 3; # cache responses after 3 requests of the same file
proxy_cache_valid 200 206 24h; # cache 200 and 206 responses for 24 hours
proxy_cache_bypass $cookie_nocache $arg_nocache; # add cache bypass option
add_header X-Proxy-Cache $upstream_cache_status; # add response header to indicate cache hits and misses

10
docker/nginx/nginx.conf
View File

@ -35,11 +35,13 @@ http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $upstream_response_time '
'$upstream_bytes_sent $upstream_bytes_received '
'"$upstream_http_content_type"';
#access_log logs/access.log main;
access_log logs/access.log main;
# See Move default writable paths to a dedicated directory (#119)
# https://github.com/openresty/docker-openresty/issues/119

2
packages/handshake-api/Dockerfile
View File

@ -1,4 +1,4 @@
FROM node:14.8.0-alpine
FROM node:14.9.0-alpine
WORKDIR /usr/app

3
packages/handshake-api/package.json
View File

@ -6,7 +6,8 @@
"dependencies": {
"express": "^4.17.1",
"express-http-proxy": "^1.6.2",
"hs-client": "^0.0.9"
"hs-client": "^0.0.9",
"node-cache": "^5.1.2"
},
"devDependencies": {
"prettier": "^2.0.5"

6
packages/handshake-api/src/index.js
View File

@ -1,6 +1,7 @@
const url = require("url");
const express = require("express");
const proxy = require("express-http-proxy");
const NodeCache = require("node-cache");
const { NodeClient } = require("hs-client");
const host = process.env.HOSTNAME || "0.0.0.0";
@ -18,16 +19,21 @@ const clientOptions = {
apiKey: hsdApiKey,
};
const client = new NodeClient(clientOptions);
const cache = new NodeCache({ stdTTL: 300 }); // cache for 5 minutes
// Match both `sia://HASH` and `HASH` links.
const startsWithSkylinkRegExp = /^(sia:\/\/)?[a-zA-Z0-9_-]{46}/;
const getDomainRecords = async (name) => {
if (cache.has(name)) return cache.get(name);
const response = await client.execute("getnameresource", [name]);
const records = response?.records ?? null;
console.log(`${name} => ${JSON.stringify(records)}`);
cache.set(name, records);
return records;
};

1
packages/health-check/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
state/

5
packages/health-check/Dockerfile
View File

@ -1,10 +1,11 @@
FROM node:14.8.0-alpine
FROM node:14.9.0-alpine
WORKDIR /usr/app
COPY package.json .
RUN yarn --no-lockfile
COPY src/* src/
COPY src src
COPY cli cli
EXPOSE 3100
ENV NODE_ENV production

7
packages/health-check/cli/disable Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env node
process.env.NODE_ENV = process.env.NODE_ENV || "production";
const db = require("../src/db");
db.set("disabled", true).write();

7
packages/health-check/cli/enable Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env node
process.env.NODE_ENV = process.env.NODE_ENV || "production";
const db = require("../src/db");
db.set("disabled", false).write();

14
packages/health-check/src/api/critical.js Normal file
View File

@ -0,0 +1,14 @@
const db = require("../db");
const { getYesterdayISOString } = require("../utils");
// returns all critical health check entries
module.exports = (req, res) => {
const yesterday = getYesterdayISOString();
const entries = db
.get("critical")
.orderBy("date", "desc")
.filter(({ date }) => date > yesterday)
.value();
res.send(entries);
};

8
packages/health-check/src/api/disabled.js Normal file
View File

@ -0,0 +1,8 @@
const db = require("../db");
// returns a disabled flag status
module.exports = (req, res) => {
const disabled = db.get("disabled").value();
res.send({ disabled });
};

75
packages/health-check/src/api/index.js Normal file
View File

@ -0,0 +1,75 @@
const { StatusCodes } = require("http-status-codes");
const { sum, sumBy } = require("lodash");
const db = require("../db");
/**
* Get status code that should be returned in the API response.
* - OK (200) in case everything is healthy
* - SERVICE_UNAVAILABLE (503) in case of any failures or if disabled
*/
function getStatusCode() {
// check whether the portal has been manually disabled
const disabled = getDisabled();
if (disabled) {
return StatusCodes.SERVICE_UNAVAILABLE;
}
// grab the most recent critical entry element from DB
const entry = getMostRecentCriticalEntry();
// in case there is no entry yet or at least one check failed in the most recent entry
if (!entry || entry.checks.some(({ up }) => !up)) {
return StatusCodes.SERVICE_UNAVAILABLE;
}
return StatusCodes.OK;
}
/**
* Get the sample of most recent critical entries and
* calculate the average response time of all of them
*/
function getAverageResponseTime() {
// get most recent 10 successfull checks for the calculation
const sample = db
.get("critical")
.orderBy("date", "desc")
.filter(({ checks }) => checks.every(({ up }) => up))
.take(10)
.value();
// calculate average time of response
return Math.round(sum(sample.map(({ checks }) => sumBy(checks, "time"))) / sample.size);
}
/**
* Get one, most current critical entry
*/
function getMostRecentCriticalEntry() {
return db.get("critical").orderBy("date", "desc").head().value();
}
/**
* Get the disabled flag state (manual portal disable)
*/
function getDisabled() {
return db.get("disabled").value();
}
module.exports = (req, res) => {
const statusCode = getStatusCode();
const timeout = statusCode === StatusCodes.OK ? getAverageResponseTime() : 0;
// We want to delay the response for the load balancer to be able to prioritize
// servers based on the successful response time of this endpoint. Load balancer
// will pull the server if the response is an error so there is no point in delaying
// failures, hence 0 timeout on those.
setTimeout(() => {
// include some health information in the response body
const entry = getMostRecentCriticalEntry();
const disabled = getDisabled();
res.status(statusCode).send({ disabled, entry });
}, timeout);
};

14
packages/health-check/src/api/verbose.js Normal file
View File

@ -0,0 +1,14 @@
const db = require("../db");
const { getYesterdayISOString } = require("../utils");
// returns all verbose health check entries
module.exports = (req, res) => {
const yesterday = getYesterdayISOString();
const entries = db
.get("verbose")
.orderBy("date", "desc")
.filter(({ date }) => date > yesterday)
.value();
res.send(entries);
};

51
packages/health-check/src/basicChecks.js
View File

@ -1,51 +0,0 @@
const superagent = require("superagent");
const { StatusCodes } = require("http-status-codes");
// uploadCheck returns the result of uploading a sample file
async function uploadCheck(done) {
const time = process.hrtime();
superagent
.post(`http://${process.env.PORTAL_URL}/skynet/skyfile`)
.attach("file", "package.json", "package.json")
.end((err, res) => {
const statusCode = (res && res.statusCode) || (err && err.statusCode) || null;
done({
name: "upload_file",
up: statusCode === StatusCodes.OK,
statusCode,
time: catchRequestTime(time),
critical: true,
});
});
}
// downloadCheck returns the result of downloading the hard coded link
function downloadCheck(done) {
const time = process.hrtime();
const skylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q";
superagent.get(`http://${process.env.PORTAL_URL}/${skylink}?nocache=true`).end((err, res) => {
const statusCode = (res && res.statusCode) || (err && err.statusCode) || null;
done({
name: "download_file",
up: statusCode === StatusCodes.OK,
statusCode,
time: catchRequestTime(time),
critical: true,
});
});
}
// catchRequestTime records the time it took to resolve the request in
// milliseconds
function catchRequestTime(start) {
const diff = process.hrtime(start);
return Math.round((diff[0] * 1e9 + diff[1]) / 1e6); // msec
}
module.exports.basicChecks = [uploadCheck, downloadCheck];
module.exports.catchRequestTime = catchRequestTime;

46
packages/health-check/src/checks/critical.js Normal file
View File

@ -0,0 +1,46 @@
const superagent = require("superagent");
const { StatusCodes } = require("http-status-codes");
const { calculateElapsedTime } = require("../utils");
// uploadCheck returns the result of uploading a sample file
async function uploadCheck(done) {
const time = process.hrtime();
superagent
.post(`http://${process.env.PORTAL_URL}/skynet/skyfile`)
.attach("file", "package.json", "package.json")
.end((error, response) => {
const statusCode = (response && response.statusCode) || (error && error.statusCode) || null;
done({
name: "upload_file",
up: statusCode === StatusCodes.OK,
statusCode,
time: calculateElapsedTime(time),
});
});
}
// downloadCheck returns the result of downloading the hard coded link
async function downloadCheck(done) {
const time = process.hrtime();
const skylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q";
let statusCode;
try {
const response = await superagent.get(`http://${process.env.PORTAL_URL}/${skylink}?nocache=true`);
statusCode = response.statusCode;
} catch (error) {
statusCode = error.statusCode || error.status;
}
done({
name: "download_file",
up: statusCode === StatusCodes.OK,
statusCode,
time: calculateElapsedTime(time),
});
}
module.exports.criticalChecks = [uploadCheck, downloadCheck];

965
packages/health-check/src/checks/verbose.js Normal file
View File

@ -0,0 +1,965 @@
const superagent = require("superagent");
const hash = require("object-hash");
const { detailedDiff } = require("deep-object-diff");
const { isEqual } = require("lodash");
const { calculateElapsedTime } = require("../utils");
// audioExampleCheck returns the result of trying to download the skylink
// for the Example audio file on siasky.net
function audioExampleCheck(done) {
const linkInfo = {
name: "Audio Example",
skylink: "_A2zt5SKoqwnnZU4cBF8uBycSKULXMyeg1c5ZISBr2Q3dA",
bodyHash: "be335f5ad9bc357248f3d35c7e49df491afb6b12",
metadata: { filename: "feel-good.mp3" },
};
skylinkVerification(done, linkInfo);
}
// covid19PaperCheck returns the result of trying to download the skylink
// for a known Covid19 paper
function covid19PaperCheck(done) {
const linkInfo = {
name: "Covid-19 Paper",
skylink: "PAMZVmfutxWoG6Wnl5BRKuWLkDNZR42k_okRRvksJekA3A",
bodyHash: "81b9fb74829a96ceafa429840d1ef0ce44376ddd",
metadata: {
filename: "An Effective Treatment for Coronavirus (COVID-19).pdf",
subfiles: {
"An Effective Treatment for Coronavirus (COVID-19).pdf": {
filename: "An Effective Treatment for Coronavirus (COVID-19).pdf",
contenttype: "application/pdf",
len: 474803,
},
},
},
};
skylinkVerification(done, linkInfo);
}
// covid19CoroNopePaperCheck returns the result of trying to download the skylink
// for another known Covid19 paper
function covid19CoroNopePaperCheck(done) {
const linkInfo = {
name: "Covid-19 CoroNope Paper",
skylink: "bACLKGmcmX4NCp47WwOOJf0lU666VLeT5HRWpWVtqZPjEA",
bodyHash: "901f6fd65ef595f70b6bfebbb2d05942351ef2b3",
metadata: { filename: "coronope.pdf" },
};
skylinkVerification(done, linkInfo);
}
// dappExampleCheck returns the result of trying to download the skylink
// for the Example Dapp on siasky.net
function dappExampleCheck(done) {
const linkInfo = {
name: "Dapp Example (UniSwap)",
skylink: "EADWpKD0myqH2tZa6xtKebg6kNnwYnI94fl4R8UKgNrmOA",
bodyHash: "d6ad2506590bb45b5acc6a8a964a3da4d657354f",
metadata: {
filename: "build",
length: 15578459,
subfiles: {
"451.html": {
filename: "451.html",
contenttype: "text/html",
offset: 40966,
len: 200,
},
"asset-manifest.json": {
filename: "asset-manifest.json",
contenttype: "application/json",
offset: 35832,
len: 5134,
},
"favicon.ico": {
filename: "favicon.ico",
contenttype: "image/vnd.microsoft.icon",
len: 31701,
},
"index.html": {
filename: "index.html",
contenttype: "text/html",
offset: 31701,
len: 4131,
},
"locales/de.json": {
filename: "locales/de.json",
contenttype: "application/json",
offset: 15542609,
len: 4376,
},
"locales/en.json": {
filename: "locales/en.json",
contenttype: "application/json",
offset: 15558827,
len: 4049,
},
"locales/es-AR.json": {
filename: "locales/es-AR.json",
contenttype: "application/json",
offset: 15551984,
len: 3624,
},
"locales/es-US.json": {
filename: "locales/es-US.json",
contenttype: "application/json",
offset: 15574829,
len: 3630,
},
"locales/it-IT.json": {
filename: "locales/it-IT.json",
contenttype: "application/json",
offset: 15538386,
len: 4223,
},
"locales/ro.json": {
filename: "locales/ro.json",
contenttype: "application/json",
offset: 15562876,
len: 3794,
},
"locales/ru.json": {
filename: "locales/ru.json",
contenttype: "application/json",
offset: 15546985,
len: 4999,
},
"locales/vi.json": {
filename: "locales/vi.json",
contenttype: "application/json",
offset: 15569928,
len: 4901,
},
"locales/zh-CN.json": {
filename: "locales/zh-CN.json",
contenttype: "application/json",
offset: 15555608,
len: 3219,
},
"locales/zh-TW.json": {
filename: "locales/zh-TW.json",
contenttype: "application/json",
offset: 15566670,
len: 3258,
},
"manifest.json": {
filename: "manifest.json",
contenttype: "application/json",
offset: 41166,
len: 297,
},
"precache-manifest.cd4677068c6058f8626d6818e2c12fd3.js": {
filename: "precache-manifest.cd4677068c6058f8626d6818e2c12fd3.js",
contenttype: "text/javascript",
offset: 41463,
len: 4721,
},
"service-worker.js": {
filename: "service-worker.js",
contenttype: "text/javascript",
offset: 46184,
len: 1185,
},
"static/css/0.07de6c03.chunk.css": {
filename: "static/css/0.07de6c03.chunk.css",
contenttype: "text/css",
offset: 15537249,
len: 285,
},
"static/css/0.07de6c03.chunk.css.map": {
filename: "static/css/0.07de6c03.chunk.css.map",
contenttype: "application/octet-stream",
offset: 15537818,
len: 568,
},
"static/css/5.d75e0ccb.chunk.css": {
filename: "static/css/5.d75e0ccb.chunk.css",
contenttype: "text/css",
offset: 15537534,
len: 284,
},
"static/css/5.d75e0ccb.chunk.css.map": {
filename: "static/css/5.d75e0ccb.chunk.css.map",
contenttype: "application/octet-stream",
offset: 15536511,
len: 738,
},
"static/js/0.58b0f69f.chunk.js": {
filename: "static/js/0.58b0f69f.chunk.js",
contenttype: "text/javascript",
offset: 7300150,
len: 30029,
},
"static/js/0.58b0f69f.chunk.js.map": {
filename: "static/js/0.58b0f69f.chunk.js.map",
contenttype: "application/octet-stream",
offset: 12111459,
len: 81144,
},
"static/js/1.19c370e0.chunk.js": {
filename: "static/js/1.19c370e0.chunk.js",
contenttype: "text/javascript",
offset: 15495781,
len: 40203,
},
"static/js/1.19c370e0.chunk.js.map": {
filename: "static/js/1.19c370e0.chunk.js.map",
contenttype: "application/octet-stream",
offset: 7330179,
len: 104594,
},
"static/js/10.8ea29dcd.chunk.js": {
filename: "static/js/10.8ea29dcd.chunk.js",
contenttype: "text/javascript",
offset: 15483299,
len: 12345,
},
"static/js/10.8ea29dcd.chunk.js.map": {
filename: "static/js/10.8ea29dcd.chunk.js.map",
contenttype: "application/octet-stream",
offset: 14524416,
len: 30393,
},
"static/js/11.764b8915.chunk.js": {
filename: "static/js/11.764b8915.chunk.js",
contenttype: "text/javascript",
offset: 12208196,
len: 7103,
},
"static/js/11.764b8915.chunk.js.map": {
filename: "static/js/11.764b8915.chunk.js.map",
contenttype: "application/octet-stream",
offset: 12192603,
len: 15593,
},
"static/js/12.88d4fbe5.chunk.js": {
filename: "static/js/12.88d4fbe5.chunk.js",
contenttype: "text/javascript",
offset: 12055261,
len: 16721,
},
"static/js/12.88d4fbe5.chunk.js.map": {
filename: "static/js/12.88d4fbe5.chunk.js.map",
contenttype: "application/octet-stream",
offset: 14460215,
len: 46695,
},
"static/js/13.ea207f69.chunk.js": {
filename: "static/js/13.ea207f69.chunk.js",
contenttype: "text/javascript",
offset: 7168280,
len: 347,
},
"static/js/13.ea207f69.chunk.js.map": {
filename: "static/js/13.ea207f69.chunk.js.map",
contenttype: "application/octet-stream",
offset: 6928538,
len: 563,
},
"static/js/14.d8bc0d4c.chunk.js": {
filename: "static/js/14.d8bc0d4c.chunk.js",
contenttype: "text/javascript",
offset: 12870711,
len: 336,
},
"static/js/14.d8bc0d4c.chunk.js.map": {
filename: "static/js/14.d8bc0d4c.chunk.js.map",
contenttype: "application/octet-stream",
offset: 15535984,
len: 527,
},
"static/js/15.e6215497.chunk.js": {
filename: "static/js/15.e6215497.chunk.js",
contenttype: "text/javascript",
offset: 15495644,
len: 137,
},
"static/js/15.e6215497.chunk.js.map": {
filename: "static/js/15.e6215497.chunk.js.map",
contenttype: "application/octet-stream",
offset: 6928431,
len: 107,
},
"static/js/2.f6da9598.chunk.js": {
filename: "static/js/2.f6da9598.chunk.js",
contenttype: "text/javascript",
offset: 14506910,
len: 17506,
},
"static/js/2.f6da9598.chunk.js.map": {
filename: "static/js/2.f6da9598.chunk.js.map",
contenttype: "application/octet-stream",
offset: 12071982,
len: 39477,
},
"static/js/5.5cc0868a.chunk.js": {
filename: "static/js/5.5cc0868a.chunk.js",
contenttype: "text/javascript",
offset: 10199338,
len: 1842002,
},
"static/js/5.5cc0868a.chunk.js.LICENSE": {
filename: "static/js/5.5cc0868a.chunk.js.LICENSE",
contenttype: "application/octet-stream",
offset: 14554809,
len: 3119,
},
"static/js/5.5cc0868a.chunk.js.map": {
filename: "static/js/5.5cc0868a.chunk.js.map",
contenttype: "application/octet-stream",
offset: 289328,
len: 6632626,
},
"static/js/6.b7681521.chunk.js": {
filename: "static/js/6.b7681521.chunk.js",
contenttype: "text/javascript",
offset: 14237363,
len: 222852,
},
"static/js/6.b7681521.chunk.js.map": {
filename: "static/js/6.b7681521.chunk.js.map",
contenttype: "application/octet-stream",
offset: 12215299,
len: 655412,
},
"static/js/7.0614dbc4.chunk.js": {
filename: "static/js/7.0614dbc4.chunk.js",
contenttype: "text/javascript",
offset: 6921954,
len: 6477,
},
"static/js/7.0614dbc4.chunk.js.map": {
filename: "static/js/7.0614dbc4.chunk.js.map",
contenttype: "application/octet-stream",
offset: 12041340,
len: 13921,
},
"static/js/8.7975098c.chunk.js": {
filename: "static/js/8.7975098c.chunk.js",
contenttype: "text/javascript",
offset: 13796515,
len: 420712,
},
"static/js/8.7975098c.chunk.js.LICENSE": {
filename: "static/js/8.7975098c.chunk.js.LICENSE",
contenttype: "application/octet-stream",
offset: 13796191,
len: 324,
},
"static/js/8.7975098c.chunk.js.map": {
filename: "static/js/8.7975098c.chunk.js.map",
contenttype: "application/octet-stream",
offset: 12871047,
len: 925144,
},
"static/js/9.cc860b76.chunk.js": {
filename: "static/js/9.cc860b76.chunk.js",
contenttype: "text/javascript",
offset: 14557928,
len: 920812,
},
"static/js/9.cc860b76.chunk.js.LICENSE": {
filename: "static/js/9.cc860b76.chunk.js.LICENSE",
contenttype: "application/octet-stream",
offset: 15478740,
len: 4559,
},
"static/js/9.cc860b76.chunk.js.map": {
filename: "static/js/9.cc860b76.chunk.js.map",
contenttype: "application/octet-stream",
offset: 7434773,
len: 2764565,
},
"static/js/main.a7822f79.chunk.js": {
filename: "static/js/main.a7822f79.chunk.js",
contenttype: "text/javascript",
offset: 7168627,
len: 131523,
},
"static/js/main.a7822f79.chunk.js.map": {
filename: "static/js/main.a7822f79.chunk.js.map",
contenttype: "application/octet-stream",
offset: 6929101,
len: 239179,
},
"static/js/runtime-main.68d129c6.js": {
filename: "static/js/runtime-main.68d129c6.js",
contenttype: "text/javascript",
offset: 14217227,
len: 3546,
},
"static/js/runtime-main.68d129c6.js.map": {
filename: "static/js/runtime-main.68d129c6.js.map",
contenttype: "application/octet-stream",
offset: 14220773,
len: 16590,
},
"static/media/arrow-down-blue.cd061363.svg": {
filename: "static/media/arrow-down-blue.cd061363.svg",
contenttype: "image/svg+xml",
offset: 219284,
len: 326,
},
"static/media/arrow-down-grey.c0dedd2f.svg": {
filename: "static/media/arrow-down-grey.c0dedd2f.svg",
contenttype: "image/svg+xml",
offset: 196726,
len: 326,
},
"static/media/arrow-right-white.337ad716.png": {
filename: "static/media/arrow-right-white.337ad716.png",
contenttype: "image/png",
offset: 197052,
len: 12999,
},
"static/media/arrow-right.d285b6cf.svg": {
filename: "static/media/arrow-right.d285b6cf.svg",
contenttype: "image/svg+xml",
offset: 289065,
len: 263,
},
"static/media/circle-grey.ed2a1dad.svg": {
filename: "static/media/circle-grey.ed2a1dad.svg",
contenttype: "image/svg+xml",
offset: 210213,
len: 321,
},
"static/media/circle.2d975615.svg": {
filename: "static/media/circle.2d975615.svg",
contenttype: "image/svg+xml",
offset: 210534,
len: 321,
},
"static/media/coinbaseWalletIcon.62578f59.svg": {
filename: "static/media/coinbaseWalletIcon.62578f59.svg",
contenttype: "image/svg+xml",
offset: 220450,
len: 53626,
},
"static/media/dropdown-blue.b20914ec.svg": {
filename: "static/media/dropdown-blue.b20914ec.svg",
contenttype: "image/svg+xml",
offset: 47369,
len: 164,
},
"static/media/dropdown.7d32d2fa.svg": {
filename: "static/media/dropdown.7d32d2fa.svg",
contenttype: "image/svg+xml",
offset: 287941,
len: 164,
},
"static/media/dropup-blue.b96d70e1.svg": {
filename: "static/media/dropup-blue.b96d70e1.svg",
contenttype: "image/svg+xml",
offset: 210051,
len: 162,
},
"static/media/ethereum-logo.802c6eac.svg": {
filename: "static/media/ethereum-logo.802c6eac.svg",
contenttype: "image/svg+xml",
offset: 219610,
len: 840,
},
"static/media/magnifying-glass.67440097.svg": {
filename: "static/media/magnifying-glass.67440097.svg",
contenttype: "image/svg+xml",
offset: 210855,
len: 8429,
},
"static/media/metamask.023762b6.png": {
filename: "static/media/metamask.023762b6.png",
contenttype: "image/png",
offset: 61600,
len: 114217,
},
"static/media/plus-blue.e8021e51.svg": {
filename: "static/media/plus-blue.e8021e51.svg",
contenttype: "image/svg+xml",
offset: 196237,
len: 190,
},
"static/media/plus-grey.d8e0be7d.svg": {
filename: "static/media/plus-grey.d8e0be7d.svg",
contenttype: "image/svg+xml",
offset: 288875,
len: 190,
},
"static/media/portisIcon.b234b2bf.png": {
filename: "static/media/portisIcon.b234b2bf.png",
contenttype: "image/png",
offset: 274076,
len: 13865,
},
"static/media/question-mark.1ae4d9f4.svg": {
filename: "static/media/question-mark.1ae4d9f4.svg",
contenttype: "image/svg+xml",
offset: 175817,
len: 818,
},
"static/media/question.cc0a2451.svg": {
filename: "static/media/question.cc0a2451.svg",
contenttype: "image/svg+xml",
offset: 288105,
len: 770,
},
"static/media/spinner.be00fc4a.svg": {
filename: "static/media/spinner.be00fc4a.svg",
contenttype: "image/svg+xml",
offset: 47533,
len: 694,
},
"static/media/trustWallet.edcc1ab5.png": {
filename: "static/media/trustWallet.edcc1ab5.png",
contenttype: "image/png",
offset: 176635,
len: 19602,
},
"static/media/walletConnectIcon.8215855c.svg": {
filename: "static/media/walletConnectIcon.8215855c.svg",
contenttype: "image/svg+xml",
offset: 48227,
len: 13373,
},
"static/media/x.5b8e2186.svg": {
filename: "static/media/x.5b8e2186.svg",
contenttype: "image/svg+xml",
offset: 196427,
len: 299,
},
},
},
};
skylinkVerification(done, linkInfo);
}
const developMomentumBodyHash = "53b44a9d3cfa9b3d66ce5c29976f4383725d3652";
const developMomentumMetadata = require("./fixtures/developMomentumMetadata.json");
// developMomentumCheck returns the result of trying to download the skylink
// for the Develop Momentum Application
function developMomentumCheck(done) {
const linkInfo = {
description: "Develop Momentum Index File",
skylink: "EAA1fG_ip4C1Vi1Ijvsr1oyr8jpH0Bo9HXya0T3kw-elGw/",
bodyHash: developMomentumBodyHash,
metadata: developMomentumMetadata,
};
skylinkVerification(done, linkInfo);
}
// developMomentumRedirectCheck returns the result of trying to download the skylink
// for the Develop Momentum Application without the tailing slash
function developMomentumRedirectCheck(done) {
const linkInfo = {
description: "Develop Momentum Index File",
skylink: "EAA1fG_ip4C1Vi1Ijvsr1oyr8jpH0Bo9HXya0T3kw-elGw",
bodyHash: developMomentumBodyHash,
metadata: developMomentumMetadata,
};
skylinkVerification(done, linkInfo);
}
// developMomentumIndexFileCheck returns the result of trying to download the skylink
// for the Develop Momentum Application with a trailing /index.html
function developMomentumIndexFileCheck(done) {
const linkInfo = {
name: "Develop Momentum Index File",
skylink: "EAA1fG_ip4C1Vi1Ijvsr1oyr8jpH0Bo9HXya0T3kw-elGw/index.html",
bodyHash: developMomentumBodyHash,
metadata: {
filename: "/index.html",
length: 4981,
subfiles: { "index.html": { filename: "index.html", contenttype: "text/html", len: 4981 } },
},
};
skylinkVerification(done, linkInfo);
}
// htmlExampleCheck returns the result of trying to download the skylink
// for the Example HTML file on siasky.net
function htmlExampleCheck(done) {
const linkInfo = {
name: "HTML Example",
skylink: "PAL0w4SdA5rFCDGEutgpeQ50Om-YkBabtXVOJAkmedslKw",
bodyHash: "c932fd56f98b6db589e56be8018817f13bb29f72",
metadata: { filename: "introduction – Sia API Documentation.html" },
};
skylinkVerification(done, linkInfo);
}
// imageExampleCheck returns the result of trying to download the skylink
// for the Example image on siasky.net
function imageExampleCheck(done) {
const linkInfo = {
name: "Image Example",
skylink: "IADUs8d9CQjUO34LmdaaNPK_STuZo24rpKVfYW3wPPM2uQ",
bodyHash: "313207978d0a88bf2b961f098804e9ab0f82837f",
metadata: { filename: "sia-lm.png" },
};
skylinkVerification(done, linkInfo);
}
// jsonExampleCheck returns the result of trying to download the skylink
// for the Example JSON file on siasky.net
function jsonExampleCheck(done) {
const linkInfo = {
name: "JSON Example",
skylink: "AAC0uO43g64ULpyrW0zO3bjEknSFbAhm8c-RFP21EQlmSQ",
bodyHash: "198771c3d07d5c7302aadcc0697a7298e5e8ccc3",
metadata: { filename: "consensus.json" },
};
skylinkVerification(done, linkInfo);
}
// pdfExampleCheck returns the result of trying to download the skylink
// for the Example PDF file on siasky.net
function pdfExampleCheck(done) {
const linkInfo = {
name: "PDF Example",
skylink: "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg",
bodyHash: "9bd8162e1575569a9041972f7f62d65887063dc3",
metadata: { filename: "sia.pdf" },
};
skylinkVerification(done, linkInfo);
}
// randomImageCheck returns the result of trying to download the skylink for
// a Random Image.
function randomImageCheck(done) {
const linkInfo = {
name: "Random Image",
skylink: "PAHx7JmsU9EFGbqm5q0LNKT2wKfoJ_mhPI8zWlNEXZ8uOQ/",
bodyHash: "4c73c5a0eddd5823be677d7f93bf80cc9338ee9f",
metadata: {
filename: "30355444.png",
subfiles: { "30355444.png": { filename: "30355444.png", contenttype: "image/png", len: 350473 } },
defaultpath: "/30355444.png",
},
};
skylinkVerification(done, linkInfo);
}
// randomImageRedirectCheck returns the result of trying to download the skylink for
// a Random Image with no trailing slash.
function randomImageRedirectCheck(done) {
const linkInfo = {
name: "Random Image Redirect",
skylink: "PAHx7JmsU9EFGbqm5q0LNKT2wKfoJ_mhPI8zWlNEXZ8uOQ",
bodyHash: "4c73c5a0eddd5823be677d7f93bf80cc9338ee9f",
metadata: {
filename: "30355444.png",
subfiles: { "30355444.png": { filename: "30355444.png", contenttype: "image/png", len: 350473 } },
defaultpath: "/30355444.png",
},
};
skylinkVerification(done, linkInfo);
}
// skyBayCheck returns the result of trying to download the skylink for the SkyBay Application.
function skyBayCheck(done) {
const linkInfo = {
name: "SkyBay",
skylink: "EABkMjXzxJRpPz0eO0Or5fy2eo-rz3prdigGwRlyNd9mwA/",
bodyHash: "25d63937c9734fb08d2749c6517d1b3de8ecb856",
metadata: {
filename: "skybay.html",
subfiles: { "skybay.html": { filename: "skybay.html", contenttype: "text/html", len: 11655 } },
},
};
skylinkVerification(done, linkInfo);
}
// skyBayRedirectCheck returns the result of trying to download the skylink
// for the SkyBay Application with no trailing slash.
function skyBayRedirectCheck(done) {
const linkInfo = {
name: "SkyBay Redirect",
skylink: "EABkMjXzxJRpPz0eO0Or5fy2eo-rz3prdigGwRlyNd9mwA",
bodyHash: "25d63937c9734fb08d2749c6517d1b3de8ecb856",
metadata: {
filename: "skybay.html",
subfiles: { "skybay.html": { filename: "skybay.html", contenttype: "text/html", len: 11655 } },
},
};
skylinkVerification(done, linkInfo);
}
// skyBinCheck returns the result of trying to download the skylink for the SkyBin Application.
function skyBinCheck(done) {
const linkInfo = {
name: "SkyBin",
skylink: "CAAVU14pB9GRIqCrejD7rlS27HltGGiiCLICzmrBV0wVtA/",
bodyHash: "767ec67c417e11b97c5db7dad9ea3b6b27cb0d39",
metadata: { filename: "skybin.html" },
};
skylinkVerification(done, linkInfo);
}
// skyBinRedirectCheck returns the result of trying to download the skylink
// for the SkyBin Application with no trailing slash.
function skyBinRedirectCheck(done) {
const linkInfo = {
name: "SkyBin Redirect",
skylink: "CAAVU14pB9GRIqCrejD7rlS27HltGGiiCLICzmrBV0wVtA",
bodyHash: "767ec67c417e11b97c5db7dad9ea3b6b27cb0d39",
metadata: { filename: "skybin.html" },
};
skylinkVerification(done, linkInfo);
}
const skyGalleryBodyHash = "077e54054748d278114f1870f8045a162eb73641";
const skyGalleryMetadata = require("./fixtures/skygalleryMetadata.json");
// skyGalleryCheck returns the result of trying to download the skylink for the SkyGallery Application.
function skyGalleryCheck(done) {
const linkInfo = {
name: "SkyGallery",
skylink: "AADW6GsQcetwDBaDYnGCSTbYjSKY743NtY1A5VRx5sj3Dg/",
bodyHash: skyGalleryBodyHash,
metadata: skyGalleryMetadata,
};
skylinkVerification(done, linkInfo);
}
// skyGalleryIndexFileCheck returns the result of trying to download the skylink
// for the SkyGallery Application with a trailing /index.html
function skyGalleryIndexFileCheck(done) {
const linkInfo = {
name: "SkyGallery Index File",
skylink: "AADW6GsQcetwDBaDYnGCSTbYjSKY743NtY1A5VRx5sj3Dg/index.html",
bodyHash: skyGalleryBodyHash,
metadata: {
filename: "/index.html",
length: 2534,
subfiles: { "index.html": { filename: "index.html", contenttype: "text/html", len: 2534 } },
},
};
skylinkVerification(done, linkInfo);
}
// skyGalleryRedirectCheck returns the result of trying to download the skylink
// for the SkyGallery Application with no trailing slash.
function skyGalleryRedirectCheck(done) {
const linkInfo = {
name: "SkyGallery Redirect",
skylink: "AADW6GsQcetwDBaDYnGCSTbYjSKY743NtY1A5VRx5sj3Dg",
bodyHash: skyGalleryBodyHash,
metadata: skyGalleryMetadata,
};
skylinkVerification(done, linkInfo);
}
// uncensoredLibraryCheck returns the result of trying to download the skylink
// for the uncensored library skylink
function uncensoredLibraryCheck(done) {
const linkInfo = {
name: "Uncensored Library",
skylink: "AAC5glnZyNJ4Ieb4MhnYJGtID6qdMqEjl0or5EvEMt7bWQ",
bodyHash: "60da6cb958699c5acd7f2a2911656ff32fca89a7",
metadata: {
filename: "Unzip_The_Uncensored_Library_Map.zip",
subfiles: {
"Unzip_The_Uncensored_Library_Map.zip": {
filename: "Unzip_The_Uncensored_Library_Map.zip",
contenttype: "application/zip",
len: 76744822,
},
},
},
};
skylinkVerification(done, linkInfo);
}
const uniswapBodyHash = "3965f9a7def085b3a764ddc76a528eda38d72359";
const uniswapMetadata = require("./fixtures/uniswapMetadata.json");
// uniswapCheck returns the result of trying to download the skylink
// for the Uniswap Application
function uniswapCheck(done) {
const linkInfo = {
description: "Uniswap",
skylink: "IAC6CkhNYuWZqMVr1gob1B6tPg4MrBGRzTaDvAIAeu9A9w/",
bodyHash: uniswapBodyHash,
metadata: uniswapMetadata,
};
skylinkVerification(done, linkInfo);
}
// uniswapRedirectCheck returns the result of trying to download the skylink
// for the Uniswap Application without a trailing slash
function uniswapRedirectCheck(done) {
const linkInfo = {
description: "Uniswap",
skylink: "IAC6CkhNYuWZqMVr1gob1B6tPg4MrBGRzTaDvAIAeu9A9w",
bodyHash: uniswapBodyHash,
metadata: uniswapMetadata,
};
skylinkVerification(done, linkInfo);
}
// uniswapIndexFileCheck returns the result of trying to download the skylink
// for the Uniswap Application with a trailing /index.html
function uniswapIndexFileCheck(done) {
const linkInfo = {
name: "Uniswap Skylink Index File",
skylink: "IAC6CkhNYuWZqMVr1gob1B6tPg4MrBGRzTaDvAIAeu9A9w/index.html",
bodyHash: uniswapBodyHash,
metadata: {
filename: "/index.html",
length: 3268,
subfiles: { "index.html": { filename: "index.html", contenttype: "text/html", len: 3268 } },
},
};
skylinkVerification(done, linkInfo);
}
// uniswapHNSCheck returns the result of trying to download the skylink
// for the Uniswap Application with the HNS domain
function uniswapHNSCheck(done) {
const linkInfo = {
description: "Uniswap HNS",
skylink: "hns/uniswap-dex/",
bodyHash: uniswapBodyHash,
metadata: uniswapMetadata,
};
skylinkVerification(done, linkInfo);
}
// uniswapHNSRedirectCheck returns the result of trying to download the skylink
// for the Uniswap Application with the HNS domain and without a trailing slash
function uniswapHNSRedirectCheck(done) {
const linkInfo = {
description: "Uniswap HNS Redirect",
skylink: "hns/uniswap-dex",
bodyHash: uniswapBodyHash,
metadata: uniswapMetadata,
};
skylinkVerification(done, linkInfo);
}
// uniswapHNSResolverCheck returns the result of trying to download the skylink
// for the Uniswap Application via the HNS resolver endpoint
function uniswapHNSResolverCheck(done) {
const linkInfo = {
description: "Uniswap HNS Resolver",
skylink: "hnsres/uniswap-dex/",
bodyHash: "44a3f0f56861ae841a6cb19cb0b3edf98ad610f8",
};
skylinkVerification(done, linkInfo);
}
// uniswapHNSResolverRedirectCheck returns the result of trying to download the skylink
// for the Uniswap Application via the HNS resolver endpoint without the
// trailing slash
function uniswapHNSResolverRedirectCheck(done) {
const linkInfo = {
description: "Uniswap HNS Resolver Redirect",
skylink: "hnsres/uniswap-dex",
bodyHash: "44a3f0f56861ae841a6cb19cb0b3edf98ad610f8",
};
skylinkVerification(done, linkInfo);
}
// skylinkVerification verifies a skylink against provided information.
function skylinkVerification(done, { name, skylink, bodyHash, metadata }) {
const time = process.hrtime();
// Create the query for the skylink
const query = `http://${process.env.PORTAL_URL}/${skylink}?nocache=true`;
// Get the Skylink
superagent
.get(query)
.responseType("blob")
.then(
(response) => {
const entry = { name, up: true, statusCode: response.statusCode, time: calculateElapsedTime(time) };
const info = {};
// Check if the response body is valid by checking against the known hash
const currentBodyHash = hash(response.body);
if (currentBodyHash !== bodyHash) {
entry.up = false;
info.bodyHash = { expected: bodyHash, current: currentBodyHash };
}
// Check if the metadata is valid by deep comparing expected value with response
const metadataHeader = response.header["skynet-file-metadata"];
const currentMetadata = metadataHeader && JSON.parse(metadataHeader);
if (!isEqual(currentMetadata, metadata)) {
entry.up = false;
info.metadata = detailedDiff(metadata, currentMetadata);
}
if (Object.keys(info).length) entry.info = info; // add info only if it exists
done(entry); // Return the entry information
},
(error) => {
const statusCode = error.statusCode || error.status;
const entry = { name, up: false, statusCode, time: calculateElapsedTime(time) };
done(entry); // Return the entry information
}
);
}
module.exports.verboseChecks = [
audioExampleCheck,
covid19PaperCheck,
covid19CoroNopePaperCheck,
dappExampleCheck,
developMomentumIndexFileCheck,
developMomentumCheck,
developMomentumRedirectCheck,
htmlExampleCheck,
imageExampleCheck,
jsonExampleCheck,
pdfExampleCheck,
randomImageCheck,
randomImageRedirectCheck,
skyBayCheck,
skyBayRedirectCheck,
skyBinCheck,
skyBinRedirectCheck,
skyGalleryCheck,
skyGalleryIndexFileCheck,
skyGalleryRedirectCheck,
uncensoredLibraryCheck,
uniswapIndexFileCheck,
uniswapCheck,
uniswapRedirectCheck,
uniswapHNSCheck,
uniswapHNSRedirectCheck,
uniswapHNSResolverCheck,
uniswapHNSResolverRedirectCheck,
];

7
packages/health-check/src/db.js
View File

@ -1,10 +1,13 @@
const fs = require("fs");
const low = require("lowdb");
const FileSync = require("lowdb/adapters/FileSync");
const Memory = require("lowdb/adapters/Memory");
const adapter = process.env.NODE_ENV === "production" ? new FileSync("state/state.json") : new Memory();
if (!fs.existsSync("state")) fs.mkdirSync("state");
const adapter = new FileSync("state/state.json");
const db = low(adapter);
db.defaults({ entries: [] }).write();
db.defaults({ disabled: false, critical: [], verbose: [] }).write();
module.exports = db;

56
packages/health-check/src/endpointHealthCheck.js
View File

@ -1,56 +0,0 @@
const { StatusCodes } = require("http-status-codes");
const { sum, sumBy } = require("lodash");
const db = require("./db");
// getStatus returns the server's current health check status
function getStatus() {
// Grab entry element from DB
const entry = db.get("entries").orderBy("date", "desc").head().value();
// Check that every critical check entry is up
if (entry && entry.checks.every(({ up, critical }) => up && critical)) {
return StatusCodes.OK;
}
// At least one check failed
return StatusCodes.SERVICE_UNAVAILABLE;
}
// getTimeout returns the average time out from a sample of 10 health check
// entries.
function getTimeout() {
if (getStatus() === StatusCodes.SERVICE_UNAVAILABLE) {
return 0;
}
// Grab 10 entries from the database as a sample to determine the average
// timeout for the server.
const sample = db
.get("entries")
.orderBy("date", "desc")
.filter(({ checks }) => checks.every(({ up, critical }) => up && critical))
.take(10)
.value();
// Return average timeout
return Math.round(sum(sample.map(({ checks }) => sumBy(checks, "time"))) / sample.size);
}
// getEntriesSinceYesterday gets the health check entries since yesterday
function getEntriesSinceYesterday() {
const yesterday = new Date();
yesterday.setDate(yesterday.getDate() - 1);
return db
.get("entries")
.orderBy("date", "desc")
.filter(({ date }) => date >= yesterday.toISOString())
.value();
}
module.exports = (req, res) => {
setTimeout(() => {
res.status(getStatus()).send(getEntriesSinceYesterday());
}, getTimeout());
};

5
packages/health-check/src/index.js
View File

@ -17,7 +17,10 @@ const server = express();
server.use(bodyparser.urlencoded({ extended: false }));
server.use(bodyparser.json());
server.get("/health-check", require("./endpointHealthCheck"));
server.get("/health-check", require("./api/index"));
server.get("/health-check/critical", require("./api/critical"));
server.get("/health-check/verbose", require("./api/verbose"));
server.get("/health-check/disabled", require("./api/disabled"));
server.listen(port, host, (error) => {
if (error) throw error;

28
packages/health-check/src/schedule.js
View File

@ -1,28 +1,30 @@
const schedule = require("node-schedule");
const db = require("./db");
const { basicChecks } = require("./basicChecks");
const { verboseChecks } = require("./verboseChecks");
const { criticalChecks } = require("./checks/critical");
const { verboseChecks } = require("./checks/verbose");
// execute the basic health-check script every 5 minutes
const basicJob = schedule.scheduleJob("*/5 * * * *", async () => {
const entry = { date: new Date().toISOString(), checks: [] };
// execute the critical health-check script every 5 minutes
const criticalJob = schedule.scheduleJob("*/5 * * * *", async () => {
const entry = {
date: new Date().toISOString(),
checks: await Promise.all(criticalChecks.map((check) => new Promise(check))),
};
entry.checks = await Promise.all(basicChecks.map((check) => new Promise(check)));
db.get("entries").push(entry).write();
db.get("critical").push(entry).write();
});
// execute the verbose health-check script once per hour
const verboseJob = schedule.scheduleJob("0 * * * *", async () => {
const entry = { date: new Date().toISOString(), checks: [] };
const entry = {
date: new Date().toISOString(),
checks: await Promise.all(verboseChecks.map((check) => new Promise(check))),
};
entry.checks = await Promise.all(verboseChecks.map((check) => new Promise(check)));
db.get("entries").push(entry).write();
db.get("verbose").push(entry).write();
});
// Launch Health check jobs
setTimeout(() => {
basicJob.invoke();
criticalJob.invoke();
verboseJob.invoke();
}, 60 * 1000); // delay for 60s to give other services time to start up

21
packages/health-check/src/utils.js Normal file
View File

@ -0,0 +1,21 @@
/**
* Get the time between start and now in milliseconds
*/
function calculateElapsedTime(start) {
const diff = process.hrtime(start);
return Math.round((diff[0] * 1e9 + diff[1]) / 1e6); // msec
}
/**
* Get the ISO string with yesterday's date set (- 24 hours)
*/
function getYesterdayISOString() {
const date = new Date();
date.setDate(date.getDate() - 1);
return date.toISOString();
}
module.exports = { calculateElapsedTime, getYesterdayISOString };

502
packages/health-check/src/verboseChecks.js
View File

@ -1,502 +0,0 @@
const superagent = require("superagent");
const hash = require("object-hash");
const { detailedDiff } = require("deep-object-diff");
const { isEqual } = require("lodash");
const checks = require("./basicChecks");
// audioExampleCheck returns the result of trying to download the skylink
// for the Example audio file on siasky.net
function audioExampleCheck(done) {
const linkInfo = {
description: "Audio Example",
skylink: "_A2zt5SKoqwnnZU4cBF8uBycSKULXMyeg1c5ZISBr2Q3dA",
bodyHash: "be335f5ad9bc357248f3d35c7e49df491afb6b12",
metadata: { filename: "feel-good.mp3" },
};
skylinkVerification(done, linkInfo);
}
// covid19PaperCheck returns the result of trying to download the skylink
// for a known Covid19 paper
function covid19PaperCheck(done) {
const linkInfo = {
description: "Covid-19 Paper",
skylink: "PAMZVmfutxWoG6Wnl5BRKuWLkDNZR42k_okRRvksJekA3A",
bodyHash: "81b9fb74829a96ceafa429840d1ef0ce44376ddd",
metadata: {
filename: "An Effective Treatment for Coronavirus (COVID-19).pdf",
subfiles: {
"An Effective Treatment for Coronavirus (COVID-19).pdf": {
filename: "An Effective Treatment for Coronavirus (COVID-19).pdf",
contenttype: "application/pdf",
len: 474803,
},
},
},
};
skylinkVerification(done, linkInfo);
}
// covid19CoroNopePaperCheck returns the result of trying to download the skylink
// for another known Covid19 paper
function covid19CoroNopePaperCheck(done) {
const linkInfo = {
description: "Covid-19 CoroNope Paper",
skylink: "bACLKGmcmX4NCp47WwOOJf0lU666VLeT5HRWpWVtqZPjEA",
bodyHash: "901f6fd65ef595f70b6bfebbb2d05942351ef2b3",
metadata: { filename: "coronope.pdf" },
};
skylinkVerification(done, linkInfo);
}
// dappExampleCheck returns the result of trying to download the skylink
// for the Example Dapp on siasky.net
function dappExampleCheck(done) {
const linkInfo = {
description: "Dapp Example (UniSwap)",
skylink: "EAC5HJr5Pu086EAZG4fP_r6Pnd7Ft366vt6t2AnjkoFb9Q/index.html",
bodyHash: "d6ad2506590bb45b5acc6a8a964a3da4d657354f",
metadata: {
filename: "/index.html",
length: 4131,
subfiles: { "index.html": { filename: "index.html", contenttype: "text/html", len: 4131 } },
},
};
skylinkVerification(done, linkInfo);
}
const developMomentumBodyHash = "53b44a9d3cfa9b3d66ce5c29976f4383725d3652";
const developMomentumMetadata = require("./fixtures/developMomentumMetadata.json");
// developMomentumCheck returns the result of trying to download the skylink
// for the Develop Momentum Application
function developMomentumCheck(done) {
const linkInfo = {
description: "Develop Momentum Index File",
skylink: "EAA1fG_ip4C1Vi1Ijvsr1oyr8jpH0Bo9HXya0T3kw-elGw/",
bodyHash: developMomentumBodyHash,
metadata: developMomentumMetadata,
};
skylinkVerification(done, linkInfo);
}
// developMomentumRedirectCheck returns the result of trying to download the skylink
// for the Develop Momentum Application without the tailing slash
function developMomentumRedirectCheck(done) {
const linkInfo = {
description: "Develop Momentum Index File",
skylink: "EAA1fG_ip4C1Vi1Ijvsr1oyr8jpH0Bo9HXya0T3kw-elGw",
bodyHash: developMomentumBodyHash,
metadata: developMomentumMetadata,
};
skylinkVerification(done, linkInfo);
}
// developMomentumIndexFileCheck returns the result of trying to download the skylink
// for the Develop Momentum Application with a trailing /index.html
function developMomentumIndexFileCheck(done) {
const linkInfo = {
description: "Develop Momentum Index File",
skylink: "EAA1fG_ip4C1Vi1Ijvsr1oyr8jpH0Bo9HXya0T3kw-elGw/index.html",
bodyHash: developMomentumBodyHash,
metadata: {
filename: "/index.html",
length: 4981,
subfiles: { "index.html": { filename: "index.html", contenttype: "text/html", len: 4981 } },
},
};
skylinkVerification(done, linkInfo);
}
// htmlExampleCheck returns the result of trying to download the skylink
// for the Example HTML file on siasky.net
function htmlExampleCheck(done) {
const linkInfo = {
description: "HTML Example",
skylink: "PAL0w4SdA5rFCDGEutgpeQ50Om-YkBabtXVOJAkmedslKw",
bodyHash: "c932fd56f98b6db589e56be8018817f13bb29f72",
metadata: { filename: "introduction – Sia API Documentation.html" },
};
skylinkVerification(done, linkInfo);
}
// imageExampleCheck returns the result of trying to download the skylink
// for the Example image on siasky.net
function imageExampleCheck(done) {
const linkInfo = {
description: "Image Example",
skylink: "IADUs8d9CQjUO34LmdaaNPK_STuZo24rpKVfYW3wPPM2uQ",
bodyHash: "313207978d0a88bf2b961f098804e9ab0f82837f",
metadata: { filename: "sia-lm.png" },
};
skylinkVerification(done, linkInfo);
}
// jsonExampleCheck returns the result of trying to download the skylink
// for the Example JSON file on siasky.net
function jsonExampleCheck(done) {
const linkInfo = {
description: "JSON Example",
skylink: "AAC0uO43g64ULpyrW0zO3bjEknSFbAhm8c-RFP21EQlmSQ",
bodyHash: "198771c3d07d5c7302aadcc0697a7298e5e8ccc3",
metadata: { filename: "consensus.json" },
};
skylinkVerification(done, linkInfo);
}
// pdfExampleCheck returns the result of trying to download the skylink
// for the Example PDF file on siasky.net
function pdfExampleCheck(done) {
const linkInfo = {
description: "PDF Example",
skylink: "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg",
bodyHash: "9bd8162e1575569a9041972f7f62d65887063dc3",
metadata: { filename: "sia.pdf" },
};
skylinkVerification(done, linkInfo);
}
// randomImageCheck returns the result of trying to download the skylink for
// a Random Image.
function randomImageCheck(done) {
const linkInfo = {
description: "Random Image",
skylink: "PAHx7JmsU9EFGbqm5q0LNKT2wKfoJ_mhPI8zWlNEXZ8uOQ/",
bodyHash: "4c73c5a0eddd5823be677d7f93bf80cc9338ee9f",
metadata: {
filename: "30355444.png",
subfiles: { "30355444.png": { filename: "30355444.png", contenttype: "image/png", len: 350473 } },
defaultpath: "/30355444.png",
},
};
skylinkVerification(done, linkInfo);
}
// randomImageRedirectCheck returns the result of trying to download the skylink for
// a Random Image with no trailing slash.
function randomImageRedirectCheck(done) {
const linkInfo = {
description: "Random Image Redirect",
skylink: "PAHx7JmsU9EFGbqm5q0LNKT2wKfoJ_mhPI8zWlNEXZ8uOQ",
bodyHash: "4c73c5a0eddd5823be677d7f93bf80cc9338ee9f",
metadata: {
filename: "30355444.png",
subfiles: { "30355444.png": { filename: "30355444.png", contenttype: "image/png", len: 350473 } },
defaultpath: "/30355444.png",
},
};
skylinkVerification(done, linkInfo);
}
// skyBayCheck returns the result of trying to download the skylink for the SkyBay Application.
function skyBayCheck(done) {
const linkInfo = {
description: "SkyBay",
skylink: "EABkMjXzxJRpPz0eO0Or5fy2eo-rz3prdigGwRlyNd9mwA/",
bodyHash: "25d63937c9734fb08d2749c6517d1b3de8ecb856",
metadata: {
filename: "skybay.html",
subfiles: { "skybay.html": { filename: "skybay.html", contenttype: "text/html", len: 11655 } },
},
};
skylinkVerification(done, linkInfo);
}
// skyBayRedirectCheck returns the result of trying to download the skylink
// for the SkyBay Application with no trailing slash.
function skyBayRedirectCheck(done) {
const linkInfo = {
description: "SkyBay Redirect",
skylink: "EABkMjXzxJRpPz0eO0Or5fy2eo-rz3prdigGwRlyNd9mwA",
bodyHash: "25d63937c9734fb08d2749c6517d1b3de8ecb856",
metadata: {
filename: "skybay.html",
subfiles: { "skybay.html": { filename: "skybay.html", contenttype: "text/html", len: 11655 } },
},
};
skylinkVerification(done, linkInfo);
}
// skyBinCheck returns the result of trying to download the skylink for the SkyBin Application.
function skyBinCheck(done) {
const linkInfo = {
description: "SkyBin",
skylink: "CAAVU14pB9GRIqCrejD7rlS27HltGGiiCLICzmrBV0wVtA/",
bodyHash: "767ec67c417e11b97c5db7dad9ea3b6b27cb0d39",
metadata: { filename: "skybin.html" },
};
skylinkVerification(done, linkInfo);
}
// skyBinRedirectCheck returns the result of trying to download the skylink
// for the SkyBin Application with no trailing slash.
function skyBinRedirectCheck(done) {
const linkInfo = {
description: "SkyBin Redirect",
skylink: "CAAVU14pB9GRIqCrejD7rlS27HltGGiiCLICzmrBV0wVtA",
bodyHash: "767ec67c417e11b97c5db7dad9ea3b6b27cb0d39",
metadata: { filename: "skybin.html" },
};
skylinkVerification(done, linkInfo);
}
const skyGalleryBodyHash = "077e54054748d278114f1870f8045a162eb73641";
const skyGalleryMetadata = require("./fixtures/skygalleryMetadata.json");
// skyGalleryCheck returns the result of trying to download the skylink for the SkyGallery Application.
function skyGalleryCheck(done) {
const linkInfo = {
description: "SkyGallery",
skylink: "AADW6GsQcetwDBaDYnGCSTbYjSKY743NtY1A5VRx5sj3Dg/",
bodyHash: skyGalleryBodyHash,
metadata: skyGalleryMetadata,
};
skylinkVerification(done, linkInfo);
}
// skyGalleryIndexFileCheck returns the result of trying to download the skylink
// for the SkyGallery Application with a trailing /index.html
function skyGalleryIndexFileCheck(done) {
const linkInfo = {
description: "SkyGallery Index File",
skylink: "AADW6GsQcetwDBaDYnGCSTbYjSKY743NtY1A5VRx5sj3Dg/index.html",
bodyHash: skyGalleryBodyHash,
metadata: {
filename: "/index.html",
length: 2534,
subfiles: { "index.html": { filename: "index.html", contenttype: "text/html", len: 2534 } },
},
};
skylinkVerification(done, linkInfo);
}
// skyGalleryRedirectCheck returns the result of trying to download the skylink
// for the SkyGallery Application with no trailing slash.
function skyGalleryRedirectCheck(done) {
const linkInfo = {
description: "SkyGallery Redirect",
skylink: "AADW6GsQcetwDBaDYnGCSTbYjSKY743NtY1A5VRx5sj3Dg",
bodyHash: skyGalleryBodyHash,
metadata: skyGalleryMetadata,
};
skylinkVerification(done, linkInfo);
}
// uncensoredLibraryCheck returns the result of trying to download the skylink
// for the uncensored library skylink
function uncensoredLibraryCheck(done) {
const linkInfo = {
description: "Uncensored Library",
skylink: "AAC5glnZyNJ4Ieb4MhnYJGtID6qdMqEjl0or5EvEMt7bWQ",
bodyHash: "60da6cb958699c5acd7f2a2911656ff32fca89a7",
metadata: {
filename: "Unzip_The_Uncensored_Library_Map.zip",
subfiles: {
"Unzip_The_Uncensored_Library_Map.zip": {
filename: "Unzip_The_Uncensored_Library_Map.zip",
contenttype: "application/zip",
len: 76744822,
},
},
},
};
skylinkVerification(done, linkInfo);
}
const uniswapBodyHash = "3965f9a7def085b3a764ddc76a528eda38d72359";
const uniswapMetadata = require("./fixtures/uniswapMetadata.json");
// uniswapCheck returns the result of trying to download the skylink
// for the Uniswap Application
function uniswapCheck(done) {
const linkInfo = {
description: "Uniswap",
skylink: "IAC6CkhNYuWZqMVr1gob1B6tPg4MrBGRzTaDvAIAeu9A9w/",
bodyHash: uniswapBodyHash,
metadata: uniswapMetadata,
};
skylinkVerification(done, linkInfo);
}
// uniswapRedirectCheck returns the result of trying to download the skylink
// for the Uniswap Application without a trailing slash
function uniswapRedirectCheck(done) {
const linkInfo = {
description: "Uniswap",
skylink: "IAC6CkhNYuWZqMVr1gob1B6tPg4MrBGRzTaDvAIAeu9A9w",
bodyHash: uniswapBodyHash,
metadata: uniswapMetadata,
};
skylinkVerification(done, linkInfo);
}
// uniswapIndexFileCheck returns the result of trying to download the skylink
// for the Uniswap Application with a trailing /index.html
function uniswapIndexFileCheck(done) {
const linkInfo = {
description: "Uniswap Skylink Index File",
skylink: "IAC6CkhNYuWZqMVr1gob1B6tPg4MrBGRzTaDvAIAeu9A9w/index.html",
bodyHash: uniswapBodyHash,
metadata: {
filename: "/index.html",
length: 3268,
subfiles: { "index.html": { filename: "index.html", contenttype: "text/html", len: 3268 } },
},
};
skylinkVerification(done, linkInfo);
}
// uniswapHNSCheck returns the result of trying to download the skylink
// for the Uniswap Application with the HNS domain
function uniswapHNSCheck(done) {
const linkInfo = {
description: "Uniswap HNS",
skylink: "hns/uniswap-dex/",
bodyHash: uniswapBodyHash,
metadata: uniswapMetadata,
};
skylinkVerification(done, linkInfo);
}
// uniswapHNSRedirectCheck returns the result of trying to download the skylink
// for the Uniswap Application with the HNS domain and without a trailing slash
function uniswapHNSRedirectCheck(done) {
const linkInfo = {
description: "Uniswap HNS Redirect",
skylink: "hns/uniswap-dex",
bodyHash: uniswapBodyHash,
metadata: uniswapMetadata,
};
skylinkVerification(done, linkInfo);
}
// uniswapHNSResolverCheck returns the result of trying to download the skylink
// for the Uniswap Application via the HNS resolver endpoint
function uniswapHNSResolverCheck(done) {
const linkInfo = {
description: "Uniswap HNS Resolver",
skylink: "hnsres/uniswap-dex/",
bodyHash: "44a3f0f56861ae841a6cb19cb0b3edf98ad610f8",
};
skylinkVerification(done, linkInfo);
}
// uniswapHNSResolverRedirectCheck returns the result of trying to download the skylink
// for the Uniswap Application via the HNS resolver endpoint without the
// trailing slash
function uniswapHNSResolverRedirectCheck(done) {
const linkInfo = {
description: "Uniswap HNS Resolver Redirect",
skylink: "hnsres/uniswap-dex",
bodyHash: "44a3f0f56861ae841a6cb19cb0b3edf98ad610f8",
};
skylinkVerification(done, linkInfo);
}
// skylinkVerification verifies a skylink against known information provided in
// the linkInfo.
function skylinkVerification(done, linkInfo) {
const time = process.hrtime();
// Create the query for the skylink
const query = `http://${process.env.PORTAL_URL}/${linkInfo.skylink}?nocache=true`;
// Get the Skylink
superagent
.get(query)
.responseType("blob")
.end((err, res) => {
// Record the statusCode
const statusCode = (res && res.statusCode) || (err && err.statusCode) || null;
let info = null;
// Determine if the skylink is up. Start with checking if there was an
// error in the request.
let up = err === null;
if (up) {
// Check if the response body is valid by checking against the known
// hash
const validBody = hash(res.body) === linkInfo.bodyHash;
// Check if the metadata is valid
const metadata = res.header["skynet-file-metadata"] ? JSON.parse(res.header["skynet-file-metadata"]) : null;
const validMetadata = isEqual(metadata, linkInfo.metadata);
// Redetermine if the Skylink is up based on the results from the body
// and metadata hash checks
up = up && validBody && validMetadata;
info = {
body: { valid: validBody },
metadata: { valid: validMetadata, diff: detailedDiff(metadata, linkInfo.metadata) },
};
}
// Return the entry information
done({
name: linkInfo.description,
up,
info,
statusCode,
time: checks.catchRequestTime(time),
critical: true,
});
});
}
module.exports.verboseChecks = [
audioExampleCheck,
covid19PaperCheck,
covid19CoroNopePaperCheck,
dappExampleCheck,
developMomentumIndexFileCheck,
developMomentumCheck,
developMomentumRedirectCheck,
htmlExampleCheck,
imageExampleCheck,
jsonExampleCheck,
pdfExampleCheck,
randomImageCheck,
randomImageRedirectCheck,
skyBayCheck,
skyBayRedirectCheck,
skyBinCheck,
skyBinRedirectCheck,
skyGalleryCheck,
skyGalleryIndexFileCheck,
skyGalleryRedirectCheck,
uncensoredLibraryCheck,
uniswapIndexFileCheck,
uniswapCheck,
uniswapRedirectCheck,
uniswapHNSCheck,
uniswapHNSRedirectCheck,
uniswapHNSResolverCheck,
uniswapHNSResolverRedirectCheck,
];

2
packages/webapp/Dockerfile
View File

@ -1,4 +1,4 @@
FROM node:14.8.0-alpine
FROM node:14.9.0
WORKDIR /usr/app

78
packages/webapp/package.json
View File

@ -4,48 +4,48 @@
"version": "1.0.0",
"author": "Nebulous",
"dependencies": {
"axios": "^0.20.0",
"bytes": "^3.1.0",
"classnames": "^2.2.6",
"gatsby": "^2.24.47",
"gatsby-image": "^2.4.16",
"gatsby-plugin-manifest": "^2.4.23",
"gatsby-plugin-matomo": "^0.8.3",
"gatsby-plugin-react-helmet": "^3.3.10",
"gatsby-plugin-remove-serviceworker": "^1.0.0",
"gatsby-plugin-robots-txt": "^1.5.1",
"gatsby-plugin-sass": "^2.3.12",
"gatsby-plugin-sharp": "^2.6.27",
"gatsby-source-filesystem": "^2.3.24",
"gatsby-transformer-sharp": "^2.5.13",
"http-status-codes": "^2.1.1",
"jsonp": "^0.2.1",
"node-sass": "^4.14.0",
"path-browserify": "^1.0.1",
"prop-types": "^15.7.2",
"react": "^16.13.1",
"react-countup": "^4.3.3",
"react-dom": "^16.13.1",
"react-dropzone": "^11.0.2",
"react-helmet": "^6.1.0",
"react-mailchimp-form": "^1.0.2",
"react-mailchimp-subscribe": "^2.1.0",
"react-reveal": "^1.2.2",
"react-syntax-highlighter": "^13.5.0",
"react-visibility-sensor": "^5.1.1",
"axios": "0.20.0",
"bytes": "3.1.0",
"classnames": "2.2.6",
"gatsby": "2.24.54",
"gatsby-image": "2.4.17",
"gatsby-plugin-manifest": "2.4.28",
"gatsby-plugin-matomo": "0.8.3",
"gatsby-plugin-react-helmet": "3.3.10",
"gatsby-plugin-remove-serviceworker": "1.0.0",
"gatsby-plugin-robots-txt": "1.5.1",
"gatsby-plugin-sass": "2.3.12",
"gatsby-plugin-sharp": "2.6.33",
"gatsby-source-filesystem": "2.3.28",
"gatsby-transformer-sharp": "2.5.14",
"http-status-codes": "2.1.2",
"jsonp": "0.2.1",
"node-sass": "4.14.1",
"path-browserify": "1.0.1",
"prop-types": "15.7.2",
"react": "16.13.1",
"react-countup": "4.3.3",
"react-dom": "16.13.1",
"react-dropzone": "11.0.3",
"react-helmet": "6.1.0",
"react-mailchimp-form": "1.0.2",
"react-mailchimp-subscribe": "2.1.0",
"react-reveal": "1.2.2",
"react-syntax-highlighter": "13.5.3",
"react-visibility-sensor": "5.1.1",
"skynet-js": "0.0.8",
"typeface-metropolis": "^0.0.74"
"typeface-metropolis": "0.0.74"
},
"devDependencies": {
"cypress": "^5.0.0",
"cypress-file-upload": "^4.0.7",
"eslint": "^7.7.0",
"eslint-config-prettier": "^6.11.0",
"eslint-plugin-cypress": "^2.11.1",
"eslint-plugin-react": "^7.20.6",
"husky": "^4.2.5",
"lint-staged": "^10.2.11",
"prettier": "^2.0.5"
"cypress": "5.1.0",
"cypress-file-upload": "4.1.1",
"eslint": "7.8.1",
"eslint-config-prettier": "6.11.0",
"eslint-plugin-cypress": "2.11.1",
"eslint-plugin-react": "7.20.6",
"husky": "4.3.0",
"lint-staged": "10.3.0",
"prettier": "2.1.1"
},
"keywords": [
"sia",

41
packages/webapp/src/components/CodeExamples/Code.js
View File

@ -1,11 +1,14 @@
export const python = `from siaskynet import Skynet
export const python = `import siaskynet as skynet
# create a client
client = skynet.SkynetClient()
# upload
skylink = Skynet.upload_file("./src.jpg")
skylink = client.upload_file("./src.jpg")
print("Upload successful, skylink: " + skylink)
# download
Skynet.download_file("./dst.jpg", skylink)
client.download_file("./dst.jpg", skylink)
print("Download successful")`;
export const curl = `# upload
@ -14,22 +17,18 @@ curl -X POST "https://siasky.net/skynet/skyfile" -F file=@src.jpg
# download
curl "https://siasky.net/[skylink]" -o dst.jpg`;
export const node = `const skynet = require('@nebulous/skynet');
export const node = `const { SkynetClient } = require('@nebulous/skynet');
// create a client
const client = new SkynetClient();
(async () => {
// upload
const skylink = await skynet.UploadFile(
"./src.jpg",
skynet.DefaultUploadOptions
);
const skylink = await client.UploadFile("./src.jpg");
console.log(\`Upload successful, skylink: \${skylink}\`);
// download
await skynet.DownloadFile(
"./dst.jpg",
skylink,
skynet.DefaultDownloadOptions
);
await client.DownloadFile("./dst.jpg", skylink);
console.log('Download successful');
})()`;
@ -39,21 +38,21 @@ import (
"fmt"
skynet "github.com/NebulousLabs/go-skynet"
)
var client = skynet.New()
func main() {
// upload
skylink, err := skynet.UploadFile("./src.jpg", skynet.DefaultUploadOptions)
skylink, err := client.UploadFile("./src.jpg", skynet.DefaultUploadOptions)
if err != nil {
fmt.Printf("Unable to upload: %v", err.Error())
return
panic("Unable to upload: " + err.Error())
}
fmt.Printf("Upload successful, skylink: %v\\n", skylink)
// download
err = skynet.DownloadFile("./dst.jpg", skylink, skynet.DefaultDownloadOptions)
err = client.DownloadFile("./dst.jpg", skylink, skynet.DefaultDownloadOptions)
if err != nil {
fmt.Printf("Something went wrong, please try again.\\nError: %v", err.Error())
return
panic("Something went wrong, please try again.\\nError: " + err.Error())
}
fmt.Println("Download successful")
}`;

2
packages/webapp/src/components/HomeSamples/HomeSamples.js
View File

@ -12,7 +12,7 @@ const samples = [
{ type: "Audio", url: "/_A2zt5SKoqwnnZU4cBF8uBycSKULXMyeg1c5ZISBr2Q3dA" },
{ type: "Video", url: "/CABAB_1Dt0FJsxqsu_J4TodNCbCGvtFf1Uys_3EgzOlTcg" },
{ type: "JSON", url: "/AAC0uO43g64ULpyrW0zO3bjEknSFbAhm8c-RFP21EQlmSQ" },
{ type: "Dapp", url: "/EAC5HJr5Pu086EAZG4fP_r6Pnd7Ft366vt6t2AnjkoFb9Q/index.html" },
{ type: "Dapp", url: "/IAC6CkhNYuWZqMVr1gob1B6tPg4MrBGRzTaDvAIAeu9A9w/" },
];
export default function HomeSamples() {

17
setup-scripts/funds-checker.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3
"""
health-checker runs simple health checks on a portal node using the siad API and
funds-checker runs simple checks on a portal node using the siad API and
dispatches messages to a Discord channel.
"""
@ -21,23 +21,22 @@ async def exit_after(delay):
async def on_ready():
await run_checks()
asyncio.create_task(exit_after(3))
await client.close()
async def run_checks():
print("Running Skynet portal health checks")
print("Running Skynet portal funds checks")
try:
await check_health()
await check_funds()
except: # catch all exceptions
trace = traceback.format_exc()
await send_msg(client, "```\n{}\n```".format(trace), force_notify=True)
# check_health checks that the wallet is unlocked, that it has at least 1
# allowance worth of money left, and if more than hald the allowance is spent. If
# all checks pass it sends a informational message.
async def check_health():
# check_funds checks that the wallet is unlocked, that it has at least 1
# allowance worth of money left, and if less than half the allowance is spent.
# If all checks pass it sends an informational message.
async def check_funds():
print("\nChecking wallet/funds health...")
wallet_get = siad.get_wallet()
renter_get = siad.get_renter()
@ -73,7 +72,7 @@ async def check_health():
return
# Send an informational heartbeat if all checks passed.
await send_msg(client, "Health checks passed:\n{} \n{}".format(balance_msg, alloc_msg))
await send_msg(client, "Funds checks passed:\n{} \n{}".format(balance_msg, alloc_msg))
client.run(bot_token)

168
setup-scripts/health-checker.py Executable file
View File

@ -0,0 +1,168 @@
#!/usr/bin/env python3
import asyncio
import io
import json
import os
import re
import sys
import traceback
from datetime import datetime, timedelta
import discord
import pytz.reference
import requests
from bot_utils import setup, send_msg
from tzlocal import get_localzone
"""
health-checker reads the /health-check endpoint of the portal and dispatches
messages to a Discord channel.
"""
# Get the number of hours to look back in the logs or use 1 as default.
CHECK_HOURS = 1
if len(sys.argv) > 3:
CHECK_HOURS = int(sys.argv[3])
# Discord messages have a limit on their length set at 2000 bytes. We use
# a lower limit in order to leave some space for additional message text.
DISCORD_MAX_MESSAGE_LENGTH = 1900
GB = 1 << 30 # 1 GiB in bytes
# We are going to issue Discord warnings if the free space on a server falls
# under this threshold.
FREE_DISK_SPACE_THRESHOLD = 50 * GB
bot_token = setup()
client = discord.Client()
# exit_after kills the script if it hasn't exited on its own after `delay` seconds
async def exit_after(delay):
await asyncio.sleep(delay)
os._exit(0)
@client.event
async def on_ready():
await run_checks()
asyncio.create_task(exit_after(3))
async def run_checks():
print("Running Skynet portal health checks")
try:
await check_load_average()
await check_disk()
# await check_health() # FIXME: adjust it to work with https://github.com/NebulousLabs/skynet-webportal/pull/389
except:
trace = traceback.format_exc()
print("[DEBUG] run_checks() failed.")
if len(trace) < DISCORD_MAX_MESSAGE_LENGTH:
await send_msg(client, "```\n{}\n```".format(trace), force_notify=False)
else:
await send_msg(client, "Failed to run the portal health checks!",
file=discord.File(io.BytesIO(trace.encode()), filename="failed_checks.log"),
force_notify=True)
# check_load_average monitors the system's load average value and issues a
# warning message if it exceeds 10.
async def check_load_average():
uptime_string = os.popen("uptime").read().strip()
if sys.platform == "Darwin":
pattern = "^.*load averages: \d*\.\d* \d*\.\d* (\d*\.\d*)$"
else:
pattern = "^.*load average: \d*\.\d*, \d*\.\d*, (\d*\.\d*)$"
load_av = re.match(pattern, uptime_string).group(1)
if float(load_av) > 10:
await send_msg(client, "High system load detected: `uptime: {}`".format(uptime_string), force_notify=True)
# check_disk checks the amount of free space on the /home partition and issues
# a warning message if it's under FREE_DISK_SPACE_THRESHOLD GB.
async def check_disk():
# We check free disk space in 1024 byte units, so it's easy to convert.
df = os.popen("df --block-size=1024").read().strip()
volumes = {}
for line in df.split("\n")[1:]:
fields = list(filter(None, line.split(" ")))
# -1 is "mounted on", 3 is "available space" in KiB which we want in bytes
volumes[fields[-1]] = fields[3] * 1024
# List of mount point, longest to shortest. We'll use that to find the best
# fit for the volume we want to check.
mount_points = sorted(volumes.keys(), key=len, reverse=True)
wd = os.popen("pwd").read().strip()
vol = ""
for mp in mount_points:
if wd.startswith(mp):
vol = mp
break
if vol == "":
msg = "Failed to check free disk space! Didn't find a suitable mount point to check.\ndf output:\n{}".format(df)
await send_msg(client, msg)
return
if int(volumes[vol]) < FREE_DISK_SPACE_THRESHOLD:
free_space_gb = "{:.2f}".format(int(volumes[vol]) / GB)
await send_msg(client, "WARNING! Low disk space: {}GiB".format(free_space_gb), force_notify=True)
return
# check_health checks /health-check endpoint and reports recent issues
async def check_health():
print("\nChecking portal health status...")
try:
res = requests.get("http://localhost/health-check", verify=False)
except:
trace = traceback.format_exc()
print("[DEBUG] check_health() failed.")
if len(trace) < DISCORD_MAX_MESSAGE_LENGTH:
await send_msg(client, "```\n{}\n```".format(trace), force_notify=False)
else:
await send_msg(client, "Failed to run the checks!",
file=discord.File(io.BytesIO(trace.encode()), filename="failed_checks.log"),
force_notify=True)
return
# Check the health records.
passed_checks = 0
failed_checks = 0
failed_critical = 0
failed_records = []
time_limit_unaware = datetime.now() - timedelta(hours=CHECK_HOURS) # local time
time_limit = time_limit_unaware.astimezone(get_localzone()) # time with time zone
for rec in res.json():
time_unaware = datetime.strptime(rec['date'], '%Y-%m-%dT%H:%M:%S.%fZ') # time in UTC
time = pytz.utc.localize(time_unaware) # time with time zone
if time < time_limit:
continue
bad = False
for check in rec['checks']:
if check['up'] == False:
bad = True
failed_checks += 1
if check['critical']:
failed_critical += 1
if bad:
# We append the entire record, so we can get the full context.
failed_records.append(rec)
passed_checks += 1
checks = passed_checks + failed_checks
if len(failed_records) > 0:
message = "Found {}/{} failed checks ({} critical) over the last {} hours!".format(failed_checks, checks,
failed_critical, CHECK_HOURS)
file = discord.File(io.BytesIO(json.dumps(failed_records, indent=2).encode()), filename="failed_checks.log")
notifyTeam = failed_critical > 0
await send_msg(client, message, file=file, force_notify=notifyTeam)
return
# Send an informational heartbeat if all checks passed but only if it's in
# the first CHECK_HOURS hours of the day, essentially the first call.
if datetime.now().hour < CHECK_HOURS:
await send_msg(client, "Health checks passed: {}/{}\n".format(passed_checks, checks))
client.run(bot_token)

75
setup-scripts/log-checker.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
import discord, sys, traceback, io, os, asyncio, re
import discord, sys, traceback, io, os, asyncio
from bot_utils import setup, send_msg
from datetime import datetime, timedelta
from subprocess import Popen, PIPE
@ -18,8 +18,19 @@ Arguments:
"""
# The default check interval in hours.
DEFAULT_CHECK_INTERVAL = 1
# Get the container name as an argument or use "sia" as default.
CONTAINER_NAME = "sia"
if len(sys.argv) > 2:
CONTAINER_NAME = sys.argv[2]
# Get the number of hours to look back in the logs or use 1 as default.
CHECK_HOURS = 1
if len(sys.argv) > 3:
CHECK_HOURS = int(sys.argv[3])
# Discord messages have a limit on their length set at 2000 bytes. We use
# a lower limit in order to leave some space for additional message text.
DISCORD_MAX_MESSAGE_LENGTH = 1900
bot_token = setup()
client = discord.Client()
@ -40,48 +51,23 @@ async def on_ready():
async def run_checks():
print("Running Skynet portal log checks")
try:
await check_load_average()
await check_docker_logs()
except: # catch all exceptions
trace = traceback.format_exc()
await send_msg(client, "```\n{}\n```".format(trace), force_notify=False)
# check_load_average monitors the system's load average value and issues a
# warning message if it exceeds 10.
async def check_load_average():
uptime_string = os.popen("uptime").read().strip()
# pattern = ""
if sys.platform == "Darwin":
pattern = "^.*load averages: \d*\.\d* \d*\.\d* (\d*\.\d*)$"
else:
pattern = "^.*load average: \d*\.\d*, \d*\.\d*, (\d*\.\d*)$"
load_av = re.match(pattern, uptime_string).group(1)
if float(load_av) > 10:
await send_msg(client, "High system load detected: `uptime: {}`".format(uptime_string), force_notify=True)
# check_docker_logs checks the docker logs by filtering on the docker image name
async def check_docker_logs():
print("\nChecking docker logs...")
# Get the container name as an argument or use "sia" as default.
container_name = "sia"
if len(sys.argv) > 2:
container_name = sys.argv[2]
# Get the number of hours to look back in the logs or use 1 as default.
check_hours = DEFAULT_CHECK_INTERVAL
if len(sys.argv) > 3:
check_hours = int(sys.argv[3])
now = datetime.now()
time = now - timedelta(hours=check_hours)
time_string = "{}h".format(check_hours)
time = now - timedelta(hours=CHECK_HOURS)
time_string = "{}h".format(CHECK_HOURS)
# Read the logs.
print("[DEBUG] Will run `docker logs --since {} {}`".format(time_string, container_name))
proc = Popen(["docker", "logs", "--since", time_string, container_name], stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True)
print("[DEBUG] Will run `docker logs --since {} {}`".format(time_string, CONTAINER_NAME))
proc = Popen(["docker", "logs", "--since", time_string, CONTAINER_NAME], stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True)
std_out, std_err = proc.communicate()
if len(std_err) > 0:
@ -90,27 +76,28 @@ async def check_docker_logs():
if len(std_err) > one_mb:
pos = std_err.find("\n", -one_mb)
std_err = std_err[pos+1:]
upload_name = "{}-{}-{}-{}-{}:{}:{}_err.log".format(container_name, time.year, time.month, time.day, time.hour, time.minute, time.second)
upload_name = "{}-{}-{}-{}-{}:{}:{}_err.log".format(CONTAINER_NAME, time.year, time.month, time.day, time.hour, time.minute, time.second)
await send_msg(client, "Error(s) found in log!", file=discord.File(io.BytesIO(std_err.encode()), filename=upload_name), force_notify=True)
# Send at most 1900 characters of logs, rounded down to the nearest new line.
# This is a limitation in the size of Discord messages - they can be at most
# 2000 characters long (and we send some extra characters before the error log).
if len(std_err) > 1900:
pos = std_err.find("\n", -1900)
# Send at most DISCORD_MAX_MESSAGE_LENGTH characters of logs, rounded
# down to the nearest new line. This is a limitation in the size of
# Discord messages - they can be at most 2000 characters long (and we
# send some extra characters before the error log).
if len(std_err) > DISCORD_MAX_MESSAGE_LENGTH:
pos = std_err.find("\n", -DISCORD_MAX_MESSAGE_LENGTH)
std_err = std_err[pos+1:]
await send_msg(client, "Error(s) preview:\n{}".format(std_err), force_notify=True)
return
# If there are any critical errors. upload the whole log file.
if 'Critical' in std_out or 'panic' in std_out:
upload_name = "{}-{}-{}-{}-{}:{}:{}.log".format(container_name, time.year, time.month, time.day, time.hour, time.minute, time.second)
await send_msg(client, "Critical error found in log!", file=discord.File(io.BytesIO(std_out.encode()), filename=upload_name), force_notify=True)
# If there are any critical or severe errors. upload the whole log file.
if 'Critical' in std_out or 'Severe' in std_out or 'panic' in std_out:
upload_name = "{}-{}-{}-{}-{}:{}:{}.log".format(CONTAINER_NAME, time.year, time.month, time.day, time.hour, time.minute, time.second)
await send_msg(client, "Critical or Severe error found in log!", file=discord.File(io.BytesIO(std_out.encode()), filename=upload_name), force_notify=True)
return
# No critical errors, return a heartbeat type message
# No critical or severe errors, return a heartbeat type message
pretty_before = time.strftime("%I:%M%p")
pretty_now = now.strftime("%I:%M%p")
await send_msg(client, "No critical warnings in log from `{}` to `{}`".format(pretty_before, pretty_now))
await send_msg(client, "No critical or severe warnings in log from `{}` to `{}`".format(pretty_before, pretty_now))
client.run(bot_token)

9
setup-scripts/setup-health-check-scripts.sh
View File

@ -5,11 +5,12 @@ set -e # exit on first error
sudo apt-get update
sudo apt-get -y install python3-pip
pip3 install discord.py
pip3 install python-dotenv
pip3 install discord.py python-dotenv requests pytz tzlocal
fundsCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env"
logsCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8"
healthCheck="0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1"
(crontab -u user -l; echo "$fundsCheck" ) | crontab -u user -
(crontab -u user -l; echo "$logsCheck" ) | crontab -u user -
(crontab -u user -l; echo "$fundsCheck" ) | crontab -u user -
(crontab -u user -l; echo "$logsCheck" ) | crontab -u user -
(crontab -u user -l; echo "$healthCheck" ) | crontab -u user -

1167
yarn.lock
View File

File diff suppressed because it is too large Load Diff