LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor functions and address comments

This commit is contained in:
Matthew Sevey 2022-04-27 15:40:31 -04:00
parent 084c12fde4
commit 67c3fd1cf3
No known key found for this signature in database
GPG Key ID: 9ADDD344F13057F6
3 changed files with 116 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
packages/health-check/src/checks/critical.js
View File

@ -2,7 +2,14 @@ const crypto = require("crypto");
const got = require("got");
const FormData = require("form-data");
const { isEqual } = require("lodash");
const { calculateElapsedTime, getResponseContent, getAuthCookie, isPortalModuleEnabled } = require("../utils");
const {
calculateElapsedTime,
getResponseContent,
getAuthCookie,
isPortalModuleEnabled,
uploadFunc,
sectorSize,
} = require("../utils");
const { SkynetClient, stringToUint8ArrayUtf8, genKeyPairAndSeed } = require("skynet-js");
const MODULE_BLOCKER = "b";
@ -10,8 +17,6 @@ const MODULE_BLOCKER = "b";
const skynetClient = new SkynetClient(`https://${process.env.PORTAL_DOMAIN}`);
const exampleSkylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q";
const sectorSize = 1 << 22; // 40 MiB
// check that any relevant configuration is properly set in skyd
async function skydConfigCheck(done) {
const time = process.hrtime();
@ -51,81 +56,6 @@ async function uploadLargeFileCheck(done) {
return uploadFunc(done, payload, "upload_large_file", true);
}
// uploadFunc handles the upload and health check for the upload checks
async function uploadFunc(done, payload, name, isLarge = false) {
// Get time for calculating the elapsed time for the check
const time = process.hrtime();
// Initialize check params
const authCookie = await getAuthCookie();
const data = { up: false };
const form = new FormData();
form.append("file", payload, { filename: `${name}.txt`, contentType: "text/plain" });
try {
// Upload file
const response = await got.post(`https://${process.env.PORTAL_DOMAIN}/skynet/skyfile`, {
body: form,
headers: { cookie: authCookie },
});
// Check file health
const responseContent = getResponseContent(response);
const skylink = responseContent.skylink;
await skylinkHealthCheck(skylink, 60, authCookie, isLarge);
// Update data response
data.statusCode = response.statusCode;
data.up = true;
data.ip = response.ip;
} catch (error) {
data.statusCode = error.response?.statusCode || error.statusCode || error.status;
data.errorMessage = error.message;
data.errorResponseContent = getResponseContent(error.response);
data.ip = error?.response?.ip ?? null;
}
done({ name, time: calculateElapsedTime(time), ...data });
}
// skylinkHealthCheck checks if the skylink has reached full redundancy
async function skylinkHealthCheck(skylink, numRetries = 30, authCookie, isLarge = false) {
// Get the health of the skylink
const response = await got(`https://${process.env.PORTAL_DOMAIN}/skynet/health/skylink/${skylink}`, {
headers: { cookie: authCookie },
});
const healthData = getResponseContent(response);
// Check Basesectorredundancy first
if (healthData.basesectorredundancy !== 10 && numRetries > 0) {
// Semi-smart sleep before retrying. Sleep longer if the redundancy is
// lower.
await new Promise((r) => setTimeout(r, (10 - healthData.basesectorredundancy) * 1000));
return skylinkHealthCheck(skylink, numRetries - 1, authCookie, isLarge);
}
// Check the Fanout redundancy if it is a large file
if (isLarge && healthData.fanoutredundancy != 3 && numRetries > 0) {
// Semi-smart sleep before retrying. Sleep longer if the redundancy is
// lower.
await new Promise((r) => setTimeout(r, (3 - healthData.fanoutredundancy) * 10000));
return skylinkHealthCheck(skylink, numRetries - 1, authCookie, isLarge);
}
// Throw error if the basesectorredundancy never reached 10x
if (healthData.basesectorredundancy !== 10 && numRetries === 0) {
throw new Error(`File uploaded but basesector did not reach full redundancy: ${healthData.basesectorredundancy}`);
}
// Throw error if the fanoutredundancy never reached 3x
if (isLarge && healthData.fanoutredundancy !== 3 && numRetries === 0) {
throw new Error(`File uploaded but fanout did not reach full redundancy: ${healthData.fanoutredundancy}`);
}
return response;
}
// websiteCheck checks whether the main website is working
async function websiteCheck(done) {
return done(await genericAccessCheck("website", `https://${process.env.PORTAL_DOMAIN}`));

10
packages/health-check/src/checks/extended.js
View File

@ -2,7 +2,7 @@ const got = require("got");
const hasha = require("hasha");
const { detailedDiff } = require("deep-object-diff");
const { isEqual } = require("lodash");
const { calculateElapsedTime, ensureValidJSON, getResponseContent, getAuthCookie } = require("../utils");
const { calculateElapsedTime, ensureValidJSON, getResponseContent, getAuthCookie, uploadFunc } = require("../utils");
const { parseSkylink } = require("skynet-js");
// audioExampleCheck returns the result of trying to download the skylink
@ -1206,6 +1206,13 @@ async function skylinkVerification(done, expected, { followRedirect = true, meth
}
}
// uploadLargeFileCheck returns the result of uploading a large file
async function uploadLargeFileCheck(done) {
const payload = Buffer.from(crypto.randomBytes(sectorSize));
return uploadFunc(done, payload, "upload_large_file", true);
}
module.exports = [
audioExampleCheck,
covid19PaperCheck,
@ -1244,4 +1251,5 @@ module.exports = [
skappHackerPaste,
skappHowAboutSkapp,
skappSkyDeploy,
uploadLargeFileCheck,
];

99
packages/health-check/src/utils.js
View File

@ -3,6 +3,20 @@ const ipRegex = new RegExp(
`^(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)(?:\\.(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)){3}$`
);
// sectorSize is the skyd sector size
const sectorSize = 1 << 22; // 40 MiB
// SECOND is a helper constant for defining the number of milliseconds in a
// second
const SECOND = 1000;
// defaultBaseSectorRedundancy is the default baseSectorRedundancy defined by
// skyd
const defaultBaseSectorRedundancy = 10;
// defaultFanoutRedundancy is the default fanout redundancy defined by skyd
const defaultFanoutRedundancy = 3;
/**
* Get the time between start and now in milliseconds
*/
@ -132,6 +146,89 @@ function isPortalModuleEnabled(module) {
return process.env.PORTAL_MODULES && process.env.PORTAL_MODULES.indexOf(module) !== -1;
}
// sleep is a helper method of sleeping for for given time. The input time is
// expected in seconds
async function sleep(seconds) {
return new Promise((r) => setTimeout(r, seconds * SECOND));
}
// skylinkHealthCheck checks if the skylink has reached full redundancy
async function skylinkHealthCheck(skylink, numRetries = 30, authCookie, isLarge = false) {
// Get the health of the skylink
const response = await got(`https://${process.env.PORTAL_DOMAIN}/skynet/health/skylink/${skylink}`, {
headers: { cookie: authCookie },
});
const healthData = getResponseContent(response);
// Check Basesectorredundancy first
if (healthData.basesectorredundancy !== defaultBaseSectorRedundancy && numRetries > 0) {
// Semi-smart sleep before retrying. Sleep longer if the redundancy is
// lower.
await sleep(10 - healthData.basesectorredundancy);
return skylinkHealthCheck(skylink, numRetries - 1, authCookie, isLarge);
}
// Check the Fanout redundancy if it is a large file
if (isLarge && healthData.fanoutredundancy != defaultFanoutRedundancy && numRetries > 0) {
// Semi-smart sleep before retrying. Sleep longer if the redundancy is
// lower.
await sleep((defaultFanoutRedundancy - healthData.fanoutredundancy) * 10);
return skylinkHealthCheck(skylink, numRetries - 1, authCookie, isLarge);
}
// Throw error if the basesectorredundancy never reached 10x
if (healthData.basesectorredundancy !== defaultBaseSectorRedundancy && numRetries === 0) {
throw new Error(`File uploaded but basesector did not reach full redundancy: ${healthData.basesectorredundancy}`);
}
// Throw error if the fanoutredundancy never reached 3x
if (isLarge && healthData.fanoutredundancy !== defaultFanoutRedundancy && numRetries === 0) {
throw new Error(`File uploaded but fanout did not reach full redundancy: ${healthData.fanoutredundancy}`);
}
return response;
}
// uploadFunc handles the upload and health check for the upload checks
async function uploadFunc(done, payload, name, isLarge = false) {
// Get time for calculating the elapsed time for the check
const time = process.hrtime();
// Initialize check params
const authCookie = await getAuthCookie();
const data = { up: false };
const form = new FormData();
form.append("file", payload, { filename: `${name}.txt`, contentType: "text/plain" });
try {
// Upload file
const response = await got.post(`https://${process.env.PORTAL_DOMAIN}/skynet/skyfile`, {
body: form,
headers: { cookie: authCookie },
});
// Check file health
const responseContent = getResponseContent(response);
const skylink = responseContent.skylink;
await skylinkHealthCheck(skylink, 60, authCookie, isLarge);
// Update data response
data.statusCode = response.statusCode;
data.up = true;
data.ip = response.ip;
data.skylink = skylink;
} catch (error) {
data.statusCode = error.response?.statusCode || error.statusCode || error.status;
data.errorMessage = error.message;
data.errorResponseContent = getResponseContent(error.response);
data.ip = error?.response?.ip ?? null;
data.skylink = skylink;
}
done({ name, time: calculateElapsedTime(time), ...data });
}
module.exports = {
calculateElapsedTime,
getYesterdayISOString,
@ -141,4 +238,6 @@ module.exports = {
isPortalModuleEnabled,
ipCheckService,
ipRegex,
sectorSize,
uploadFunc,
};