LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into accounts

This commit is contained in:
Karol Wypchlo 2021-01-29 12:49:44 +01:00
parent e20d4ece14 2d91181843
commit 1054712966
10 changed files with 1257 additions and 2211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
docker/nginx/conf.d/client.conf
View File

@ -177,6 +177,9 @@ server {
-- example response: '{"skylink":"sia://XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"}'
local hnsres_json = json.decode(hnsres_res.body)
-- define local variable containing rest of the skylink if provided
local skylink_rest
if hnsres_json.skylink then
-- try to match the skylink with sia:// prefix
skylink, skylink_rest = string.match(hnsres_json.skylink, "sia://([^/?]+)(.*)")
@ -190,7 +193,7 @@ server {
local datakey = hnsres_json.registry.datakey
-- make a get request to /skynet/registry endpoint with the credentials from text record
local registry_res = ngx.location.capture("/skynet/registry?publickey=" .. publickey .. "&datakey=" .. datakey)
local registry_res = ngx.location.capture("/skynet/registry/cached?publickey=" .. publickey .. "&datakey=" .. datakey)
-- we want to fail with a generic 404 when /skynet/registry returns anything but 200 OK
if registry_res.status ~= ngx.HTTP_OK then
@ -213,7 +216,7 @@ server {
end
ngx.var.skylink = skylink
if request_uri_rest == "/" and skylink_rest ~= "" and skylink_rest ~= "/" then
if request_uri_rest == "/" and skylink_rest ~= nil and skylink_rest ~= "" and skylink_rest ~= "/" then
ngx.var.rest = skylink_rest
else
ngx.var.rest = request_uri_rest
@ -248,6 +251,22 @@ server {
proxy_pass http://handshake-api:3100;
}
# internal registry endpoint that caches calls for a certain period of time
# it is not suitable for every registry call but some requests might be cached
# and we are using it currently for caching registry resolutions from /hns calls
location /skynet/registry/cached {
internal; # internal endpoint only
access_log off; # do not log traffic
proxy_cache skynet;
proxy_cache_key publickey=$arg_publickey&datakey=$arg_datakey; # cache based on publickey and datakey
proxy_cache_valid 200 30s; # cache only 200 responses and only for 30 seconds
proxy_cache_lock on; # queue cache requests for the same resource until it is fully cached
proxy_cache_bypass $cookie_nocache $arg_nocache; # add cache bypass option
proxy_pass http://127.0.0.1/skynet/registry$is_args$args;
}
location /skynet/registry {
include /etc/nginx/conf.d/include/cors;
include /etc/nginx/conf.d/include/sia-auth;

2
packages/health-check/src/checks/critical.js
View File

@ -1,6 +1,6 @@
const fs = require("fs");
const superagent = require("superagent");
const tmp = require('tmp');
const tmp = require("tmp");
const { StatusCodes } = require("http-status-codes");
const { calculateElapsedTime, getResponseContent } = require("../utils");

105
packages/health-check/src/checks/verbose.js
View File

@ -874,11 +874,11 @@ function skyGalleryRedirectCheck(done) {
// for the uncensored library skylink
function uncensoredLibraryCheck(done) {
const linkInfo = {
name: "Uncensored Library",
name: "The Uncensored Library V2",
skylink: "AAC5glnZyNJ4Ieb4MhnYJGtID6qdMqEjl0or5EvEMt7bWQ",
bodyHash: "60da6cb958699c5acd7f2a2911656ff32fca89a7",
headers: {
"skynet-skylink": undefined,
"skynet-skylink": "AAC5glnZyNJ4Ieb4MhnYJGtID6qdMqEjl0or5EvEMt7bWQ",
"skynet-file-metadata": {
filename: "Unzip_The_Uncensored_Library_Map.zip",
subfiles: {
@ -889,8 +889,59 @@ function uncensoredLibraryCheck(done) {
},
},
},
"content-disposition": 'attachment; filename="Unzip_The_Uncensored_Library_Map.zip"',
"content-type": "application/octet-stream",
"content-disposition": 'inline; filename="Unzip_The_Uncensored_Library_Map.zip"',
"content-type": "application/zip",
},
};
skylinkVerification(done, linkInfo);
}
function uncensoredLibraryPressReleaseCheck(done) {
const linkInfo = {
name: "The Uncensored Library - Press Release",
skylink: "AABHwuml_EhvyY8Gm7j1E2xGwodUNAJgX0A6-Cd22p9kNA",
bodyHash: "323217f643c3e3f1fe7532e72ac01bb0748c97be",
headers: {
"skynet-skylink": "AABHwuml_EhvyY8Gm7j1E2xGwodUNAJgX0A6-Cd22p9kNA",
"skynet-file-metadata": {
filename: "press-release-Reporters-Without-Borders-The-Uncensored-Library.zip",
subfiles: {
"press-release-Reporters-Without-Borders-The-Uncensored-Library.zip": {
filename: "press-release-Reporters-Without-Borders-The-Uncensored-Library.zip",
contenttype: "application/zip",
len: 383501533,
},
},
},
"content-disposition": 'inline; filename="press-release-Reporters-Without-Borders-The-Uncensored-Library.zip"',
"content-type": "application/zip",
},
};
// request too large, use head just to verify the headers
skylinkVerification(done, linkInfo, { method: "head" });
}
function uncensoredLibraryV2Check(done) {
const linkInfo = {
name: "The Uncensored Library V2",
skylink: "AAAs-JOsRGWgABYIo7AwTDqSX79-BxQKjDj0wiRGoRPFnw",
bodyHash: "1c6a885c060af8325eee82a11e9d64a13b228015",
headers: {
"skynet-skylink": "AAAs-JOsRGWgABYIo7AwTDqSX79-BxQKjDj0wiRGoRPFnw",
"skynet-file-metadata": {
filename: "The Uncensored Library V2.zip",
subfiles: {
"The Uncensored Library V2.zip": {
filename: "The Uncensored Library V2.zip",
contenttype: "application/zip",
len: 101262134,
},
},
},
"content-disposition": 'inline; filename="The Uncensored Library V2.zip"',
"content-type": "application/zip",
},
};
@ -1024,6 +1075,20 @@ function fileEndpointCheck(done) {
skylinkVerification(done, linkInfo);
}
// check whether hns/note-to-self would properly redirect to note-to-self/
function hnsEndpointDirectoryRedirect(done) {
const expected = {
name: "hns endpoint directory redirect",
skylink: "hns/note-to-self",
statusCode: 307,
headers: {
location: "note-to-self/",
},
};
skylinkVerification(done, expected, { redirects: 0 });
}
function parseHeaderString(header) {
try {
return JSON.parse(header);
@ -1033,26 +1098,35 @@ function parseHeaderString(header) {
}
// skylinkVerification verifies a skylink against provided information.
function skylinkVerification(done, { name, skylink, bodyHash, headers }) {
function skylinkVerification(done, { name, skylink, bodyHash, headers, statusCode }, { redirects, method } = {}) {
const time = process.hrtime();
// Create the query for the skylink
const query = `${process.env.PORTAL_URL}/${skylink}?nocache=true`;
const query = `${process.env.PORTAL_URL}/${skylink}`;
// Get the Skylink
superagent
.get(query)
superagent[method || "get"](query)
.set("cookie", "nocache=true")
.redirects(redirects)
.ok((res) => (redirects === undefined ? res.ok : res.status < 400))
.responseType("blob")
.then(
(response) => {
const entry = { name, up: true, statusCode: response.statusCode, time: calculateElapsedTime(time) };
const info = {};
// Check if the response body is valid by checking against the known hash
const currentBodyHash = hash(response.body);
if (currentBodyHash !== bodyHash) {
if (statusCode && statusCode !== response.statusCode) {
entry.up = false;
info.bodyHash = { expected: bodyHash, current: currentBodyHash };
info.statusCode = { expected: statusCode, current: response.statusCode };
}
// Check if the response body is valid by checking against the known hash
if (bodyHash) {
const currentBodyHash = hash(response.body);
if (currentBodyHash !== bodyHash) {
entry.up = false;
info.bodyHash = { expected: bodyHash, current: currentBodyHash };
}
}
if (headers) {
@ -1065,7 +1139,7 @@ function skylinkVerification(done, { name, skylink, bodyHash, headers }) {
if (typeof currentHeader === "object") {
info.headers[headerName] = ensureValidJSON(detailedDiff(expectedHeader, currentHeader));
} else {
info.headers[headerName] = currentHeader;
info.headers[headerName] = { expected: expectedHeader, current: currentHeader };
}
}
});
@ -1076,6 +1150,8 @@ function skylinkVerification(done, { name, skylink, bodyHash, headers }) {
done(entry); // Return the entry information
},
(error) => {
console.log(error);
done({
name,
up: false,
@ -1109,6 +1185,8 @@ module.exports = [
skyGalleryIndexFileCheck,
skyGalleryRedirectCheck,
uncensoredLibraryCheck,
uncensoredLibraryPressReleaseCheck,
uncensoredLibraryV2Check,
fileEndpointCheck,
bitcoinWhitepaper,
// uniswapIndexFileCheck,
@ -1118,4 +1196,5 @@ module.exports = [
// uniswapHNSRedirectCheck,
uniswapHNSResolverCheck,
uniswapHNSResolverRedirectCheck,
hnsEndpointDirectoryRedirect,
];

36
packages/health-check/src/run.js
View File

@ -1,21 +1,25 @@
require('yargs/yargs')(process.argv.slice(2))
.command('$0 <type>', 'Skynet portal health checks', (yargs) => {
yargs.positional('type', {
describe: 'Type of checks to run',
type: 'string',
choices: ["critical", "verbose"]
})
.option("portal-url", {
require("yargs/yargs")(process.argv.slice(2)).command(
"$0 <type>",
"Skynet portal health checks",
(yargs) => {
yargs
.positional("type", {
describe: "Type of checks to run",
type: "string",
choices: ["critical", "verbose"],
})
.option("portal-url", {
describe: "Skynet portal url",
default: process.env.PORTAL_URL || "https://siasky.net",
type: "string",
})
.option("state-dir", {
})
.option("state-dir", {
describe: "State directory",
default: process.env.STATE_DIR || "state",
type: "string",
})
}, async ({ type, portalUrl, stateDir }) => {
});
},
async ({ type, portalUrl, stateDir }) => {
process.env.PORTAL_URL = portalUrl;
process.env.STATE_DIR = stateDir;
@ -23,11 +27,11 @@ require('yargs/yargs')(process.argv.slice(2))
const checks = require(`../src/checks/${type}`);
const entry = {
date: new Date().toISOString(),
checks: await Promise.all(checks.map((check) => new Promise(check))),
date: new Date().toISOString(),
checks: await Promise.all(checks.map((check) => new Promise(check))),
};
// read before writing to make sure no external changes are overwritten
db.read().get(type).push(entry).write();
})
.argv
}
).argv;

7
packages/webapp/cypress/integration/intergration.spec.js
View File

@ -12,12 +12,12 @@ context("Skynet", () => {
});
it("should be able to upload a file", () => {
cy.server();
cy.route("POST", "/skynet/skyfile").as("upload");
cy.intercept("POST", "/skynet/skyfile").as("upload");
const fileName = "check.json";
cy.get('.home-upload input[type="file"]').attachFile(fileName);
cy.wait(1000); // delay for drag-and-drop to work properly every time
cy.get('.home-upload input[type="file"]').attachFile(fileName, { subjectType: "drag-n-drop" });
cy.get(".home-upload").scrollIntoView();
cy.get(".home-uploaded-files").children().should("have.length", 1);
@ -32,7 +32,6 @@ context("Skynet", () => {
cy.contains("Copy Link").click();
cy.contains("Copied!").should("be.visible");
cy.contains("Copied!").should("not.be.visible");
});
});
});

2
packages/webapp/gatsby-config.js
View File

@ -18,8 +18,6 @@ module.exports = {
path: `${__dirname}/src/images`,
},
},
`gatsby-transformer-sharp`,
`gatsby-plugin-sharp`,
`gatsby-plugin-robots-txt`,
{
resolve: `gatsby-plugin-manifest`,

4
packages/webapp/package.json
View File

@ -16,9 +16,7 @@
"gatsby-plugin-remove-serviceworker": "1.0.0",
"gatsby-plugin-robots-txt": "1.5.5",
"gatsby-plugin-sass": "3.1.0",
"gatsby-plugin-sharp": "2.13.1",
"gatsby-source-filesystem": "2.10.0",
"gatsby-transformer-sharp": "2.11.0",
"http-status-codes": "2.1.4",
"js-cookie": "^2.2.1",
"jsonp": "0.2.1",
@ -34,7 +32,7 @@
"react-mailchimp-subscribe": "2.1.0",
"react-syntax-highlighter": "15.4.3",
"react-visibility-sensor": "5.1.1",
"skynet-js": "2.4.0"
"skynet-js": "3.0.0"
},
"devDependencies": {
"cypress": "6.3.0",

2
packages/webapp/src/components/CodeExamples/Code.js
View File

@ -14,7 +14,7 @@ const client = new SkynetClient();
async function example() {
try {
// upload
const skylink = await client.uploadFile(file);
const { skylink } = await client.uploadFile(file);
console.log(\`Upload successful, skylink: \${skylink}\`);
// download

2
packages/webapp/src/components/HomeUpload/HomeUpload.js
View File

@ -130,7 +130,7 @@ export default function HomeUpload() {
response = await client.uploadFile(file, { onUploadProgress });
}
onFileStateChange(file, { status: "complete", url: client.getSkylinkUrl(response) });
onFileStateChange(file, { status: "complete", url: client.getSkylinkUrl(response.skylink) });
} catch (error) {
if (error.response && error.response.status === StatusCodes.TOO_MANY_REQUESTS) {
onFileStateChange(file, { progress: -1 });

3283
yarn.lock
View File

File diff suppressed because it is too large Load Diff