LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #889 from SkynetLabs/support-tus-with-multiple-servers-setup 

Support tus with multiple servers setup
This commit is contained in:
Karol Wypchło 2021-06-17 14:06:40 +02:00 committed by GitHub
parent 9b2285e86a 34eb822cf8
commit fbdae038df
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 60 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docker/nginx/conf.d/client.conf
View File

@ -330,12 +330,13 @@ server {
proxy_request_buffering off; # stream uploaded files through the proxy as it comes in
proxy_set_header Expect $http_expect;
# rewrite proxy request to use correct host uri from env variable (required to return correct location header)
set_by_lua $SKYNET_SERVER_API 'return os.getenv("SKYNET_SERVER_API")';
proxy_redirect https://siad $SKYNET_SERVER_API;
# proxy /skynet/tus requests to siad endpoint with all arguments
proxy_pass http://siad;
# rewrite tus headers to use correct uri
proxy_redirect https://siad/ https://$domain.$tld/;
# extract skylink from base64 encoded upload metadata and assign to a proper header
header_filter_by_lua_block {
if ngx.header["Upload-Metadata"] then

2
docker/nginx/conf.d/include/cors
View File

@ -13,4 +13,4 @@ more_set_headers 'Access-Control-Allow-Origin: $http_origin';
more_set_headers 'Access-Control-Allow-Credentials: true';
more_set_headers 'Access-Control-Allow-Methods: GET, POST, HEAD, OPTIONS, PUT, PATCH, DELETE';
more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,X-HTTP-Method-Override,upload-offset,upload-metadata,upload-length,tus-version,tus-resumable,tus-extension,tus-max-size,location';
more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range,Skynet-File-Metadata,Skynet-Skylink,Skynet-Portal-Api,upload-offset,upload-metadata,upload-length,tus-version,tus-resumable,tus-extension,tus-max-size,location';
more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range,Skynet-File-Metadata,Skynet-Skylink,Skynet-Portal-Api,Skynet-Server-Api,upload-offset,upload-metadata,upload-length,tus-version,tus-resumable,tus-extension,tus-max-size,location';

13
docker/nginx/nginx.conf
View File

@ -25,8 +25,10 @@ worker_processes 1;
#pid logs/nginx.pid;
env SKYNET_PORTAL_API; # declare env variable to use it in config
env ACCOUNTS_ENABLED; # declare env variable to use it in config
# declare env variables to use it in config
env SKYNET_PORTAL_API;
env SKYNET_SERVER_API;
env ACCOUNTS_ENABLED;
events {
worker_connections 1024;
@ -64,8 +66,11 @@ http {
#gzip on;
# include skynet-portal-api header on every request
header_filter_by_lua 'ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API")';
# include skynet-portal-api and skynet-server-api header on every request
header_filter_by_lua_block {
ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API")
ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API")
}
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/conf.extra.d/*.conf;

48
packages/health-check/src/checks/critical.js
View File

@ -33,35 +33,60 @@ async function uploadCheck(done) {
// websiteCheck checks whether the main website is working
async function websiteCheck(done) {
return genericAccessCheck("website", process.env.SKYNET_PORTAL_API, done);
return done(await genericAccessCheck("website", process.env.SKYNET_PORTAL_API));
}
// downloadCheck returns the result of downloading the hard coded link
async function downloadCheck(done) {
const url = await skynetClient.getSkylinkUrl(exampleSkylink);
return genericAccessCheck("skylink", url, done);
return done(await genericAccessCheck("skylink", url));
}
// skylinkSubdomainCheck returns the result of downloading the hard coded link via subdomain
async function skylinkSubdomainCheck(done) {
const url = await skynetClient.getSkylinkUrl(exampleSkylink, { subdomain: true });
return genericAccessCheck("skylink_via_subdomain", url, done);
return done(await genericAccessCheck("skylink_via_subdomain", url));
}
// handshakeSubdomainCheck returns the result of downloading the skylink via handshake domain
async function handshakeSubdomainCheck(done) {
const url = await skynetClient.getHnsUrl("note-to-self", { subdomain: true });
return genericAccessCheck("hns_via_subdomain", url, done);
return done(await genericAccessCheck("hns_via_subdomain", url));
}
// accountWebsiteCheck returns the result of accessing account dashboard website
async function accountWebsiteCheck(done) {
const url = `${process.env.SKYNET_DASHBOARD_URL}/auth/login`;
return genericAccessCheck("account_website", url, done);
return done(await genericAccessCheck("account_website", url));
}
// directServerApiAccessCheck returns the basic server api check on direct server address
async function directServerApiAccessCheck(done) {
if (!process.env.SKYNET_SERVER_API) {
return done({ up: false, info: { message: "SKYNET_SERVER_API env variable not configured" } });
}
const [domainAccessCheck, directAccessCheck] = await Promise.all([
genericAccessCheck("portal_api_access", process.env.SKYNET_PORTAL_API),
genericAccessCheck("direct_server_api_access", process.env.SKYNET_SERVER_API),
]);
if (domainAccessCheck.ip !== directAccessCheck.ip) {
directAccessCheck.up = false;
directAccessCheck.info = {
message: "Access ip mismatch between domain and direct access",
response: {
domain: { name: process.env.SKYNET_PORTAL_API, ip: domainAccessCheck.ip },
domain: { name: process.env.SKYNET_SERVER_API, ip: directAccessCheck.ip },
},
};
}
return done(directAccessCheck);
}
// accountHealthCheck returns the result of accounts service health checks
@ -86,7 +111,7 @@ async function accountHealthCheck(done) {
done({ name: "accounts", time: calculateElapsedTime(time), ...data });
}
async function genericAccessCheck(name, url, done) {
async function genericAccessCheck(name, url) {
const time = process.hrtime();
const data = { up: false, url };
@ -103,10 +128,17 @@ async function genericAccessCheck(name, url, done) {
data.ip = error?.response?.ip ?? null;
}
done({ name, time: calculateElapsedTime(time), ...data });
return { name, time: calculateElapsedTime(time), ...data };
}
const checks = [uploadCheck, websiteCheck, downloadCheck, skylinkSubdomainCheck, handshakeSubdomainCheck];
const checks = [
uploadCheck,
websiteCheck,
downloadCheck,
skylinkSubdomainCheck,
handshakeSubdomainCheck,
directServerApiAccessCheck,
];
if (process.env.ACCOUNTS_ENABLED === "1") {
checks.push(accountHealthCheck, accountWebsiteCheck);

2
setup-scripts/bot_utils.py
View File

@ -53,7 +53,7 @@ def setup():
bot_token = os.environ["DISCORD_BOT_TOKEN"]
global portal_name
portal_name = os.getenv("PORTAL_NAME")
portal_name = os.getenv("SKYNET_SERVER_API")
# Get a port or use default
global port

9
setup-scripts/setup-docker-services.sh
View File

@ -22,15 +22,16 @@ docker-compose --version # sanity check
# Create dummy .env file for docker-compose usage with variables
# * SSL_CERTIFICATE_STRING - certificate string that will be used to generate ssl certificates, read more in docker/caddy/Caddyfile
# * SKYNET_PORTAL_API - absolute url to the portal api ie. https://example.com
# * SKYNET_DASHBOARD_URL - (optional) absolute url to the portal dashboard ie. https://account.example.com
# * SKYNET_PORTAL_API - absolute url to the portal api ie. https://siasky.net (general portal address)
# * SKYNET_SERVER_API - absolute url to the server api ie. https://eu-ger-1.siasky.net (direct server address, if this is single server portal use the same address as SKYNET_PORTAL_API)
# * SKYNET_DASHBOARD_URL - (optional) absolute url to the portal dashboard ie. https://account.siasky.net
# * EMAIL_ADDRESS - this is the administrator contact email you need to supply for communication regarding SSL certification
# * HSD_API_KEY - this is auto generated secure key for your handshake service integration
# * CLOUDFLARE_AUTH_TOKEN - (optional) if using cloudflare as dns loadbalancer (need to change it in Caddyfile too)
# * AWS_ACCESS_KEY_ID - (optional) if using route53 as a dns loadbalancer
# * AWS_SECRET_ACCESS_KEY - (optional) if using route53 as a dns loadbalancer
# * API_PORT - (optional) the port on which siad is listening, defaults to 9980
# * PORTAL_NAME - the name of the portal, required by the discord bot
# * PORTAL_NAME - a string representing name of your portal e.g. `siasky.xyz` or `my skynet portal` (internal use only)
# * DISCORD_BOT_TOKEN - (optional) only required if you're using the discord notifications integration
# * SKYNET_DB_USER - (optional) if using `accounts` this is the MongoDB username
# * SKYNET_DB_PASS - (optional) if using `accounts` this is the MongoDB password
@ -43,7 +44,7 @@ docker-compose --version # sanity check
# * CR_CLUSTER_NODES - (optional) if using `accounts` the list of servers (with ports) which make up your CockroachDB cluster, e.g. `helsinki.siasky.net:26257,germany.siasky.net:26257,us-east.siasky.net:26257`
if ! [ -f /home/user/skynet-webportal/.env ]; then
HSD_API_KEY=$(openssl rand -base64 32) # generate safe random key for handshake
printf "SSL_CERTIFICATE_STRING=example.com, *.example.com, *.hns.example.com\nSKYNET_PORTAL_API=https://example.com\nSKYNET_DASHBOARD_URL=https://account.example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nPORTAL_NAME=\nDISCORD_BOT_TOKEN=\n" > /home/user/skynet-webportal/.env
printf "SSL_CERTIFICATE_STRING=siasky.net, *.siasky.net, *.hns.siasky.net\nSKYNET_PORTAL_API=https://siasky.net\nSKYNET_SERVER_API=https://eu-dc-1.siasky.net\nSKYNET_DASHBOARD_URL=https://account.example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nPORTAL_NAME=\nDISCORD_BOT_TOKEN=\n" > /home/user/skynet-webportal/.env
fi
# Start docker container with nginx and client