From f7f03d183a2c7d7014165de0c10308655de3e9c8 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 16 Feb 2022 14:13:37 +0100 Subject: [PATCH 1/2] replace api env vars with domain vars --- docker-compose.accounts.yml | 1 - docker/nginx/conf.d/include/location-hns | 4 ++-- docker/nginx/conf.d/include/location-skylink | 4 ++-- docker/nginx/conf.d/server/server.api | 16 ++++++++-------- docker/nginx/nginx.conf | 8 ++++---- packages/dashboard/src/components/Layout.js | 4 ++-- packages/dashboard/src/pages/downloads.js | 2 +- packages/dashboard/src/pages/index.js | 2 +- packages/dashboard/src/pages/uploads.js | 2 +- packages/health-check/bin/cli | 5 +++-- packages/health-check/src/checks/critical.js | 18 +++++++++--------- packages/health-check/src/checks/extended.js | 4 ++-- packages/health-check/src/index.js | 4 ---- setup-scripts/bot_utils.py | 2 +- setup-scripts/setup-docker-services.sh | 4 +--- 15 files changed, 37 insertions(+), 43 deletions(-) diff --git a/docker-compose.accounts.yml b/docker-compose.accounts.yml index e03d679a..a3941f6b 100644 --- a/docker-compose.accounts.yml +++ b/docker-compose.accounts.yml @@ -66,7 +66,6 @@ services: env_file: - .env environment: - - NEXT_PUBLIC_SKYNET_PORTAL_API=${SKYNET_PORTAL_API} - NEXT_PUBLIC_PORTAL_DOMAIN=${PORTAL_DOMAIN} - NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=${STRIPE_PUBLISHABLE_KEY} volumes: diff --git a/docker/nginx/conf.d/include/location-hns b/docker/nginx/conf.d/include/location-hns index 22e50317..c8f08d2e 100644 --- a/docker/nginx/conf.d/include/location-hns +++ b/docker/nginx/conf.d/include/location-hns @@ -81,8 +81,8 @@ proxy_pass https://127.0.0.1/$skylink$path$is_args$args; # in case siad returns location header, we need to replace the skylink with the domain name header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") - ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") + ngx.header["Skynet-Portal-Api"] = "https://" .. os.getenv("PORTAL_DOMAIN") + ngx.header["Skynet-Server-Api"] = "https://" .. os.getenv("SERVER_DOMAIN") if ngx.header.location then -- match location redirect part after the skylink diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index b0d2066e..e26844c7 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -82,8 +82,8 @@ access_by_lua_block { } header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") - ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") + ngx.header["Skynet-Portal-Api"] = "https://" .. os.getenv("PORTAL_DOMAIN") + ngx.header["Skynet-Server-Api"] = "https://" .. os.getenv("SERVER_DOMAIN") -- not empty skynet_proof means this is a skylink v2 request -- so we should replace the Skynet-Proof header with the one diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index bcdd3705..f56661b0 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -248,8 +248,8 @@ location /skynet/tus { proxy_set_header X-Forwarded-Proto $scheme; # rewrite proxy request to use correct host uri from env variable (required to return correct location header) - set_by_lua $SKYNET_SERVER_API 'return os.getenv("SKYNET_SERVER_API")'; - proxy_redirect $scheme://$host $SKYNET_SERVER_API; + set_by_lua_block $server_domain { return os.getenv("SERVER_DOMAIN") } + proxy_redirect $scheme://$host $scheme://$server_domain; # proxy /skynet/tus requests to siad endpoint with all arguments proxy_pass http://sia:9980; @@ -276,8 +276,8 @@ location /skynet/tus { # extract skylink from base64 encoded upload metadata and assign to a proper header header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") - ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") + ngx.header["Skynet-Portal-Api"] = "https://" .. os.getenv("PORTAL_DOMAIN") + ngx.header["Skynet-Server-Api"] = "https://" .. os.getenv("SERVER_DOMAIN") if ngx.header["Upload-Metadata"] then local encodedSkylink = string.match(ngx.header["Upload-Metadata"], "Skylink ([^,?]+)") @@ -311,8 +311,8 @@ location /skynet/metadata { include /etc/nginx/conf.d/include/portal-access-check; header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") - ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") + ngx.header["Skynet-Portal-Api"] = "https://" .. os.getenv("PORTAL_DOMAIN") + ngx.header["Skynet-Server-Api"] = "https://" .. os.getenv("SERVER_DOMAIN") } proxy_set_header User-Agent: Sia-Agent; @@ -324,8 +324,8 @@ location /skynet/resolve { include /etc/nginx/conf.d/include/portal-access-check; header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") - ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") + ngx.header["Skynet-Portal-Api"] = "https://" .. os.getenv("PORTAL_DOMAIN") + ngx.header["Skynet-Server-Api"] = "https://" .. os.getenv("SERVER_DOMAIN") } proxy_set_header User-Agent: Sia-Agent; diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index 5d7e5944..795753fc 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -26,8 +26,8 @@ worker_processes auto; #pid logs/nginx.pid; # declare env variables to use it in config -env SKYNET_PORTAL_API; -env SKYNET_SERVER_API; +env PORTAL_DOMAIN; +env SERVER_DOMAIN; env PORTAL_MODULES; env ACCOUNTS_LIMIT_ACCESS; env SIA_API_PASSWORD; @@ -95,8 +95,8 @@ http { # include skynet-portal-api and skynet-server-api header on every request header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") - ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") + ngx.header["Skynet-Portal-Api"] = "https://" .. os.getenv("PORTAL_DOMAIN") + ngx.header["Skynet-Server-Api"] = "https://" .. os.getenv("SERVER_DOMAIN") } # ratelimit specified IPs diff --git a/packages/dashboard/src/components/Layout.js b/packages/dashboard/src/components/Layout.js index 848bfffa..f99ef767 100644 --- a/packages/dashboard/src/components/Layout.js +++ b/packages/dashboard/src/components/Layout.js @@ -82,7 +82,7 @@ export default function Layout({ title, children }) { skynetClient.getSkylinkUrl(skylink); const getRelativeDate = ({ downloadedOn }) => dayjs(downloadedOn).format("YYYY-MM-DD HH:mm:ss"); const headers = [ diff --git a/packages/dashboard/src/pages/index.js b/packages/dashboard/src/pages/index.js index cbb2e68b..23bd44bb 100644 --- a/packages/dashboard/src/pages/index.js +++ b/packages/dashboard/src/pages/index.js @@ -11,7 +11,7 @@ import React from "react"; dayjs.extend(relativeTime); -const skynetClient = new SkynetClient(process.env.NEXT_PUBLIC_SKYNET_PORTAL_API); +const skynetClient = new SkynetClient(`https://${process.env.NEXT_PUBLIC_PORTAL_DOMAIN}`); function SkylinkList({ items = [], timestamp }) { return ( diff --git a/packages/dashboard/src/pages/uploads.js b/packages/dashboard/src/pages/uploads.js index 4e2e2def..b9cfe3e5 100644 --- a/packages/dashboard/src/pages/uploads.js +++ b/packages/dashboard/src/pages/uploads.js @@ -8,7 +8,7 @@ import Table from "../components/Table"; import { SkynetClient } from "skynet-js"; import useAccountsApi from "../services/useAccountsApi"; -const skynetClient = new SkynetClient(process.env.NEXT_PUBLIC_SKYNET_PORTAL_API); +const skynetClient = new SkynetClient(`https://${process.env.NEXT_PUBLIC_PORTAL_DOMAIN}`); const getSkylinkLink = ({ skylink }) => skynetClient.getSkylinkUrl(skylink); const getRelativeDate = ({ uploadedOn }) => dayjs(uploadedOn).format("YYYY-MM-DD HH:mm:ss"); const headers = [ diff --git a/packages/health-check/bin/cli b/packages/health-check/bin/cli index dbf39344..21b6b9f7 100755 --- a/packages/health-check/bin/cli +++ b/packages/health-check/bin/cli @@ -48,7 +48,7 @@ require("yargs/yargs")(process.argv.slice(2)) }) .option("portal-url", { describe: "Skynet portal url", - default: process.env.SKYNET_PORTAL_API || "https://siasky.net", + default: process.env.PORTAL_DOMAIN ? `https://${process.env.PORTAL_DOMAIN}` : "https://siasky.net", type: "string", }) .option("state-dir", { @@ -58,7 +58,8 @@ require("yargs/yargs")(process.argv.slice(2)) }); }, async ({ type, portalUrl, stateDir }) => { - process.env.SKYNET_PORTAL_API = portalUrl; + const { hostname: portalDomain } = new URL(portalUrl); // extract domain from portal url + process.env.PORTAL_DOMAIN = portalDomain; process.env.STATE_DIR = stateDir; const util = require("util"); diff --git a/packages/health-check/src/checks/critical.js b/packages/health-check/src/checks/critical.js index ac406f10..546d9f88 100644 --- a/packages/health-check/src/checks/critical.js +++ b/packages/health-check/src/checks/critical.js @@ -6,7 +6,7 @@ const { SkynetClient, stringToUint8ArrayUtf8, genKeyPairAndSeed } = require("sky const MODULE_BLOCKER = "b"; -const skynetClient = new SkynetClient(process.env.SKYNET_PORTAL_API); +const skynetClient = new SkynetClient(`https://${process.env.PORTAL_DOMAIN}`); const exampleSkylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q"; // check that any relevant configuration is properly set in skyd @@ -45,7 +45,7 @@ async function uploadCheck(done) { form.append("file", payload, { filename: "time.txt", contentType: "text/plain" }); try { - const response = await got.post(`${process.env.SKYNET_PORTAL_API}/skynet/skyfile`, { + const response = await got.post(`https://${process.env.PORTAL_DOMAIN}/skynet/skyfile`, { body: form, headers: { cookie: authCookie }, }); @@ -65,7 +65,7 @@ async function uploadCheck(done) { // websiteCheck checks whether the main website is working async function websiteCheck(done) { - return done(await genericAccessCheck("website", process.env.SKYNET_PORTAL_API)); + return done(await genericAccessCheck("website", `https://${process.env.PORTAL_DOMAIN}`)); } // downloadCheck returns the result of downloading the hard coded link @@ -130,13 +130,13 @@ async function registryWriteAndReadCheck(done) { // directServerApiAccessCheck returns the basic server api check on direct server address async function directServerApiAccessCheck(done) { - if (!process.env.SKYNET_SERVER_API) { - return done({ up: false, errors: [{ message: "SKYNET_SERVER_API env variable not configured" }] }); + if (!process.env.SERVER_DOMAIN) { + return done({ up: false, errors: [{ message: "SERVER_DOMAIN env variable not configured" }] }); } const [portalAccessCheck, serverAccessCheck] = await Promise.all([ - genericAccessCheck("portal_api_access", process.env.SKYNET_PORTAL_API), - genericAccessCheck("server_api_access", process.env.SKYNET_SERVER_API), + genericAccessCheck("portal_api_access", `https://${process.env.PORTAL_DOMAIN}`), + genericAccessCheck("server_api_access", `https://${process.env.SERVER_DOMAIN}`), ]); if (portalAccessCheck.ip !== serverAccessCheck.ip) { @@ -145,8 +145,8 @@ async function directServerApiAccessCheck(done) { serverAccessCheck.errors.push({ message: "Access ip mismatch between portal and server access", response: { - portal: { name: process.env.SKYNET_PORTAL_API, ip: portalAccessCheck.ip }, - server: { name: process.env.SKYNET_SERVER_API, ip: serverAccessCheck.ip }, + portal: { name: process.env.PORTAL_DOMAIN, ip: portalAccessCheck.ip }, + server: { name: process.env.SERVER_DOMAIN, ip: serverAccessCheck.ip }, }, }); } diff --git a/packages/health-check/src/checks/extended.js b/packages/health-check/src/checks/extended.js index 170bd9d1..aaf74389 100644 --- a/packages/health-check/src/checks/extended.js +++ b/packages/health-check/src/checks/extended.js @@ -1135,7 +1135,7 @@ async function skylinkVerification(done, expected, { followRedirect = true, meth const details = { name: expected.name, skylink: expected.skylink }; try { - const query = `${process.env.SKYNET_PORTAL_API}/${expected.skylink}`; + const query = `https://${process.env.PORTAL_DOMAIN}/${expected.skylink}`; const response = await got[method](query, { followRedirect, headers: { cookie: `nocache=true;${authCookie}` } }); const entry = { ...details, up: true, statusCode: response.statusCode, time: calculateElapsedTime(time) }; const info = {}; @@ -1171,7 +1171,7 @@ async function skylinkVerification(done, expected, { followRedirect = true, meth if (expected.metadata && expected.skylink) { const skylink = parseSkylink(expected.skylink); - const url = `${process.env.SKYNET_PORTAL_API}/skynet/metadata/${skylink}`; + const url = `https://${process.env.PORTAL_DOMAIN}/skynet/metadata/${skylink}`; try { const metadata = await got(url, { headers: { cookie: `nocache=true;${authCookie}` } }).json(); if (!isEqual(expected.metadata, metadata)) { diff --git a/packages/health-check/src/index.js b/packages/health-check/src/index.js index abaa7a59..f3369783 100644 --- a/packages/health-check/src/index.js +++ b/packages/health-check/src/index.js @@ -1,9 +1,5 @@ process.env.NODE_ENV = process.env.NODE_ENV || "development"; -if (!process.env.SKYNET_PORTAL_API) { - throw new Error("You need to provide SKYNET_PORTAL_API environment variable"); -} - if (!process.env.PORTAL_DOMAIN) { throw new Error("You need to provide PORTAL_DOMAIN environment variable"); } diff --git a/setup-scripts/bot_utils.py b/setup-scripts/bot_utils.py index 5312bbee..23cc64c0 100644 --- a/setup-scripts/bot_utils.py +++ b/setup-scripts/bot_utils.py @@ -75,7 +75,7 @@ async def send_msg(msg, force_notify=False, file=None): webhook = DiscordWebhook(url=webhook_url, rate_limit_retry=True) # Add the portal name. - msg = "**{}**: {}".format(os.getenv("SKYNET_SERVER_API"), msg) + msg = "**{}**: {}".format(os.getenv("SERVER_DOMAIN"), msg) if file and isinstance(file, str): is_json = is_json_string(file) diff --git a/setup-scripts/setup-docker-services.sh b/setup-scripts/setup-docker-services.sh index 2bd8c5ae..76c203be 100755 --- a/setup-scripts/setup-docker-services.sh +++ b/setup-scripts/setup-docker-services.sh @@ -23,8 +23,6 @@ docker-compose --version # sanity check # Create dummy .env file for docker-compose usage with variables # * PORTAL_DOMAIN - (required) is a skynet portal domain (ex. siasky.net) # * SERVER_DOMAIN - (optional) is an optional direct server domain (ex. eu-ger-1.siasky.net) - leave blank unless it is different than PORTAL_DOMAIN -# * SKYNET_PORTAL_API - absolute url to the portal api ie. https://siasky.net (general portal address) -# * SKYNET_SERVER_API - absolute url to the server api ie. https://eu-ger-1.siasky.net (direct server address, if this is single server portal use the same address as SKYNET_PORTAL_API) # * EMAIL_ADDRESS - this is the administrator contact email you need to supply for communication regarding SSL certification # * HSD_API_KEY - this is auto generated secure key for your handshake service integration # * CLOUDFLARE_AUTH_TOKEN - (optional) if using cloudflare as dns loadbalancer (need to change it in Caddyfile too) @@ -43,7 +41,7 @@ docker-compose --version # sanity check # * COOKIE_ENC_KEY - (optional) if using `accounts` encryption key, at least 32 bytes if ! [ -f /home/user/skynet-webportal/.env ]; then HSD_API_KEY=$(openssl rand -base64 32) # generate safe random key for handshake - printf "PORTAL_DOMAIN=siasky.net\nSERVER_DOMAIN=\nSKYNET_PORTAL_API=https://siasky.net\nSKYNET_SERVER_API=https://eu-dc-1.siasky.net\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nDISCORD_WEBHOOK_URL=\nDISCORD_MENTION_USER_ID=\nDISCORD_MENTION_ROLE_ID=\n" > /home/user/skynet-webportal/.env + printf "PORTAL_DOMAIN=siasky.net\nSERVER_DOMAIN=\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nDISCORD_WEBHOOK_URL=\nDISCORD_MENTION_USER_ID=\nDISCORD_MENTION_ROLE_ID=\n" > /home/user/skynet-webportal/.env fi # Start docker container with nginx and client From 8a536cf43cecdda07d5d80c82865443cee323834 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 16 Feb 2022 15:09:33 +0100 Subject: [PATCH 2/2] use ngx.var.scheme instead of https --- docker/nginx/conf.d/include/location-hns | 4 ++-- docker/nginx/conf.d/include/location-skylink | 4 ++-- docker/nginx/conf.d/server/server.api | 12 ++++++------ docker/nginx/nginx.conf | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docker/nginx/conf.d/include/location-hns b/docker/nginx/conf.d/include/location-hns index c8f08d2e..b0c7322d 100644 --- a/docker/nginx/conf.d/include/location-hns +++ b/docker/nginx/conf.d/include/location-hns @@ -81,8 +81,8 @@ proxy_pass https://127.0.0.1/$skylink$path$is_args$args; # in case siad returns location header, we need to replace the skylink with the domain name header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = "https://" .. os.getenv("PORTAL_DOMAIN") - ngx.header["Skynet-Server-Api"] = "https://" .. os.getenv("SERVER_DOMAIN") + ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN") + ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN") if ngx.header.location then -- match location redirect part after the skylink diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index e26844c7..cf250cea 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -82,8 +82,8 @@ access_by_lua_block { } header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = "https://" .. os.getenv("PORTAL_DOMAIN") - ngx.header["Skynet-Server-Api"] = "https://" .. os.getenv("SERVER_DOMAIN") + ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN") + ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN") -- not empty skynet_proof means this is a skylink v2 request -- so we should replace the Skynet-Proof header with the one diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index f56661b0..15ee50e1 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -276,8 +276,8 @@ location /skynet/tus { # extract skylink from base64 encoded upload metadata and assign to a proper header header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = "https://" .. os.getenv("PORTAL_DOMAIN") - ngx.header["Skynet-Server-Api"] = "https://" .. os.getenv("SERVER_DOMAIN") + ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN") + ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN") if ngx.header["Upload-Metadata"] then local encodedSkylink = string.match(ngx.header["Upload-Metadata"], "Skylink ([^,?]+)") @@ -311,8 +311,8 @@ location /skynet/metadata { include /etc/nginx/conf.d/include/portal-access-check; header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = "https://" .. os.getenv("PORTAL_DOMAIN") - ngx.header["Skynet-Server-Api"] = "https://" .. os.getenv("SERVER_DOMAIN") + ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN") + ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN") } proxy_set_header User-Agent: Sia-Agent; @@ -324,8 +324,8 @@ location /skynet/resolve { include /etc/nginx/conf.d/include/portal-access-check; header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = "https://" .. os.getenv("PORTAL_DOMAIN") - ngx.header["Skynet-Server-Api"] = "https://" .. os.getenv("SERVER_DOMAIN") + ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN") + ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN") } proxy_set_header User-Agent: Sia-Agent; diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index 795753fc..f67438e5 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -95,8 +95,8 @@ http { # include skynet-portal-api and skynet-server-api header on every request header_filter_by_lua_block { - ngx.header["Skynet-Portal-Api"] = "https://" .. os.getenv("PORTAL_DOMAIN") - ngx.header["Skynet-Server-Api"] = "https://" .. os.getenv("SERVER_DOMAIN") + ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN") + ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN") } # ratelimit specified IPs