Merge pull request #1727 from SkynetLabs/replace-api-env-vars-with-domains

replace api env vars with domain vars

docker-compose.accounts.yml

@@ -66,7 +66,6 @@ services:
     env_file:
       - .env
     environment:
-      - NEXT_PUBLIC_SKYNET_PORTAL_API=${SKYNET_PORTAL_API}
       - NEXT_PUBLIC_PORTAL_DOMAIN=${PORTAL_DOMAIN}
       - NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=${STRIPE_PUBLISHABLE_KEY}
     volumes:

docker/nginx/conf.d/include/location-hns

@@ -81,8 +81,8 @@ proxy_pass https://127.0.0.1/$skylink$path$is_args$args;
 
 # in case siad returns location header, we need to replace the skylink with the domain name
 header_filter_by_lua_block {
-    ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API")
+    ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN")
-    ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API")
+    ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN")
 
     if ngx.header.location then
         -- match location redirect part after the skylink

docker/nginx/conf.d/include/location-skylink

@@ -82,8 +82,8 @@ access_by_lua_block {
 }
 
 header_filter_by_lua_block {
-    ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API")
+    ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN")
-    ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API")
+    ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN")
 
     -- not empty skynet_proof means this is a skylink v2 request
     -- so we should replace the Skynet-Proof header with the one

docker/nginx/conf.d/server/server.api

@@ -248,8 +248,8 @@ location /skynet/tus {
     proxy_set_header X-Forwarded-Proto $scheme;
 
     # rewrite proxy request to use correct host uri from env variable (required to return correct location header)
-    set_by_lua $SKYNET_SERVER_API 'return os.getenv("SKYNET_SERVER_API")';
+    set_by_lua_block $server_domain { return os.getenv("SERVER_DOMAIN") }
-    proxy_redirect $scheme://$host $SKYNET_SERVER_API;
+    proxy_redirect $scheme://$host $scheme://$server_domain;
 
     # proxy /skynet/tus requests to siad endpoint with all arguments
     proxy_pass http://sia:9980;
@@ -276,8 +276,8 @@ location /skynet/tus {
 
     # extract skylink from base64 encoded upload metadata and assign to a proper header
     header_filter_by_lua_block {
-        ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API")
+        ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN")
-        ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API")
+        ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN")
 
         if ngx.header["Upload-Metadata"] then
             local encodedSkylink = string.match(ngx.header["Upload-Metadata"], "Skylink ([^,?]+)")
@@ -311,8 +311,8 @@ location /skynet/metadata {
     include /etc/nginx/conf.d/include/portal-access-check;
 
     header_filter_by_lua_block {
-        ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API")
+        ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN")
-        ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API")
+        ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN")
     }
 
     proxy_set_header User-Agent: Sia-Agent;
@@ -324,8 +324,8 @@ location /skynet/resolve {
     include /etc/nginx/conf.d/include/portal-access-check;
 
     header_filter_by_lua_block {
-        ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API")
+        ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN")
-        ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API")
+        ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN")
     }
 
     proxy_set_header User-Agent: Sia-Agent;

docker/nginx/nginx.conf

@@ -26,8 +26,8 @@ worker_processes auto;
 #pid        logs/nginx.pid;
 
 # declare env variables to use it in config
-env SKYNET_PORTAL_API;
+env PORTAL_DOMAIN;
-env SKYNET_SERVER_API;
+env SERVER_DOMAIN;
 env PORTAL_MODULES;
 env ACCOUNTS_LIMIT_ACCESS;
 env SIA_API_PASSWORD;
@@ -95,8 +95,8 @@ http {
 
     # include skynet-portal-api and skynet-server-api header on every request
     header_filter_by_lua_block {
-        ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API")
+        ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN")
-        ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API")
+        ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN")
     }
 
     # ratelimit specified IPs

packages/dashboard/src/components/Layout.js

@@ -82,7 +82,7 @@ export default function Layout({ title, children }) {
                         </a>
                       </Link>
                       <a
-                        href={process.env.NEXT_PUBLIC_SKYNET_PORTAL_API}
+                        href={`https://${process.env.NEXT_PUBLIC_PORTAL_DOMAIN}`}
                         className="text-gray-300 hover:bg-gray-700 hover:text-white px-3 py-2 rounded-md text-sm font-medium flex items-center"
                         target="_blank"
                         rel="noopener noreferrer"
@@ -234,7 +234,7 @@ export default function Layout({ title, children }) {
                 </a>
               </Link>
               <a
-                href={process.env.NEXT_PUBLIC_SKYNET_PORTAL_API}
+                href={`https://${process.env.NEXT_PUBLIC_PORTAL_DOMAIN}`}
                 className="text-gray-300 hover:bg-gray-700 hover:text-white block px-3 py-2 rounded-md text-base font-medium flex items-center"
                 target="_blank"
                 rel="noopener noreferrer"

packages/dashboard/src/pages/downloads.js

@@ -6,7 +6,7 @@ import Table from "../components/Table";
 import { SkynetClient } from "skynet-js";
 import useAccountsApi from "../services/useAccountsApi";
 
-const skynetClient = new SkynetClient(process.env.NEXT_PUBLIC_SKYNET_PORTAL_API);
+const skynetClient = new SkynetClient(`https://${process.env.NEXT_PUBLIC_PORTAL_DOMAIN}`);
 const getSkylinkLink = ({ skylink }) => skynetClient.getSkylinkUrl(skylink);
 const getRelativeDate = ({ downloadedOn }) => dayjs(downloadedOn).format("YYYY-MM-DD HH:mm:ss");
 const headers = [

packages/dashboard/src/pages/index.js

@@ -11,7 +11,7 @@ import React from "react";
 
 dayjs.extend(relativeTime);
 
-const skynetClient = new SkynetClient(process.env.NEXT_PUBLIC_SKYNET_PORTAL_API);
+const skynetClient = new SkynetClient(`https://${process.env.NEXT_PUBLIC_PORTAL_DOMAIN}`);
 
 function SkylinkList({ items = [], timestamp }) {
   return (

packages/dashboard/src/pages/uploads.js

@@ -8,7 +8,7 @@ import Table from "../components/Table";
 import { SkynetClient } from "skynet-js";
 import useAccountsApi from "../services/useAccountsApi";
 
-const skynetClient = new SkynetClient(process.env.NEXT_PUBLIC_SKYNET_PORTAL_API);
+const skynetClient = new SkynetClient(`https://${process.env.NEXT_PUBLIC_PORTAL_DOMAIN}`);
 const getSkylinkLink = ({ skylink }) => skynetClient.getSkylinkUrl(skylink);
 const getRelativeDate = ({ uploadedOn }) => dayjs(uploadedOn).format("YYYY-MM-DD HH:mm:ss");
 const headers = [

packages/health-check/bin/cli

@@ -48,7 +48,7 @@ require("yargs/yargs")(process.argv.slice(2))
         })
         .option("portal-url", {
           describe: "Skynet portal url",
-          default: process.env.SKYNET_PORTAL_API || "https://siasky.net",
+          default: process.env.PORTAL_DOMAIN ? `https://${process.env.PORTAL_DOMAIN}` : "https://siasky.net",
           type: "string",
         })
         .option("state-dir", {
@@ -58,7 +58,8 @@ require("yargs/yargs")(process.argv.slice(2))
         });
     },
     async ({ type, portalUrl, stateDir }) => {
-      process.env.SKYNET_PORTAL_API = portalUrl;
+      const { hostname: portalDomain } = new URL(portalUrl); // extract domain from portal url
+      process.env.PORTAL_DOMAIN = portalDomain;
       process.env.STATE_DIR = stateDir;
 
       const util = require("util");

packages/health-check/src/checks/critical.js

@@ -6,7 +6,7 @@ const { SkynetClient, stringToUint8ArrayUtf8, genKeyPairAndSeed } = require("sky
 
 const MODULE_BLOCKER = "b";
 
-const skynetClient = new SkynetClient(process.env.SKYNET_PORTAL_API);
+const skynetClient = new SkynetClient(`https://${process.env.PORTAL_DOMAIN}`);
 const exampleSkylink = "AACogzrAimYPG42tDOKhS3lXZD8YvlF8Q8R17afe95iV2Q";
 
 // check that any relevant configuration is properly set in skyd
@@ -45,7 +45,7 @@ async function uploadCheck(done) {
   form.append("file", payload, { filename: "time.txt", contentType: "text/plain" });
 
   try {
-    const response = await got.post(`${process.env.SKYNET_PORTAL_API}/skynet/skyfile`, {
+    const response = await got.post(`https://${process.env.PORTAL_DOMAIN}/skynet/skyfile`, {
       body: form,
       headers: { cookie: authCookie },
     });
@@ -65,7 +65,7 @@ async function uploadCheck(done) {
 
 // websiteCheck checks whether the main website is working
 async function websiteCheck(done) {
-  return done(await genericAccessCheck("website", process.env.SKYNET_PORTAL_API));
+  return done(await genericAccessCheck("website", `https://${process.env.PORTAL_DOMAIN}`));
 }
 
 // downloadCheck returns the result of downloading the hard coded link
@@ -130,13 +130,13 @@ async function registryWriteAndReadCheck(done) {
 
 // directServerApiAccessCheck returns the basic server api check on direct server address
 async function directServerApiAccessCheck(done) {
-  if (!process.env.SKYNET_SERVER_API) {
+  if (!process.env.SERVER_DOMAIN) {
-    return done({ up: false, errors: [{ message: "SKYNET_SERVER_API env variable not configured" }] });
+    return done({ up: false, errors: [{ message: "SERVER_DOMAIN env variable not configured" }] });
   }
 
   const [portalAccessCheck, serverAccessCheck] = await Promise.all([
-    genericAccessCheck("portal_api_access", process.env.SKYNET_PORTAL_API),
+    genericAccessCheck("portal_api_access", `https://${process.env.PORTAL_DOMAIN}`),
-    genericAccessCheck("server_api_access", process.env.SKYNET_SERVER_API),
+    genericAccessCheck("server_api_access", `https://${process.env.SERVER_DOMAIN}`),
   ]);
 
   if (portalAccessCheck.ip !== serverAccessCheck.ip) {
@@ -145,8 +145,8 @@ async function directServerApiAccessCheck(done) {
     serverAccessCheck.errors.push({
       message: "Access ip mismatch between portal and server access",
       response: {
-        portal: { name: process.env.SKYNET_PORTAL_API, ip: portalAccessCheck.ip },
+        portal: { name: process.env.PORTAL_DOMAIN, ip: portalAccessCheck.ip },
-        server: { name: process.env.SKYNET_SERVER_API, ip: serverAccessCheck.ip },
+        server: { name: process.env.SERVER_DOMAIN, ip: serverAccessCheck.ip },
       },
     });
   }

packages/health-check/src/checks/extended.js

@@ -1135,7 +1135,7 @@ async function skylinkVerification(done, expected, { followRedirect = true, meth
   const details = { name: expected.name, skylink: expected.skylink };
 
   try {
-    const query = `${process.env.SKYNET_PORTAL_API}/${expected.skylink}`;
+    const query = `https://${process.env.PORTAL_DOMAIN}/${expected.skylink}`;
     const response = await got[method](query, { followRedirect, headers: { cookie: `nocache=true;${authCookie}` } });
     const entry = { ...details, up: true, statusCode: response.statusCode, time: calculateElapsedTime(time) };
     const info = {};
@@ -1171,7 +1171,7 @@ async function skylinkVerification(done, expected, { followRedirect = true, meth
 
     if (expected.metadata && expected.skylink) {
       const skylink = parseSkylink(expected.skylink);
-      const url = `${process.env.SKYNET_PORTAL_API}/skynet/metadata/${skylink}`;
+      const url = `https://${process.env.PORTAL_DOMAIN}/skynet/metadata/${skylink}`;
       try {
         const metadata = await got(url, { headers: { cookie: `nocache=true;${authCookie}` } }).json();
         if (!isEqual(expected.metadata, metadata)) {

packages/health-check/src/index.js

@@ -1,9 +1,5 @@
 process.env.NODE_ENV = process.env.NODE_ENV || "development";
 
-if (!process.env.SKYNET_PORTAL_API) {
-  throw new Error("You need to provide SKYNET_PORTAL_API environment variable");
-}
-
 if (!process.env.PORTAL_DOMAIN) {
   throw new Error("You need to provide PORTAL_DOMAIN environment variable");
 }

setup-scripts/bot_utils.py

@@ -75,7 +75,7 @@ async def send_msg(msg, force_notify=False, file=None):
         webhook = DiscordWebhook(url=webhook_url, rate_limit_retry=True)
 
         # Add the portal name.
-        msg = "**{}**: {}".format(os.getenv("SKYNET_SERVER_API"), msg)
+        msg = "**{}**: {}".format(os.getenv("SERVER_DOMAIN"), msg)
 
         if file and isinstance(file, str):
             is_json = is_json_string(file)

setup-scripts/setup-docker-services.sh

@@ -23,8 +23,6 @@ docker-compose --version # sanity check
 # Create dummy .env file for docker-compose usage with variables
 # * PORTAL_DOMAIN - (required) is a skynet portal domain (ex. siasky.net)
 # * SERVER_DOMAIN - (optional) is an optional direct server domain (ex. eu-ger-1.siasky.net) - leave blank unless it is different than PORTAL_DOMAIN
-# * SKYNET_PORTAL_API - absolute url to the portal api ie. https://siasky.net (general portal address)
-# * SKYNET_SERVER_API - absolute url to the server api ie. https://eu-ger-1.siasky.net (direct server address, if this is single server portal use the same address as SKYNET_PORTAL_API)
 # * EMAIL_ADDRESS - this is the administrator contact email you need to supply for communication regarding SSL certification
 # * HSD_API_KEY - this is auto generated secure key for your handshake service integration
 # * CLOUDFLARE_AUTH_TOKEN - (optional) if using cloudflare as dns loadbalancer (need to change it in Caddyfile too)
@@ -43,7 +41,7 @@ docker-compose --version # sanity check
 # * COOKIE_ENC_KEY - (optional) if using `accounts` encryption key, at least 32 bytes
 if ! [ -f /home/user/skynet-webportal/.env ]; then
     HSD_API_KEY=$(openssl rand -base64 32) # generate safe random key for handshake
-    printf "PORTAL_DOMAIN=siasky.net\nSERVER_DOMAIN=\nSKYNET_PORTAL_API=https://siasky.net\nSKYNET_SERVER_API=https://eu-dc-1.siasky.net\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nDISCORD_WEBHOOK_URL=\nDISCORD_MENTION_USER_ID=\nDISCORD_MENTION_ROLE_ID=\n" > /home/user/skynet-webportal/.env
+    printf "PORTAL_DOMAIN=siasky.net\nSERVER_DOMAIN=\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nDISCORD_WEBHOOK_URL=\nDISCORD_MENTION_USER_ID=\nDISCORD_MENTION_ROLE_ID=\n" > /home/user/skynet-webportal/.env
 fi
 
 # Start docker container with nginx and client