diff --git a/docker-compose.accounts.yml b/docker-compose.accounts.yml index 0689ebc5..3ebad669 100644 --- a/docker-compose.accounts.yml +++ b/docker-compose.accounts.yml @@ -1,7 +1,6 @@ version: "3.7" -x-logging: - &default-logging +x-logging: &default-logging driver: json-file options: max-size: "10m" @@ -11,6 +10,7 @@ services: nginx: environment: - ACCOUNTS_ENABLED=true + - ACCOUNTS_LIMIT_ACCESS=${ACCOUNTS_LIMIT_ACCESS:-authenticated} # default to authenticated access only depends_on: - accounts diff --git a/docker-compose.yml b/docker-compose.yml index 13844136..e63df946 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,10 +25,10 @@ services: logging: *default-logging environment: - SIA_MODULES=gctwra - - SKYD_DISK_CACHE_ENABLED=false - - SKYD_DISK_CACHE_SIZE=53690000000 # 50GB - - SKYD_DISK_CACHE_MIN_HITS=3 - - SKYD_DISK_CACHE_HIT_PERIOD=3600 # 1h + - SKYD_DISK_CACHE_ENABLED=${SKYD_DISK_CACHE_ENABLED:-false} + - SKYD_DISK_CACHE_SIZE=${SKYD_DISK_CACHE_SIZE:-53690000000} # 50GB + - SKYD_DISK_CACHE_MIN_HITS=${SKYD_DISK_CACHE_MIN_HITS:-3} + - SKYD_DISK_CACHE_HIT_PERIOD=${SKYD_DISK_CACHE_HIT_PERIOD:-3600} # 1h env_file: - .env @@ -66,7 +66,6 @@ services: env_file: - .env volumes: - - ./docker/nginx/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf:ro - ./docker/data/nginx/cache:/data/nginx/cache - ./docker/data/nginx/blocker:/data/nginx/blocker - ./docker/data/nginx/logs:/usr/local/openresty/nginx/logs diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index 0fd1fc8b..43c4b7f9 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -11,6 +11,7 @@ COPY mo ./ COPY libs /etc/nginx/libs COPY conf.d /etc/nginx/conf.d COPY conf.d.templates /etc/nginx/conf.d.templates +COPY nginx.conf /usr/local/openresty/nginx/conf/nginx.conf CMD [ "bash", "-c", \ "./mo < /etc/nginx/conf.d.templates/server.account.conf > /etc/nginx/conf.d/server.account.conf ; \ diff --git a/docker/nginx/conf.d/include/init-optional-variables b/docker/nginx/conf.d/include/init-optional-variables index e2e74030..e072c6e6 100644 --- a/docker/nginx/conf.d/include/init-optional-variables +++ b/docker/nginx/conf.d/include/init-optional-variables @@ -3,10 +3,16 @@ # because otherwise logger with throw error # set only on hns routes -set $hns_domain ''; +set $hns_domain ""; # set only if server has been access through SERVER_DOMAIN -set $server_alias ''; +set $server_alias ""; # expose skylink variable so we can use it in access log -set $skylink ''; +set $skylink ""; + +# cached account limits (json string) - applies only if accounts are enabled +set $account_limits ""; + +# set this internal flag to true if current request should not be limited in any way +set $internal_no_limits "false"; diff --git a/docker/nginx/conf.d/include/location-hns b/docker/nginx/conf.d/include/location-hns index 8cc662da..22e50317 100644 --- a/docker/nginx/conf.d/include/location-hns +++ b/docker/nginx/conf.d/include/location-hns @@ -1,11 +1,12 @@ include /etc/nginx/conf.d/include/proxy-buffer; include /etc/nginx/conf.d/include/proxy-pass-internal; +include /etc/nginx/conf.d/include/portal-access-check; # variable definititions - we need to define a variable to be able to access it in lua by ngx.var.something set $skylink ''; # placeholder for the raw 46 bit skylink # resolve handshake domain by requesting to /hnsres endpoint and assign correct values to $skylink and $rest -access_by_lua_block { +rewrite_by_lua_block { local json = require('cjson') local httpc = require("resty.http").new() diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index 1a80f41f..7fb3d593 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -53,10 +53,7 @@ access_by_lua_block { -- check if skylink v1 is present on blocklist (compare hashes) if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then - ngx.status = ngx.HTTP_ILLEGAL - ngx.header["content-type"] = "text/plain" - ngx.say("Unavailable For Legal Reasons") - return ngx.exit(ngx.status) + return require("skynet.blocklist").exit_illegal() end -- if skylink is found on nocache list then set internal nocache variable @@ -65,21 +62,16 @@ access_by_lua_block { ngx.var.nocache = "1" end - -- this block runs only when accounts are enabled - if not os.getenv("PORTAL_MODULES"):match("a") then return end + if require("skynet.account").accounts_enabled() then + -- check if portal is in authenticated only mode + if require("skynet.account").is_access_unauthorized() then + return require("skynet.account").exit_access_unauthorized() + end - -- 10.10.10.70 points to accounts service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.70:3000/user/limits", { - headers = { ["Cookie"] = "skynet-jwt=" .. ngx.var.skynet_jwt } - }) - - -- fail gracefully in case /user/limits failed - if err or (res and res.status ~= ngx.HTTP_OK) then - ngx.log(ngx.ERR, "Failed accounts service request /user/limits: ", err or ("[HTTP " .. res.status .. "] " .. res.body)) - ngx.var.limit_rate = 2621440 -- (20 * 1024 * 1024 / 8) conservative fallback to 20 mbps in case accounts failed to return limits - elseif res and res.status == ngx.HTTP_OK then - local json = require('cjson') - local limits = json.decode(res.body) + -- get account limits of currently authenticated user + local limits = require("skynet.account").get_account_limits() + + -- apply download speed limit ngx.var.limit_rate = limits.download end } @@ -97,7 +89,8 @@ header_filter_by_lua_block { end -- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold - if tonumber(ngx.header["Content-Length"]) > 1e+9 then + -- (content length can be nil for already cached files - we can ignore them) + if ngx.header["Content-Length"] and tonumber(ngx.header["Content-Length"]) > 1e+9 then ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"]) end } diff --git a/docker/nginx/conf.d/include/location-skynet-registry b/docker/nginx/conf.d/include/location-skynet-registry index c24ed71a..27a94d29 100644 --- a/docker/nginx/conf.d/include/location-skynet-registry +++ b/docker/nginx/conf.d/include/location-skynet-registry @@ -10,22 +10,16 @@ proxy_read_timeout 600; # siad should timeout with 404 after 5 minutes proxy_pass http://sia:9980/skynet/registry; access_by_lua_block { - -- this block runs only when accounts are enabled - if not os.getenv("PORTAL_MODULES"):match("a") then return end + if require("skynet.account").accounts_enabled() then + -- check if portal is in authenticated only mode + if require("skynet.account").is_access_unauthorized() then + return require("skynet.account").exit_access_unauthorized() + end - local httpc = require("resty.http").new() - - -- 10.10.10.70 points to accounts service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.70:3000/user/limits", { - headers = { ["Cookie"] = "skynet-jwt=" .. ngx.var.skynet_jwt } - }) - - -- fail gracefully in case /user/limits failed - if err or (res and res.status ~= ngx.HTTP_OK) then - ngx.log(ngx.ERR, "Failed accounts service request /user/limits: ", err or ("[HTTP " .. res.status .. "] " .. res.body)) - elseif res and res.status == ngx.HTTP_OK then - local json = require('cjson') - local limits = json.decode(res.body) + -- get account limits of currently authenticated user + local limits = require("skynet.account").get_account_limits() + + -- apply registry rate limits (forced delay) if limits.registry > 0 then ngx.sleep(limits.registry / 1000) end diff --git a/docker/nginx/conf.d/include/portal-access-check b/docker/nginx/conf.d/include/portal-access-check new file mode 100644 index 00000000..5ce1e986 --- /dev/null +++ b/docker/nginx/conf.d/include/portal-access-check @@ -0,0 +1,6 @@ +access_by_lua_block { + -- check portal access rules and exit if access is restricted + if require("skynet.account").is_access_unauthorized() then + return require("skynet.account").exit_access_unauthorized() + end +} diff --git a/docker/nginx/conf.d/include/track-download b/docker/nginx/conf.d/include/track-download index 88aaa435..606c98ad 100644 --- a/docker/nginx/conf.d/include/track-download +++ b/docker/nginx/conf.d/include/track-download @@ -1,7 +1,7 @@ # register the download in accounts service (cookies should contain jwt) log_by_lua_block { -- this block runs only when accounts are enabled - if os.getenv("PORTAL_MODULES"):match("a") then + if require("skynet.account").accounts_enabled() then local function track(premature, skylink, status, body_bytes_sent, jwt) if premature then return end diff --git a/docker/nginx/conf.d/include/track-registry b/docker/nginx/conf.d/include/track-registry index 5461ee53..8c69172b 100644 --- a/docker/nginx/conf.d/include/track-registry +++ b/docker/nginx/conf.d/include/track-registry @@ -1,7 +1,7 @@ # register the registry access in accounts service (cookies should contain jwt) log_by_lua_block { -- this block runs only when accounts are enabled - if os.getenv("PORTAL_MODULES"):match("a") then + if require("skynet.account").accounts_enabled() then local function track(premature, request_method, jwt) if premature then return end diff --git a/docker/nginx/conf.d/include/track-upload b/docker/nginx/conf.d/include/track-upload index 50d832e4..340dd437 100644 --- a/docker/nginx/conf.d/include/track-upload +++ b/docker/nginx/conf.d/include/track-upload @@ -1,7 +1,7 @@ # register the upload in accounts service (cookies should contain jwt) log_by_lua_block { -- this block runs only when accounts are enabled - if os.getenv("PORTAL_MODULES"):match("a") then + if require("skynet.account").accounts_enabled() then local function track(premature, skylink, jwt) if premature then return end diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index c8eea8e2..8d68043a 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -28,6 +28,7 @@ location / { set $skylink "0404dsjvti046fsua4ktor9grrpe76erq9jot9cvopbhsvsu76r4r30"; set $path $uri; + set $internal_no_limits "true"; include /etc/nginx/conf.d/include/location-skylink; @@ -117,6 +118,8 @@ location /abuse/report { } location /hns { + include /etc/nginx/conf.d/include/cors; + # match the request_uri and extract the hns domain and anything that is passed in the uri after it # example: /hns/something/foo/bar matches: # > hns_domain: something @@ -130,6 +133,7 @@ location /hns { location /hnsres { include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/portal-access-check; proxy_pass http://handshake-api:3100; } @@ -141,6 +145,7 @@ location /skynet/registry { location /skynet/restore { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; + include /etc/nginx/conf.d/include/portal-access-check; client_max_body_size 5M; @@ -197,6 +202,7 @@ location /skynet/skyfile { include /etc/nginx/conf.d/include/sia-auth; include /etc/nginx/conf.d/include/track-upload; include /etc/nginx/conf.d/include/generate-siapath; + include /etc/nginx/conf.d/include/portal-access-check; limit_req zone=uploads_by_ip burst=10 nodelay; limit_req zone=uploads_by_ip_throttled; @@ -214,19 +220,6 @@ location /skynet/skyfile { proxy_set_header Expect $http_expect; proxy_set_header User-Agent: Sia-Agent; - # access_by_lua_block { - # -- this block runs only when accounts are enabled - # if not os.getenv("PORTAL_MODULES"):match("a") then return end - - # ngx.var.upload_limit_rate = 5 * 1024 * 1024 - # local res = ngx.location.capture("/accounts/user", { copy_all_vars = true }) - # if res.status == ngx.HTTP_OK then - # local json = require('cjson') - # local user = json.decode(res.body) - # ngx.var.upload_limit_rate = ngx.var.upload_limit_rate * (user.tier + 1) - # end - # } - # proxy this call to siad endpoint (make sure the ip is correct) proxy_pass http://sia:9980/skynet/skyfile/$dir1/$dir2/$dir3$is_args$args; } @@ -261,27 +254,17 @@ location /skynet/tus { # proxy /skynet/tus requests to siad endpoint with all arguments proxy_pass http://sia:9980; - # set max upload size dynamically based on account limits - rewrite_by_lua_block { - -- set default limit value to 5 GB - ngx.req.set_header("SkynetMaxUploadSize", 5368709120) - - -- this block runs only when accounts are enabled - if not os.getenv("PORTAL_MODULES"):match("a") then return end - - local httpc = require("resty.http").new() - - -- fetch account limits and set max upload size accordingly - local res, err = httpc:request_uri("http://10.10.10.70:3000/user/limits", { - headers = { ["Cookie"] = "skynet-jwt=" .. ngx.var.skynet_jwt } - }) - - -- fail gracefully in case /user/limits failed - if err or (res and res.status ~= ngx.HTTP_OK) then - ngx.log(ngx.ERR, "Failed accounts service request /user/limits: ", err or ("[HTTP " .. res.status .. "] " .. res.body)) - elseif res and res.status == ngx.HTTP_OK then - local json = require('cjson') - local limits = json.decode(res.body) + access_by_lua_block { + if require("skynet.account").accounts_enabled() then + -- check if portal is in authenticated only mode + if require("skynet.account").is_access_unauthorized() then + return require("skynet.account").exit_access_unauthorized() + end + + -- get account limits of currently authenticated user + local limits = require("skynet.account").get_account_limits() + + -- apply upload size limits ngx.req.set_header("SkynetMaxUploadSize", limits.maxUploadSize) end } @@ -306,6 +289,7 @@ location /skynet/pin { include /etc/nginx/conf.d/include/sia-auth; include /etc/nginx/conf.d/include/track-upload; include /etc/nginx/conf.d/include/generate-siapath; + include /etc/nginx/conf.d/include/portal-access-check; limit_req zone=uploads_by_ip burst=10 nodelay; limit_req zone=uploads_by_ip_throttled; @@ -319,6 +303,7 @@ location /skynet/pin { location /skynet/metadata { include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/portal-access-check; header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") @@ -331,6 +316,7 @@ location /skynet/metadata { location /skynet/resolve { include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/portal-access-check; header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") @@ -357,7 +343,7 @@ location ~ "^/file/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" { include /etc/nginx/conf.d/include/location-skylink; } -location /__internal/do/not/use/authenticated { +location /__internal/do/not/use/accounts { include /etc/nginx/conf.d/include/cors; charset utf-8; @@ -366,29 +352,16 @@ location /__internal/do/not/use/authenticated { content_by_lua_block { local json = require('cjson') + local accounts_enabled = require("skynet.account").accounts_enabled() + local is_auth_required = require("skynet.account").is_auth_required() + local is_authenticated = accounts_enabled and require("skynet.account").is_authenticated() - -- this block runs only when accounts are enabled - if not os.getenv("PORTAL_MODULES"):match("a") then - ngx.say(json.encode{authenticated = false}) - return ngx.exit(ngx.HTTP_OK) - end - - local httpc = require("resty.http").new() - - -- 10.10.10.70 points to accounts service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.70:3000/user", { - headers = { ["Cookie"] = "skynet-jwt=" .. ngx.var.skynet_jwt } + ngx.say(json.encode{ + enabled = accounts_enabled, + auth_required = is_auth_required, + authenticated = is_authenticated, }) - - -- endpoint /user should return HTTP_OK for authenticated and HTTP_UNAUTHORIZED for not authenticated - if res and (res.status == ngx.HTTP_OK or res.status == ngx.HTTP_UNAUTHORIZED) then - ngx.say(json.encode{authenticated = res.status == ngx.HTTP_OK}) - return ngx.exit(ngx.HTTP_OK) - else - ngx.log(ngx.ERR, "Failed accounts service request /user: ", err or ("[HTTP " .. res.status .. "] " .. res.body)) - ngx.say(json.encode{authenticated = false}) - return ngx.exit(ngx.HTTP_OK) - end + return ngx.exit(ngx.HTTP_OK) } } diff --git a/docker/nginx/libs/skynet/account.lua b/docker/nginx/libs/skynet/account.lua new file mode 100644 index 00000000..1c495ae4 --- /dev/null +++ b/docker/nginx/libs/skynet/account.lua @@ -0,0 +1,70 @@ +local _M = {} + +-- fallback - remember to keep those updated +local anon_limits = { ["tierName"] = "anonymous", ["upload"] = 655360, ["download"] = 655360, ["maxUploadSize"] = 1073741824, ["registry"] = 250 } + +-- no limits applied +local no_limits = { ["tierName"] = "internal", ["upload"] = 0, ["download"] = 0, ["maxUploadSize"] = 0, ["registry"] = 0 } + +-- handle request exit when access to portal should be restricted +-- currently handles only HTTP_UNAUTHORIZED but can be extended in future +function _M.exit_access_unauthorized(message) + ngx.status = ngx.HTTP_UNAUTHORIZED + ngx.header["content-type"] = "text/plain" + ngx.say(message or "Portal operator restricted access to authenticated users only") + return ngx.exit(ngx.status) +end + +function _M.accounts_enabled() + return os.getenv("PORTAL_MODULES"):match("a") ~= nil +end + +function _M.get_account_limits() + local cjson = require('cjson') + + if ngx.var.internal_no_limits == "true" then + return no_limits + end + + if ngx.var.skynet_jwt == "" then + return anon_limits + end + + if ngx.var.account_limits == "" then + local httpc = require("resty.http").new() + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/user/limits", { + headers = { ["Cookie"] = "skynet-jwt=" .. ngx.var.skynet_jwt } + }) + + -- fail gracefully in case /user/limits failed + if err or (res and res.status ~= ngx.HTTP_OK) then + ngx.log(ngx.ERR, "Failed accounts service request /user/limits: ", err or ("[HTTP " .. res.status .. "] " .. res.body)) + ngx.var.account_limits = cjson.encode(anon_limits) + elseif res and res.status == ngx.HTTP_OK then + ngx.var.account_limits = res.body + end + end + + return cjson.decode(ngx.var.account_limits) +end + +-- detect whether current user is authenticated +function _M.is_authenticated() + local limits = _M.get_account_limits() + + return limits.tierName ~= anon_limits.tierName +end + +function _M.is_auth_required() + return os.getenv("ACCOUNTS_LIMIT_ACCESS") == "authenticated" +end + +-- check whether access to portal should be restricted +-- based on the configurable environment variable +function _M.is_access_unauthorized() + return _M.accounts_enabled() and _M.is_auth_required() and not _M.is_authenticated() +end + +return _M diff --git a/docker/nginx/libs/skynet/blocklist.lua b/docker/nginx/libs/skynet/blocklist.lua index 6a447297..15e8da91 100644 --- a/docker/nginx/libs/skynet/blocklist.lua +++ b/docker/nginx/libs/skynet/blocklist.lua @@ -52,4 +52,12 @@ function _M.is_blocked(skylink) return ngx.shared.blocklist:get_stale(hash) == true end +-- exit with 416 illegal content status code +function _M.exit_illegal() + ngx.status = ngx.HTTP_ILLEGAL + ngx.header["content-type"] = "text/plain" + ngx.say("Unavailable For Legal Reasons") + return ngx.exit(ngx.status) +end + return _M diff --git a/docker/nginx/libs/skynet/skylink.spec.lua b/docker/nginx/libs/skynet/skylink.spec.lua index f45b3013..0502a833 100644 --- a/docker/nginx/libs/skynet/skylink.spec.lua +++ b/docker/nginx/libs/skynet/skylink.spec.lua @@ -1,4 +1,4 @@ -local skynet_skylink = require("skynet/skylink") +local skynet_skylink = require("skynet.skylink") describe("parse", function() local base32 = "0404dsjvti046fsua4ktor9grrpe76erq9jot9cvopbhsvsu76r4r30" diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index 48a6ed41..3ae106c8 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -29,6 +29,7 @@ worker_processes auto; env SKYNET_PORTAL_API; env SKYNET_SERVER_API; env PORTAL_MODULES; +env ACCOUNTS_LIMIT_ACCESS; events { worker_connections 8192; diff --git a/packages/website/package.json b/packages/website/package.json index a8bf981b..a7aa7157 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -78,7 +78,7 @@ "build": "gatsby build", "develop": "cross-env GATSBY_API_URL=https://siasky.net gatsby develop", "format": "prettier --write \"**/*.{js,jsx,ts,tsx,json,md}\"", - "start": "yarn develop", + "start": "gatsby develop", "serve": "gatsby serve", "clean": "gatsby clean", "test": "echo \"Write tests! -> https://gatsby.dev/unit-testing\" && exit 1", diff --git a/packages/website/src/components/Navigation/Navigation.js b/packages/website/src/components/Navigation/Navigation.js index d37f8594..5e0c5ec4 100644 --- a/packages/website/src/components/Navigation/Navigation.js +++ b/packages/website/src/components/Navigation/Navigation.js @@ -2,7 +2,8 @@ import * as React from "react"; import PropTypes from "prop-types"; import Link from "../Link"; import classnames from "classnames"; -import useAuthenticatedStatus from "../../services/useAuthenticatedStatus"; +import useAccounts from "../../services/useAccounts"; +import useAccountsUrl from "../../services/useAccountsUrl"; import { LogoWhiteText, LogoBlackText, MenuMobile, MenuMobileClose, DiscordSmall } from "../Icons"; import { useWindowSize, useWindowScroll } from "react-use"; @@ -24,8 +25,8 @@ const Navigation = ({ mode, uri }) => { const [open, setOpen] = React.useState(false); const windowSize = useWindowSize(); const isWindowTop = useWindowTop(); - const { data: authenticationStatus } = useAuthenticatedStatus(); - const authenticated = authenticationStatus?.authenticated ?? false; + const { data: accounts } = useAccounts(); + const accountsUrl = useAccountsUrl(); React.useEffect(() => { setOpen(false); @@ -40,6 +41,8 @@ const Navigation = ({ mode, uri }) => { }, [open]); const mobileMenuOffset = navRef.current ? navRef.current.offsetTop : 0; + const showLoginNavigation = accounts?.enabled && !accounts?.authenticated; + const showAccountNavigation = accounts?.enabled && accounts?.authenticated; return (