LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Disable nginx proxy cache and rate limits.

This commit is contained in:
Ivaylo Novakov 2022-03-14 18:45:04 +01:00
parent a93e4e93f8
commit ae7dac59cc
No known key found for this signature in database
GPG Key ID: 06B9354AB08BE9C6
3 changed files with 34 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
docker/nginx/conf.d/include/location-skylink
View File

@ -11,13 +11,13 @@ set_by_lua_block $skylink { return require("skynet.skylink").parse(ngx.var.skyli
# $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was: # $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was:
# a) skylink v1 - it would not matter, no additional logic is executed # a) skylink v1 - it would not matter, no additional logic is executed
# b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update # b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update
# $skylink_v1 variable so then the proxy request to skyd can be cached in nginx (proxy_cache_key # $skylink_v1 variable so then the proxy request to skyd can be cached in nginx (proxy_cache_key
# in proxy-cache-downloads includes $skylink_v1 as a part of the cache key) # in proxy-cache-downloads includes $skylink_v1 as a part of the cache key)
set $skylink_v1 $skylink; set $skylink_v1 $skylink;
set $skylink_v2 $skylink; set $skylink_v2 $skylink;
# variable for Skynet-Proof header that we need to inject # variable for Skynet-Proof header that we need to inject
# into a response if the request was for skylink v2 # into a response if the request was for skylink v2
set $skynet_proof ''; set $skynet_proof '';
@ -25,85 +25,23 @@ set $skynet_proof '';
set $limit_rate 0; set $limit_rate 0;
access_by_lua_block { access_by_lua_block {
-- the block below only makes sense if we are using nginx cache local accounts = require("skynet.account")
if not ngx.var.skyd_disk_cache_enabled then if accounts.accounts_enabled() then
local httpc = require("resty.http").new()
-- detect whether requested skylink is v2
local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04"
local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ"
if isBase32v2 or isBase64v2 then
-- 10.10.10.10 points to sia service (alias not available when using resty-http)
local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/resolve/" .. ngx.var.skylink_v2, {
headers = { ["User-Agent"] = "Sia-Agent" }
})
-- print error and exit with 500 or exit with response if status is not 200
if err or (res and res.status ~= ngx.HTTP_OK) then
ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
ngx.header["content-type"] = "text/plain"
ngx.say(err or res.body)
return ngx.exit(ngx.status)
end
local json = require('cjson')
local resolve = json.decode(res.body)
ngx.var.skylink_v1 = resolve.skylink
ngx.var.skynet_proof = res.headers["Skynet-Proof"]
end
-- check if skylink v1 is present on blocklist (compare hashes)
if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then
return require("skynet.blocklist").exit_illegal()
end
-- if skylink is found on nocache list then set internal nocache variable
-- to tell nginx that it should not try and cache this file (too large)
if ngx.shared.nocache:get(ngx.var.skylink_v1) then
ngx.var.nocache = "1"
end
end
if require("skynet.account").accounts_enabled() then
-- check if portal is in authenticated only mode -- check if portal is in authenticated only mode
if require("skynet.account").is_access_unauthorized() then if accounts.is_access_unauthorized() then
return require("skynet.account").exit_access_unauthorized() return accounts.exit_access_unauthorized()
end end
-- check if portal is in subscription only mode -- check if portal is in subscription only mode
if require("skynet.account").is_access_forbidden() then if accounts.is_access_forbidden() then
return require("skynet.account").exit_access_forbidden() return accounts.exit_access_forbidden()
end end
-- get account limits of currently authenticated user
local limits = require("skynet.account").get_account_limits()
-- apply download speed limit
ngx.var.limit_rate = limits.download
end end
} }
header_filter_by_lua_block { header_filter_by_lua_block {
ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. ngx.var.skynet_portal_domain ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. ngx.var.skynet_portal_domain
ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. ngx.var.skynet_server_domain ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. ngx.var.skynet_server_domain
-- the block below only makes sense if we are using nginx cache
if not ngx.var.skyd_disk_cache_enabled then
-- not empty skynet_proof means this is a skylink v2 request
-- so we should replace the Skynet-Proof header with the one
-- we got from /skynet/resolve/ endpoint, otherwise we would
-- be serving cached empty v1 skylink Skynet-Proof header
if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then
ngx.header["Skynet-Proof"] = ngx.var.skynet_proof
end
-- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold
-- (content length can be nil for already cached files - we can ignore them)
if ngx.header["Content-Length"] and tonumber(ngx.header["Content-Length"]) > 1e+9 then
ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"])
end
end
} }
limit_rate_after 512k; limit_rate_after 512k;
@ -112,8 +50,8 @@ limit_rate $limit_rate;
proxy_read_timeout 600; proxy_read_timeout 600;
proxy_set_header User-Agent: Sia-Agent; proxy_set_header User-Agent: Sia-Agent;
# in case the requested skylink was v2 and we already resolved it to skylink v1, we are going to pass resolved # in case the requested skylink was v2 and we already resolved it to skylink v1, we are going to pass resolved
# skylink v1 to skyd to save that extra skylink v2 lookup in skyd but in turn, in case skyd returns a redirect # skylink v1 to skyd to save that extra skylink v2 lookup in skyd but in turn, in case skyd returns a redirect
# we need to rewrite the skylink v1 to skylink v2 in the location header with proxy_redirect # we need to rewrite the skylink v1 to skylink v2 in the location header with proxy_redirect
proxy_redirect $skylink_v1 $skylink_v2; proxy_redirect $skylink_v1 $skylink_v2;
proxy_pass http://sia:9980/skynet/skylink/$skylink_v1$path$is_args$args; proxy_pass http://sia:9980/skynet/skylink/$skylink_v1$path$is_args$args;

20
docker/nginx/conf.d/include/proxy-cache-downloads
View File

@ -1,13 +1,13 @@
proxy_cache skynet; # cache name #proxy_cache skynet; # cache name
proxy_cache_key $skylink_v1$path$arg_format$arg_attachment$arg_start$arg_end$http_range; # unique cache key #proxy_cache_key $skylink_v1$path$arg_format$arg_attachment$arg_start$arg_end$http_range; # unique cache key
proxy_cache_min_uses 3; # cache after 3 uses #proxy_cache_min_uses 3; # cache after 3 uses
proxy_cache_valid 200 206 307 308 48h; # keep 200, 206, 307 and 308 responses valid for up to 2 days #proxy_cache_valid 200 206 307 308 48h; # keep 200, 206, 307 and 308 responses valid for up to 2 days
add_header X-Proxy-Cache $upstream_cache_status; # add response header to indicate cache hits and misses add_header X-Proxy-Cache $upstream_cache_status; # add response header to indicate cache hits and misses
# map skyd env variable value to "1" for true and "0" for false (expected by proxy_no_cache) ## map skyd env variable value to "1" for true and "0" for false (expected by proxy_no_cache)
set_by_lua_block $skyd_disk_cache_enabled { #set_by_lua_block $skyd_disk_cache_enabled {
return os.getenv("SKYD_DISK_CACHE_ENABLED") == "true" and "1" or "0" # return os.getenv("SKYD_DISK_CACHE_ENABLED") == "true" and "1" or "0"
} #}
# bypass - this will bypass cache hit on request (status BYPASS) # bypass - this will bypass cache hit on request (status BYPASS)
# but still stores file in cache if cache conditions are met # but still stores file in cache if cache conditions are met
@ -15,7 +15,7 @@ proxy_cache_bypass $cookie_nocache $arg_nocache $skyd_disk_cache_enabled;
# no cache - this will ignore cache on request (status MISS) # no cache - this will ignore cache on request (status MISS)
# and does not store file in cache under no condition # and does not store file in cache under no condition
set_if_empty $nocache "0"; set $nocache "1";
# disable cache when nocache is set or skyd cache is enabled # disable cache when nocache is set or skyd cache is enabled
proxy_no_cache $nocache $skyd_disk_cache_enabled; proxy_no_cache $nocache

17
docker/nginx/libs/skynet/account.lua
View File

@ -14,6 +14,9 @@ local anon_limits = {
["registry"] = 250 ["registry"] = 250
} }
-- basic caching of user data
local curr_limits
-- handle request exit when access to portal should be restricted to authenticated users only -- handle request exit when access to portal should be restricted to authenticated users only
function _M.exit_access_unauthorized(message) function _M.exit_access_unauthorized(message)
ngx.status = ngx.HTTP_UNAUTHORIZED ngx.status = ngx.HTTP_UNAUTHORIZED
@ -35,20 +38,25 @@ function _M.accounts_enabled()
end end
function _M.get_account_limits() function _M.get_account_limits()
if curr_limits ~= nil then
return curr_limits
end
local cjson = require('cjson') local cjson = require('cjson')
-- TODO: This needs to accommodate authorization tokens and api keys.
if ngx.var.skynet_jwt == "" then if ngx.var.skynet_jwt == "" then
return anon_limits return anon_limits
end end
if ngx.var.account_limits == "" then if ngx.var.account_limits == "" then
local httpc = require("resty.http").new() local httpc = require("resty.http").new()
-- 10.10.10.70 points to accounts service (alias not available when using resty-http) -- 10.10.10.70 points to accounts service (alias not available when using resty-http)
local res, err = httpc:request_uri("http://10.10.10.70:3000/user/limits", { local res, err = httpc:request_uri("http://10.10.10.70:3000/user/limits", {
headers = { ["Cookie"] = "skynet-jwt=" .. ngx.var.skynet_jwt } headers = { ["Cookie"] = "skynet-jwt=" .. ngx.var.skynet_jwt }
}) })
-- fail gracefully in case /user/limits failed -- fail gracefully in case /user/limits failed
if err or (res and res.status ~= ngx.HTTP_OK) then if err or (res and res.status ~= ngx.HTTP_OK) then
ngx.log(ngx.ERR, "Failed accounts service request /user/limits: ", err or ("[HTTP " .. res.status .. "] " .. res.body)) ngx.log(ngx.ERR, "Failed accounts service request /user/limits: ", err or ("[HTTP " .. res.status .. "] " .. res.body))
@ -58,7 +66,8 @@ function _M.get_account_limits()
end end
end end
return cjson.decode(ngx.var.account_limits) curr_limits = cjson.decode(ngx.var.account_limits)
return curr_limits
end end
-- detect whether current user is authenticated -- detect whether current user is authenticated
@ -78,7 +87,7 @@ function _M.has_subscription()
end end
function _M.is_auth_required() function _M.is_auth_required()
return os.getenv("ACCOUNTS_LIMIT_ACCESS") == "authenticated" return os.getenv("ACCOUNTS_LIMIT_ACCESS") == "authenticated" or os.getenv("ACCOUNTS_LIMIT_ACCESS") == "subscription"
end end
function _M.is_subscription_required() function _M.is_subscription_required()