LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

clean up nginx skylink cache related code

This commit is contained in:
Karol Wypchlo 2022-03-15 09:40:25 +01:00
parent a93e4e93f8
commit 4f6b5fb0a8
No known key found for this signature in database
GPG Key ID: B515DE9EEBE241E1
7 changed files with 7 additions and 213 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docker-compose.yml
View File

@ -64,8 +64,6 @@ services:
logging: *default-logging
env_file:
- .env
environment:
- SKYD_DISK_CACHE_ENABLED=${SKYD_DISK_CACHE_ENABLED:-true}
volumes:
- ./docker/data/nginx/cache:/data/nginx/cache
- ./docker/data/nginx/blocker:/data/nginx/blocker

81
docker/nginx/conf.d/include/location-skylink
View File

@ -1,6 +1,5 @@
include /etc/nginx/conf.d/include/cors;
include /etc/nginx/conf.d/include/proxy-buffer;
include /etc/nginx/conf.d/include/proxy-cache-downloads;
include /etc/nginx/conf.d/include/track-download;
limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
@ -9,62 +8,10 @@ limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
# this is important because we want only one format in cache keys and logs
set_by_lua_block $skylink { return require("skynet.skylink").parse(ngx.var.skylink) }
# $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was:
# a) skylink v1 - it would not matter, no additional logic is executed
# b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update
# $skylink_v1 variable so then the proxy request to skyd can be cached in nginx (proxy_cache_key
# in proxy-cache-downloads includes $skylink_v1 as a part of the cache key)
set $skylink_v1 $skylink;
set $skylink_v2 $skylink;
# variable for Skynet-Proof header that we need to inject
# into a response if the request was for skylink v2
set $skynet_proof '';
# default download rate to unlimited
set $limit_rate 0;
access_by_lua_block {
-- the block below only makes sense if we are using nginx cache
if not ngx.var.skyd_disk_cache_enabled then
local httpc = require("resty.http").new()
-- detect whether requested skylink is v2
local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04"
local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ"
if isBase32v2 or isBase64v2 then
-- 10.10.10.10 points to sia service (alias not available when using resty-http)
local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/resolve/" .. ngx.var.skylink_v2, {
headers = { ["User-Agent"] = "Sia-Agent" }
})
-- print error and exit with 500 or exit with response if status is not 200
if err or (res and res.status ~= ngx.HTTP_OK) then
ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
ngx.header["content-type"] = "text/plain"
ngx.say(err or res.body)
return ngx.exit(ngx.status)
end
local json = require('cjson')
local resolve = json.decode(res.body)
ngx.var.skylink_v1 = resolve.skylink
ngx.var.skynet_proof = res.headers["Skynet-Proof"]
end
-- check if skylink v1 is present on blocklist (compare hashes)
if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then
return require("skynet.blocklist").exit_illegal()
end
-- if skylink is found on nocache list then set internal nocache variable
-- to tell nginx that it should not try and cache this file (too large)
if ngx.shared.nocache:get(ngx.var.skylink_v1) then
ngx.var.nocache = "1"
end
end
if require("skynet.account").accounts_enabled() then
-- check if portal is in authenticated only mode
if require("skynet.account").is_access_unauthorized() then
@ -84,36 +31,10 @@ access_by_lua_block {
end
}
header_filter_by_lua_block {
ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. ngx.var.skynet_portal_domain
ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. ngx.var.skynet_server_domain
-- the block below only makes sense if we are using nginx cache
if not ngx.var.skyd_disk_cache_enabled then
-- not empty skynet_proof means this is a skylink v2 request
-- so we should replace the Skynet-Proof header with the one
-- we got from /skynet/resolve/ endpoint, otherwise we would
-- be serving cached empty v1 skylink Skynet-Proof header
if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then
ngx.header["Skynet-Proof"] = ngx.var.skynet_proof
end
-- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold
-- (content length can be nil for already cached files - we can ignore them)
if ngx.header["Content-Length"] and tonumber(ngx.header["Content-Length"]) > 1e+9 then
ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"])
end
end
}
limit_rate_after 512k;
limit_rate $limit_rate;
proxy_read_timeout 600;
proxy_set_header User-Agent: Sia-Agent;
# in case the requested skylink was v2 and we already resolved it to skylink v1, we are going to pass resolved
# skylink v1 to skyd to save that extra skylink v2 lookup in skyd but in turn, in case skyd returns a redirect
# we need to rewrite the skylink v1 to skylink v2 in the location header with proxy_redirect
proxy_redirect $skylink_v1 $skylink_v2;
proxy_pass http://sia:9980/skynet/skylink/$skylink_v1$path$is_args$args;
proxy_pass http://sia:9980/skynet/skylink/$skylink$path$is_args$args;

21
docker/nginx/conf.d/include/proxy-cache-downloads
View File

@ -1,21 +0,0 @@
proxy_cache skynet; # cache name
proxy_cache_key $skylink_v1$path$arg_format$arg_attachment$arg_start$arg_end$http_range; # unique cache key
proxy_cache_min_uses 3; # cache after 3 uses
proxy_cache_valid 200 206 307 308 48h; # keep 200, 206, 307 and 308 responses valid for up to 2 days
add_header X-Proxy-Cache $upstream_cache_status; # add response header to indicate cache hits and misses
# map skyd env variable value to "1" for true and "0" for false (expected by proxy_no_cache)
set_by_lua_block $skyd_disk_cache_enabled {
return os.getenv("SKYD_DISK_CACHE_ENABLED") == "true" and "1" or "0"
}
# bypass - this will bypass cache hit on request (status BYPASS)
# but still stores file in cache if cache conditions are met
proxy_cache_bypass $cookie_nocache $arg_nocache $skyd_disk_cache_enabled;
# no cache - this will ignore cache on request (status MISS)
# and does not store file in cache under no condition
set_if_empty $nocache "0";
# disable cache when nocache is set or skyd cache is enabled
proxy_no_cache $nocache $skyd_disk_cache_enabled;

2
docker/nginx/conf.d/server/server.dnslink
View File

@ -38,8 +38,6 @@ location / {
end
ngx.var.skylink = require("skynet.skylink").parse(ngx.var.skylink)
ngx.var.skylink_v1 = ngx.var.skylink
ngx.var.skylink_v2 = ngx.var.skylink
}
include /etc/nginx/conf.d/include/location-skylink;

37
docker/nginx/conf.d/server/server.local
View File

@ -1,37 +1,12 @@
include /etc/nginx/conf.d/include/init-optional-variables;
# TODO: this endpoint could be removed and calls be made directly to skyd
# since we're not using any nginx specific code here any more
location /skynet/blocklist {
include /etc/nginx/conf.d/include/sia-auth;
client_max_body_size 10m; # increase max body size to account for large lists
client_body_buffer_size 10m; # force whole body to memory so we can read it
content_by_lua_block {
local httpc = require("resty.http").new()
ngx.req.read_body() -- ensure the post body data is read before using get_body_data
-- proxy blocklist update request
-- 10.10.10.10 points to sia service (alias not available when using resty-http)
local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/blocklist", {
method = "POST",
body = ngx.req.get_body_data(),
headers = {
["Content-Type"] = "application/x-www-form-urlencoded",
["Authorization"] = require("skynet.utils").authorization_header(),
["User-Agent"] = "Sia-Agent",
}
})
-- print error and exit with 500 or exit with response if status is not 204
if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then
ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
ngx.header["content-type"] = "text/plain"
ngx.say(err or res.body)
return ngx.exit(ngx.status)
end
require("skynet.blocklist").reload()
ngx.status = ngx.HTTP_NO_CONTENT
return ngx.exit(ngx.status)
}
proxy_set_header User-Agent: Sia-Agent;
proxy_pass http://sia:9980/skynet/blocklist;
}

66
docker/nginx/libs/skynet/blocklist.lua
View File

@ -1,66 +0,0 @@
local _M = {}
function _M.reload()
local httpc = require("resty.http").new()
-- fetch blocklist records (all blocked skylink hashes)
-- 10.10.10.10 points to sia service (alias not available when using resty-http)
local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/blocklist", {
headers = {
["User-Agent"] = "Sia-Agent",
}
})
-- fail whole request in case this request failed, we want to make sure
-- the blocklist is pre cached before serving first skylink
if err or (res and res.status ~= ngx.HTTP_OK) then
ngx.log(ngx.ERR, "Failed skyd service request /skynet/blocklist: ", err or ("[HTTP " .. res.status .. "] " .. res.body))
ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
ngx.header["content-type"] = "text/plain"
ngx.say(err or res.body)
return ngx.exit(ngx.status)
elseif res and res.status == ngx.HTTP_OK then
local json = require('cjson')
local data = json.decode(res.body)
-- mark all existing entries as expired
ngx.shared.blocklist:flush_all()
-- check if blocklist is table (it is null when empty)
if type(data.blocklist) == "table" then
-- set all cache entries one by one (resets expiration)
for i, hash in ipairs(data.blocklist) do
ngx.shared.blocklist:set(hash, true)
end
end
-- ensure that init flag is persisted
ngx.shared.blocklist:set("__init", true)
-- remove all leftover expired entries
ngx.shared.blocklist:flush_expired()
end
end
function _M.is_blocked(skylink)
-- make sure that blocklist has been preloaded
if not ngx.shared.blocklist:get("__init") then _M.reload() end
-- hash skylink before comparing it with blocklist
local hash = require("skynet.skylink").hash(skylink)
-- we need to use get_stale because we are expiring previous
-- entries when the blocklist is reloading and we still want
-- to block them until the reloading is finished
return ngx.shared.blocklist:get_stale(hash) == true
end
-- exit with 416 illegal content status code
function _M.exit_illegal()
ngx.status = ngx.HTTP_ILLEGAL
ngx.header["content-type"] = "text/plain"
ngx.say("Unavailable For Legal Reasons")
return ngx.exit(ngx.status)
end
return _M

11
docker/nginx/nginx.conf
View File

@ -31,7 +31,6 @@ env SERVER_DOMAIN;
env PORTAL_MODULES;
env ACCOUNTS_LIMIT_ACCESS;
env SIA_API_PASSWORD;
env SKYD_DISK_CACHE_ENABLED;
events {
worker_connections 8192;
@ -75,20 +74,10 @@ http {
# proxy cache definition
proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=50g min_free=100g inactive=48h use_temp_path=off;
# create a shared blocklist dictionary with size of 30 megabytes
# estimated capacity of 1 megabyte dictionary is 3500 blocklist entries
# that gives us capacity of around 100k entries in 30 megabyte dictionary
lua_shared_dict blocklist 30m;
# create a shared dictionary to fill with skylinks that should not
# be cached due to the large size or some other reasons
lua_shared_dict nocache 10m;
# this runs before forking out nginx worker processes
init_by_lua_block {
require "cjson"
require "resty.http"
require "skynet.blocklist"
require "skynet.skylink"
require "skynet.utils"
}