From dec29600c1ffa9bc1d9e5bb0d64acf27b6d5916c Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 31 May 2021 14:44:36 +0200 Subject: [PATCH 01/11] clean up stats endpoint --- docker/nginx/conf.d/client.conf | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index f5373143..4e1ff4ed 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -119,33 +119,6 @@ server { location /skynet/stats { include /etc/nginx/conf.d/include/cors; - set $response_body ''; # we need a variable for full response body (not chuncked) - - # modify the response to add numfiles and totalsize to account for node rotation - # example prevstats.lua: 'return { numfiles = 12345, totalsize = 123456789 }' - body_filter_by_lua_block { - local file_exists = io.open("/data/nginx/skynet/prevstats.lua") - if file_exists then - file_exists.close() - - -- because response data is chunked, we need to concat ngx.arg[1] until - -- last chunk is received (when ngx.arg[2] is set to true) - ngx.var.response_body = ngx.var.response_body .. ngx.arg[1] - - if ngx.arg[2] then - local json = require('cjson') - local prevstats = require('/data/nginx/skynet/prevstats') - local stats = json.decode(ngx.var.response_body) - stats.uploadstats.numfiles = stats.uploadstats.numfiles + prevstats.numfiles - stats.uploadstats.totalsize = stats.uploadstats.totalsize + prevstats.totalsize - ngx.arg[1] = json.encode(stats) - else - -- do not send any data in this chunk (wait for last chunk) - ngx.arg[1] = nil - end - end - } - proxy_cache skynet; proxy_cache_valid any 1m; # cache stats for 1 minute proxy_set_header User-Agent: Sia-Agent; From 60cb812148a2265bcb25cb1a456e0b04c211d3db Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:17:08 +0200 Subject: [PATCH 02/11] support v2 caching --- docker/nginx/conf.d/client.conf | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 632a0c13..ac29604c 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -409,6 +409,13 @@ server { proxy_pass http://siad; } + location /skynet/resolve { + include /etc/nginx/conf.d/include/cors; + + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://siad; + } + location ~ "^/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/proxy-buffer; @@ -427,13 +434,24 @@ server { set $skylink $2; set $path $3; + # v2 support + set $skylink_v1 $skylink; + access_by_lua_block { -- disable cache if this is skylink v2 local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04" local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ" if isBase32v2 or isBase64v2 then - ngx.var.nocache = 1 # nginx expects 0/1 for boolean + local res = ngx.location.capture("/skynet/resolve/" .. ngx.var.skylink) + if res.status == ngx.HTTP_OK then + local json = require('cjson') + local resolve = json.decode(res.body) + ngx.var.skylink_v1 = resolve.skylink + else + ngx.say(res.body) + ngx.exit(res.status) + end end -- this block runs only when accounts are enabled @@ -447,6 +465,16 @@ server { end } + # in case siad returns location header and we have skylink v2, we need to replace the skylink v1 with v2 + header_filter_by_lua_block { + if ngx.var.skylink != ngx.var.skylink_v1 then + if ngx.header.location then + ngx.header.location = ngx.header.location:gsub("^" .. ngx.var.skylink_v1, ngx.var.skylink) + end + ngx.header["Skynet-Skylink"] = ngx.var.skylink + end + } + # register the download in accounts service (cookies should contain jwt) log_by_lua_block { -- this block runs only when accounts are enabled From a2a4b6ea8f25d09653aaaf1f976959d77e0f11e1 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:20:25 +0200 Subject: [PATCH 03/11] fixed != to ~= --- docker/nginx/conf.d/client.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index ac29604c..b4dc0510 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -467,7 +467,7 @@ server { # in case siad returns location header and we have skylink v2, we need to replace the skylink v1 with v2 header_filter_by_lua_block { - if ngx.var.skylink != ngx.var.skylink_v1 then + if ngx.var.skylink ~= ngx.var.skylink_v1 then if ngx.header.location then ngx.header.location = ngx.header.location:gsub("^" .. ngx.var.skylink_v1, ngx.var.skylink) end From 176e5c7a9d220542f2e157a9435cb66576ba49ea Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:29:33 +0200 Subject: [PATCH 04/11] fixed caching --- docker/nginx/conf.d/client.conf | 11 ++++++----- docker/nginx/conf.d/include/proxy-cache-downloads | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index b4dc0510..4212a813 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -436,6 +436,7 @@ server { # v2 support set $skylink_v1 $skylink; + set $skylink_v2 $skylink; access_by_lua_block { -- disable cache if this is skylink v2 @@ -443,7 +444,7 @@ server { local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ" if isBase32v2 or isBase64v2 then - local res = ngx.location.capture("/skynet/resolve/" .. ngx.var.skylink) + local res = ngx.location.capture("/skynet/resolve/" .. ngx.var.skylink_v2) if res.status == ngx.HTTP_OK then local json = require('cjson') local resolve = json.decode(res.body) @@ -467,11 +468,11 @@ server { # in case siad returns location header and we have skylink v2, we need to replace the skylink v1 with v2 header_filter_by_lua_block { - if ngx.var.skylink ~= ngx.var.skylink_v1 then + if ngx.var.skylink_v1 ~= ngx.var.skylink_v2 then if ngx.header.location then - ngx.header.location = ngx.header.location:gsub("^" .. ngx.var.skylink_v1, ngx.var.skylink) + ngx.header.location = ngx.header.location:gsub("^" .. ngx.var.skylink_v1, ngx.var.skylink_v2) end - ngx.header["Skynet-Skylink"] = ngx.var.skylink + ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 end } @@ -498,7 +499,7 @@ server { proxy_read_timeout 600; proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) - proxy_pass http://siad/skynet/skylink/$skylink$path$is_args$args; + proxy_pass http://siad/skynet/skylink/$skylink_v1$path$is_args$args; } location @base32_subdomain { diff --git a/docker/nginx/conf.d/include/proxy-cache-downloads b/docker/nginx/conf.d/include/proxy-cache-downloads index a0dfa54f..35501503 100644 --- a/docker/nginx/conf.d/include/proxy-cache-downloads +++ b/docker/nginx/conf.d/include/proxy-cache-downloads @@ -3,7 +3,7 @@ proxy_cache skynet; # cache name slice 1m; proxy_http_version 1.1; # upgrade if necessary because 1.0 does not support byte-range requests proxy_set_header Range $slice_range; # pass slice range to proxy -proxy_cache_key $uri$arg_format$arg_attachment$slice_range; # use just the uri path, format and attachment args and slice range +proxy_cache_key $skylink_v1$path$arg_format$arg_attachment$slice_range; # use just the uri path, format and attachment args and slice range proxy_cache_min_uses 3; # cache responses after 3 requests of the same file proxy_cache_valid 200 206 24h; # cache 200 and 206 responses for 24 hours proxy_cache_lock on; # queue cache requests for the same resource until it is fully cached From c9081e9d8706e1668ec9462db013b9f63c3eb6d9 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:33:13 +0200 Subject: [PATCH 05/11] log cache key --- docker/nginx/conf.d/client.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 4212a813..23d08364 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -474,6 +474,8 @@ server { end ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 end + + ngx.header["Cache-Key"] = ngx.var.skylink_v1 .. "+++" .. ngx.var.path .. "+++" .. ngx.var.arg_format .. "+++" .. ngx.var.arg_attachment .. "+++" .. ngx.var.slice_range } # register the download in accounts service (cookies should contain jwt) From 9c4f9c83003206b6a39054d2d6ce96d32fb02d4f Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:36:08 +0200 Subject: [PATCH 06/11] log cache key --- docker/nginx/conf.d/client.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 23d08364..5031a125 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -475,7 +475,7 @@ server { ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 end - ngx.header["Cache-Key"] = ngx.var.skylink_v1 .. "+++" .. ngx.var.path .. "+++" .. ngx.var.arg_format .. "+++" .. ngx.var.arg_attachment .. "+++" .. ngx.var.slice_range + ngx.header["Cache-Key"] = ngx.var.skylink_v1 .. "+++" .. ngx.var.path } # register the download in accounts service (cookies should contain jwt) From f59d00e172f446502db34e5e83949f730143afd0 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:46:47 +0200 Subject: [PATCH 07/11] log cache key --- docker/nginx/conf.d/client.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 5031a125..b688ce66 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -470,7 +470,7 @@ server { header_filter_by_lua_block { if ngx.var.skylink_v1 ~= ngx.var.skylink_v2 then if ngx.header.location then - ngx.header.location = ngx.header.location:gsub("^" .. ngx.var.skylink_v1, ngx.var.skylink_v2) + ngx.header.location = ngx.header.location:gsub(ngx.var.skylink_v1, ngx.var.skylink_v2) end ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 end From 20b8a99f1430fab202b9b6db69de1327963cff2f Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:56:11 +0200 Subject: [PATCH 08/11] fix redirect --- docker/nginx/conf.d/client.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index b688ce66..e6b981a8 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -470,7 +470,8 @@ server { header_filter_by_lua_block { if ngx.var.skylink_v1 ~= ngx.var.skylink_v2 then if ngx.header.location then - ngx.header.location = ngx.header.location:gsub(ngx.var.skylink_v1, ngx.var.skylink_v2) + local path = string.match(ngx.header.location, "[^/?]+(.*)"); + ngx.header.location = ngx.var.skylink_v2 .. path end ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 end From 3589e2ac045a02d3c7052843ba16d43ba7db8905 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:57:50 +0200 Subject: [PATCH 09/11] remove cache key --- docker/nginx/conf.d/client.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index e6b981a8..005f8d43 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -475,8 +475,6 @@ server { end ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 end - - ngx.header["Cache-Key"] = ngx.var.skylink_v1 .. "+++" .. ngx.var.path } # register the download in accounts service (cookies should contain jwt) From 898a728f5a97a3629620ad6cbb83c2e73efae25e Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 4 Jun 2021 11:43:52 +0200 Subject: [PATCH 10/11] Increase request timeout --- docker/nginx/conf.d/client.conf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 632a0c13..24547762 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -333,7 +333,12 @@ server { limit_conn upload_conn_rl 1; client_max_body_size 1000M; # make sure to limit the size of upload to a sane value + + # increase request timeouts proxy_read_timeout 600; + proxy_connect_timeout 600; + proxy_send_timeout 600; + proxy_request_buffering off; # stream uploaded files through the proxy as it comes in proxy_set_header Expect $http_expect; proxy_set_header User-Agent: Sia-Agent; @@ -391,7 +396,12 @@ server { include /etc/nginx/conf.d/include/cors; client_max_body_size 50M; # tus chunks size is 40M + leaving 10M of breathing room + + # increase request timeouts proxy_read_timeout 600; + proxy_connect_timeout 600; + proxy_send_timeout 600; + proxy_request_buffering off; # stream uploaded files through the proxy as it comes in proxy_set_header Expect $http_expect; From bd8593039dd824baa27a599b623cfe3bf767a4ab Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 4 Jun 2021 11:58:38 +0200 Subject: [PATCH 11/11] Do not increase proxy_connect_timeout --- docker/nginx/conf.d/client.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 24547762..bd5ddef5 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -336,7 +336,6 @@ server { # increase request timeouts proxy_read_timeout 600; - proxy_connect_timeout 600; proxy_send_timeout 600; proxy_request_buffering off; # stream uploaded files through the proxy as it comes in @@ -399,7 +398,6 @@ server { # increase request timeouts proxy_read_timeout 600; - proxy_connect_timeout 600; proxy_send_timeout 600; proxy_request_buffering off; # stream uploaded files through the proxy as it comes in