From 60cb812148a2265bcb25cb1a456e0b04c211d3db Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:17:08 +0200 Subject: [PATCH 1/8] support v2 caching --- docker/nginx/conf.d/client.conf | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 632a0c13..ac29604c 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -409,6 +409,13 @@ server { proxy_pass http://siad; } + location /skynet/resolve { + include /etc/nginx/conf.d/include/cors; + + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://siad; + } + location ~ "^/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/proxy-buffer; @@ -427,13 +434,24 @@ server { set $skylink $2; set $path $3; + # v2 support + set $skylink_v1 $skylink; + access_by_lua_block { -- disable cache if this is skylink v2 local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04" local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ" if isBase32v2 or isBase64v2 then - ngx.var.nocache = 1 # nginx expects 0/1 for boolean + local res = ngx.location.capture("/skynet/resolve/" .. ngx.var.skylink) + if res.status == ngx.HTTP_OK then + local json = require('cjson') + local resolve = json.decode(res.body) + ngx.var.skylink_v1 = resolve.skylink + else + ngx.say(res.body) + ngx.exit(res.status) + end end -- this block runs only when accounts are enabled @@ -447,6 +465,16 @@ server { end } + # in case siad returns location header and we have skylink v2, we need to replace the skylink v1 with v2 + header_filter_by_lua_block { + if ngx.var.skylink != ngx.var.skylink_v1 then + if ngx.header.location then + ngx.header.location = ngx.header.location:gsub("^" .. ngx.var.skylink_v1, ngx.var.skylink) + end + ngx.header["Skynet-Skylink"] = ngx.var.skylink + end + } + # register the download in accounts service (cookies should contain jwt) log_by_lua_block { -- this block runs only when accounts are enabled From a2a4b6ea8f25d09653aaaf1f976959d77e0f11e1 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:20:25 +0200 Subject: [PATCH 2/8] fixed != to ~= --- docker/nginx/conf.d/client.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index ac29604c..b4dc0510 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -467,7 +467,7 @@ server { # in case siad returns location header and we have skylink v2, we need to replace the skylink v1 with v2 header_filter_by_lua_block { - if ngx.var.skylink != ngx.var.skylink_v1 then + if ngx.var.skylink ~= ngx.var.skylink_v1 then if ngx.header.location then ngx.header.location = ngx.header.location:gsub("^" .. ngx.var.skylink_v1, ngx.var.skylink) end From 176e5c7a9d220542f2e157a9435cb66576ba49ea Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:29:33 +0200 Subject: [PATCH 3/8] fixed caching --- docker/nginx/conf.d/client.conf | 11 ++++++----- docker/nginx/conf.d/include/proxy-cache-downloads | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index b4dc0510..4212a813 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -436,6 +436,7 @@ server { # v2 support set $skylink_v1 $skylink; + set $skylink_v2 $skylink; access_by_lua_block { -- disable cache if this is skylink v2 @@ -443,7 +444,7 @@ server { local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ" if isBase32v2 or isBase64v2 then - local res = ngx.location.capture("/skynet/resolve/" .. ngx.var.skylink) + local res = ngx.location.capture("/skynet/resolve/" .. ngx.var.skylink_v2) if res.status == ngx.HTTP_OK then local json = require('cjson') local resolve = json.decode(res.body) @@ -467,11 +468,11 @@ server { # in case siad returns location header and we have skylink v2, we need to replace the skylink v1 with v2 header_filter_by_lua_block { - if ngx.var.skylink ~= ngx.var.skylink_v1 then + if ngx.var.skylink_v1 ~= ngx.var.skylink_v2 then if ngx.header.location then - ngx.header.location = ngx.header.location:gsub("^" .. ngx.var.skylink_v1, ngx.var.skylink) + ngx.header.location = ngx.header.location:gsub("^" .. ngx.var.skylink_v1, ngx.var.skylink_v2) end - ngx.header["Skynet-Skylink"] = ngx.var.skylink + ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 end } @@ -498,7 +499,7 @@ server { proxy_read_timeout 600; proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) - proxy_pass http://siad/skynet/skylink/$skylink$path$is_args$args; + proxy_pass http://siad/skynet/skylink/$skylink_v1$path$is_args$args; } location @base32_subdomain { diff --git a/docker/nginx/conf.d/include/proxy-cache-downloads b/docker/nginx/conf.d/include/proxy-cache-downloads index a0dfa54f..35501503 100644 --- a/docker/nginx/conf.d/include/proxy-cache-downloads +++ b/docker/nginx/conf.d/include/proxy-cache-downloads @@ -3,7 +3,7 @@ proxy_cache skynet; # cache name slice 1m; proxy_http_version 1.1; # upgrade if necessary because 1.0 does not support byte-range requests proxy_set_header Range $slice_range; # pass slice range to proxy -proxy_cache_key $uri$arg_format$arg_attachment$slice_range; # use just the uri path, format and attachment args and slice range +proxy_cache_key $skylink_v1$path$arg_format$arg_attachment$slice_range; # use just the uri path, format and attachment args and slice range proxy_cache_min_uses 3; # cache responses after 3 requests of the same file proxy_cache_valid 200 206 24h; # cache 200 and 206 responses for 24 hours proxy_cache_lock on; # queue cache requests for the same resource until it is fully cached From c9081e9d8706e1668ec9462db013b9f63c3eb6d9 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:33:13 +0200 Subject: [PATCH 4/8] log cache key --- docker/nginx/conf.d/client.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 4212a813..23d08364 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -474,6 +474,8 @@ server { end ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 end + + ngx.header["Cache-Key"] = ngx.var.skylink_v1 .. "+++" .. ngx.var.path .. "+++" .. ngx.var.arg_format .. "+++" .. ngx.var.arg_attachment .. "+++" .. ngx.var.slice_range } # register the download in accounts service (cookies should contain jwt) From 9c4f9c83003206b6a39054d2d6ce96d32fb02d4f Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:36:08 +0200 Subject: [PATCH 5/8] log cache key --- docker/nginx/conf.d/client.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 23d08364..5031a125 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -475,7 +475,7 @@ server { ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 end - ngx.header["Cache-Key"] = ngx.var.skylink_v1 .. "+++" .. ngx.var.path .. "+++" .. ngx.var.arg_format .. "+++" .. ngx.var.arg_attachment .. "+++" .. ngx.var.slice_range + ngx.header["Cache-Key"] = ngx.var.skylink_v1 .. "+++" .. ngx.var.path } # register the download in accounts service (cookies should contain jwt) From f59d00e172f446502db34e5e83949f730143afd0 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:46:47 +0200 Subject: [PATCH 6/8] log cache key --- docker/nginx/conf.d/client.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 5031a125..b688ce66 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -470,7 +470,7 @@ server { header_filter_by_lua_block { if ngx.var.skylink_v1 ~= ngx.var.skylink_v2 then if ngx.header.location then - ngx.header.location = ngx.header.location:gsub("^" .. ngx.var.skylink_v1, ngx.var.skylink_v2) + ngx.header.location = ngx.header.location:gsub(ngx.var.skylink_v1, ngx.var.skylink_v2) end ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 end From 20b8a99f1430fab202b9b6db69de1327963cff2f Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:56:11 +0200 Subject: [PATCH 7/8] fix redirect --- docker/nginx/conf.d/client.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index b688ce66..e6b981a8 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -470,7 +470,8 @@ server { header_filter_by_lua_block { if ngx.var.skylink_v1 ~= ngx.var.skylink_v2 then if ngx.header.location then - ngx.header.location = ngx.header.location:gsub(ngx.var.skylink_v1, ngx.var.skylink_v2) + local path = string.match(ngx.header.location, "[^/?]+(.*)"); + ngx.header.location = ngx.var.skylink_v2 .. path end ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 end From 3589e2ac045a02d3c7052843ba16d43ba7db8905 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 1 Jun 2021 12:57:50 +0200 Subject: [PATCH 8/8] remove cache key --- docker/nginx/conf.d/client.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index e6b981a8..005f8d43 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -475,8 +475,6 @@ server { end ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 end - - ngx.header["Cache-Key"] = ngx.var.skylink_v1 .. "+++" .. ngx.var.path } # register the download in accounts service (cookies should contain jwt)