From a4ce3c935dc2a0e7e0c313dbf7855ddbb02f6ff3 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 5 Nov 2021 12:57:06 +0100 Subject: [PATCH] implement base32 decoder --- docker/nginx/Dockerfile | 1 + .../conf.d/include/init-optional-variables | 3 + docker/nginx/conf.d/include/location-skylink | 4 ++ docker/nginx/conf.d/server/server.api | 6 -- docker/nginx/conf.d/server/server.dnslink | 1 + docker/nginx/libs/skynet/skylink.lua | 61 +++++++++++++++++++ docker/nginx/nginx.conf | 8 ++- 7 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 docker/nginx/libs/skynet/skylink.lua diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index acff0e3e..fd6e4f09 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -7,6 +7,7 @@ RUN luarocks install lua-resty-http && \ -out /etc/ssl/local-certificate.crt COPY mo ./ +COPY libs /etc/nginx/libs COPY conf.d /etc/nginx/conf.d COPY conf.d.templates /etc/nginx/conf.d.templates diff --git a/docker/nginx/conf.d/include/init-optional-variables b/docker/nginx/conf.d/include/init-optional-variables index 4f1be02b..e2e74030 100644 --- a/docker/nginx/conf.d/include/init-optional-variables +++ b/docker/nginx/conf.d/include/init-optional-variables @@ -7,3 +7,6 @@ set $hns_domain ''; # set only if server has been access through SERVER_DOMAIN set $server_alias ''; + +# expose skylink variable so we can use it in access log +set $skylink ''; diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index a3692d00..143ca108 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -11,6 +11,10 @@ if ($request_method = PURGE) { limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time +# ensure that skylink that we pass around is base64 encoded (transform base32 encoded ones) +# this is important because we want only one format in cache keys and logs +set_by_lua_block $skylink { return require("skynet.skylink").parse(ngx.var.skylink) } + # $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was: # a) skylink v1 - it would not matter, no additional logic is executed # b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 9f966d0d..cd925429 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -231,9 +231,6 @@ location /skynet/metadata { header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") - - -- do not expose internal header - ngx.header["Skynet-Requested-Skylink"] = "" } proxy_set_header User-Agent: Sia-Agent; @@ -246,9 +243,6 @@ location /skynet/resolve { header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") - - -- do not expose internal header - ngx.header["Skynet-Requested-Skylink"] = "" } proxy_set_header User-Agent: Sia-Agent; diff --git a/docker/nginx/conf.d/server/server.dnslink b/docker/nginx/conf.d/server/server.dnslink index 1dd3a489..32e454cc 100644 --- a/docker/nginx/conf.d/server/server.dnslink +++ b/docker/nginx/conf.d/server/server.dnslink @@ -37,6 +37,7 @@ location / { ngx.var.skylink = cache_value end + ngx.var.skylink = require("skynet.skylink").parse(ngx.var.skylink) ngx.var.skylink_v1 = ngx.var.skylink ngx.var.skylink_v2 = ngx.var.skylink } diff --git a/docker/nginx/libs/skynet/skylink.lua b/docker/nginx/libs/skynet/skylink.lua new file mode 100644 index 00000000..f059249e --- /dev/null +++ b/docker/nginx/libs/skynet/skylink.lua @@ -0,0 +1,61 @@ +local _M = {} + +local function str_split(str, size) + local result = {} + for i=1, #str, size do + table.insert(result, str:sub(i, i + size - 1)) + end + return result +end + +local function dec2bin(num) + local result = "" + repeat + local halved = num / 2 + local int, frac = math.modf(halved) + num = int + result = math.ceil(frac) .. result + until num == 0 + return result +end + +local base32Alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUV" + +-- this is not a full implementation of base32 decoder, it doesn't +-- cover all edge cases but it works good enough for decoding skylinks +-- source: https://gist.github.com/linsonder6/3202fc06832f97905aab2a8a492a80de +function decodeBase32Skylink(str) + local binary = string.upper(str):gsub(".", function (char) + if char == "=" then return "" end + local pos = string.find(base32Alphabet, char) + pos = pos - 1 + return string.format("%05u", dec2bin(pos)) + end) + + local bytes = str_split(binary, 8) + + local decoded = {} + for _, byte in pairs(bytes) do + table.insert(decoded, string.char(tonumber(byte, 2))) + end + + local concatenated = table.concat(decoded) + + -- decoded skylink is always 34 bytes, anything + -- else is leftover and should be discarded + return string.sub(concatenated, 1, 34) +end + +-- parse any skylink and return base64 version +function _M.parse(skylink) + if string.len(skylink) == 55 then + local base64 = require("ngx.base64") + local decoded = decodeBase32Skylink(skylink) + + return base64.encode_base64url(decoded) + end + + return skylink +end + +return _M diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index c1f92d92..755bd003 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -38,6 +38,8 @@ http { include mime.types; default_type application/octet-stream; + lua_package_path "/etc/nginx/libs/?.lua;;"; + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" $upstream_response_time ' @@ -45,7 +47,7 @@ http { '"$upstream_http_content_type" "$upstream_cache_status" ' '"$server_alias" "$sent_http_skynet_skylink" ' '$upstream_connect_time $upstream_header_time ' - '$request_time "$hns_domain"'; + '$request_time "$hns_domain" "$skylink"'; access_log logs/access.log main; @@ -72,8 +74,10 @@ http { # this runs before forking out nginx worker processes init_by_lua_block { - require "cjson" + require "cjson" + require "ngx.base64" require "resty.http" + require "skynet.skylink" } # include skynet-portal-api and skynet-server-api header on every request