From a4ce3c935dc2a0e7e0c313dbf7855ddbb02f6ff3 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 5 Nov 2021 12:57:06 +0100 Subject: [PATCH 1/2] implement base32 decoder --- docker/nginx/Dockerfile | 1 + .../conf.d/include/init-optional-variables | 3 + docker/nginx/conf.d/include/location-skylink | 4 ++ docker/nginx/conf.d/server/server.api | 6 -- docker/nginx/conf.d/server/server.dnslink | 1 + docker/nginx/libs/skynet/skylink.lua | 61 +++++++++++++++++++ docker/nginx/nginx.conf | 8 ++- 7 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 docker/nginx/libs/skynet/skylink.lua diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index acff0e3e..fd6e4f09 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -7,6 +7,7 @@ RUN luarocks install lua-resty-http && \ -out /etc/ssl/local-certificate.crt COPY mo ./ +COPY libs /etc/nginx/libs COPY conf.d /etc/nginx/conf.d COPY conf.d.templates /etc/nginx/conf.d.templates diff --git a/docker/nginx/conf.d/include/init-optional-variables b/docker/nginx/conf.d/include/init-optional-variables index 4f1be02b..e2e74030 100644 --- a/docker/nginx/conf.d/include/init-optional-variables +++ b/docker/nginx/conf.d/include/init-optional-variables @@ -7,3 +7,6 @@ set $hns_domain ''; # set only if server has been access through SERVER_DOMAIN set $server_alias ''; + +# expose skylink variable so we can use it in access log +set $skylink ''; diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index a3692d00..143ca108 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -11,6 +11,10 @@ if ($request_method = PURGE) { limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time +# ensure that skylink that we pass around is base64 encoded (transform base32 encoded ones) +# this is important because we want only one format in cache keys and logs +set_by_lua_block $skylink { return require("skynet.skylink").parse(ngx.var.skylink) } + # $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was: # a) skylink v1 - it would not matter, no additional logic is executed # b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 9f966d0d..cd925429 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -231,9 +231,6 @@ location /skynet/metadata { header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") - - -- do not expose internal header - ngx.header["Skynet-Requested-Skylink"] = "" } proxy_set_header User-Agent: Sia-Agent; @@ -246,9 +243,6 @@ location /skynet/resolve { header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") - - -- do not expose internal header - ngx.header["Skynet-Requested-Skylink"] = "" } proxy_set_header User-Agent: Sia-Agent; diff --git a/docker/nginx/conf.d/server/server.dnslink b/docker/nginx/conf.d/server/server.dnslink index 1dd3a489..32e454cc 100644 --- a/docker/nginx/conf.d/server/server.dnslink +++ b/docker/nginx/conf.d/server/server.dnslink @@ -37,6 +37,7 @@ location / { ngx.var.skylink = cache_value end + ngx.var.skylink = require("skynet.skylink").parse(ngx.var.skylink) ngx.var.skylink_v1 = ngx.var.skylink ngx.var.skylink_v2 = ngx.var.skylink } diff --git a/docker/nginx/libs/skynet/skylink.lua b/docker/nginx/libs/skynet/skylink.lua new file mode 100644 index 00000000..f059249e --- /dev/null +++ b/docker/nginx/libs/skynet/skylink.lua @@ -0,0 +1,61 @@ +local _M = {} + +local function str_split(str, size) + local result = {} + for i=1, #str, size do + table.insert(result, str:sub(i, i + size - 1)) + end + return result +end + +local function dec2bin(num) + local result = "" + repeat + local halved = num / 2 + local int, frac = math.modf(halved) + num = int + result = math.ceil(frac) .. result + until num == 0 + return result +end + +local base32Alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUV" + +-- this is not a full implementation of base32 decoder, it doesn't +-- cover all edge cases but it works good enough for decoding skylinks +-- source: https://gist.github.com/linsonder6/3202fc06832f97905aab2a8a492a80de +function decodeBase32Skylink(str) + local binary = string.upper(str):gsub(".", function (char) + if char == "=" then return "" end + local pos = string.find(base32Alphabet, char) + pos = pos - 1 + return string.format("%05u", dec2bin(pos)) + end) + + local bytes = str_split(binary, 8) + + local decoded = {} + for _, byte in pairs(bytes) do + table.insert(decoded, string.char(tonumber(byte, 2))) + end + + local concatenated = table.concat(decoded) + + -- decoded skylink is always 34 bytes, anything + -- else is leftover and should be discarded + return string.sub(concatenated, 1, 34) +end + +-- parse any skylink and return base64 version +function _M.parse(skylink) + if string.len(skylink) == 55 then + local base64 = require("ngx.base64") + local decoded = decodeBase32Skylink(skylink) + + return base64.encode_base64url(decoded) + end + + return skylink +end + +return _M diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index c1f92d92..755bd003 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -38,6 +38,8 @@ http { include mime.types; default_type application/octet-stream; + lua_package_path "/etc/nginx/libs/?.lua;;"; + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" $upstream_response_time ' @@ -45,7 +47,7 @@ http { '"$upstream_http_content_type" "$upstream_cache_status" ' '"$server_alias" "$sent_http_skynet_skylink" ' '$upstream_connect_time $upstream_header_time ' - '$request_time "$hns_domain"'; + '$request_time "$hns_domain" "$skylink"'; access_log logs/access.log main; @@ -72,8 +74,10 @@ http { # this runs before forking out nginx worker processes init_by_lua_block { - require "cjson" + require "cjson" + require "ngx.base64" require "resty.http" + require "skynet.skylink" } # include skynet-portal-api and skynet-server-api header on every request From c4e771759bbc3f09849a024d07c0aa4ac9ecfbb9 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 5 Nov 2021 15:32:32 +0100 Subject: [PATCH 2/2] unit tests --- .github/workflows/nginx-lua-unit-tests.yml | 32 +++ docker/nginx/conf.d/server/server.api | 6 + docker/nginx/libs/basexx.lua | 301 +++++++++++++++++++++ docker/nginx/libs/skynet/skylink.lua | 51 +--- docker/nginx/libs/skynet/skylink.spec.lua | 14 + docker/nginx/nginx.conf | 1 - 6 files changed, 356 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/nginx-lua-unit-tests.yml create mode 100644 docker/nginx/libs/basexx.lua create mode 100644 docker/nginx/libs/skynet/skylink.spec.lua diff --git a/.github/workflows/nginx-lua-unit-tests.yml b/.github/workflows/nginx-lua-unit-tests.yml new file mode 100644 index 00000000..d0c71aa1 --- /dev/null +++ b/.github/workflows/nginx-lua-unit-tests.yml @@ -0,0 +1,32 @@ +# Install and run unit tests with busted +# Docs: http://olivinelabs.com/busted/ + +name: Nginx Lua Unit Tests + +on: + pull_request: + paths: + - "docker/nginx/libs/**.lua" + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: "3.x" + architecture: "x64" + + - name: Install Dependencies + run: | + pip install hererocks + hererocks env --lua=5.1 -rlatest + source env/bin/activate + luarocks install busted + + - name: Unit Tests + run: | + source env/bin/activate + busted --verbose --pattern=spec --directory=docker/nginx/libs . diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index cd925429..9f966d0d 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -231,6 +231,9 @@ location /skynet/metadata { header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") + + -- do not expose internal header + ngx.header["Skynet-Requested-Skylink"] = "" } proxy_set_header User-Agent: Sia-Agent; @@ -243,6 +246,9 @@ location /skynet/resolve { header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") + + -- do not expose internal header + ngx.header["Skynet-Requested-Skylink"] = "" } proxy_set_header User-Agent: Sia-Agent; diff --git a/docker/nginx/libs/basexx.lua b/docker/nginx/libs/basexx.lua new file mode 100644 index 00000000..b077ee9a --- /dev/null +++ b/docker/nginx/libs/basexx.lua @@ -0,0 +1,301 @@ +-- source: https://github.com/aiq/basexx +-- license: MIT +-- modified: exposed from_basexx and to_basexx generic functions + +-------------------------------------------------------------------------------- +-- util functions +-------------------------------------------------------------------------------- + +local function divide_string( str, max ) + local result = {} + + local start = 1 + for i = 1, #str do + if i % max == 0 then + table.insert( result, str:sub( start, i ) ) + start = i + 1 + elseif i == #str then + table.insert( result, str:sub( start, i ) ) + end + end + + return result +end + +local function number_to_bit( num, length ) + local bits = {} + + while num > 0 do + local rest = math.floor( math.fmod( num, 2 ) ) + table.insert( bits, rest ) + num = ( num - rest ) / 2 + end + + while #bits < length do + table.insert( bits, "0" ) + end + + return string.reverse( table.concat( bits ) ) +end + +local function ignore_set( str, set ) + if set then + str = str:gsub( "["..set.."]", "" ) + end + return str +end + +local function pure_from_bit( str ) + return ( str:gsub( '........', function ( cc ) + return string.char( tonumber( cc, 2 ) ) + end ) ) +end + +local function unexpected_char_error( str, pos ) + local c = string.sub( str, pos, pos ) + return string.format( "unexpected character at position %d: '%s'", pos, c ) +end + +-------------------------------------------------------------------------------- + +local basexx = {} + +-------------------------------------------------------------------------------- +-- base2(bitfield) decode and encode function +-------------------------------------------------------------------------------- + +local bitMap = { o = "0", i = "1", l = "1" } + +function basexx.from_bit( str, ignore ) + str = ignore_set( str, ignore ) + str = string.lower( str ) + str = str:gsub( '[ilo]', function( c ) return bitMap[ c ] end ) + local pos = string.find( str, "[^01]" ) + if pos then return nil, unexpected_char_error( str, pos ) end + + return pure_from_bit( str ) +end + +function basexx.to_bit( str ) + return ( str:gsub( '.', function ( c ) + local byte = string.byte( c ) + local bits = {} + for _ = 1,8 do + table.insert( bits, byte % 2 ) + byte = math.floor( byte / 2 ) + end + return table.concat( bits ):reverse() + end ) ) +end + +-------------------------------------------------------------------------------- +-- base16(hex) decode and encode function +-------------------------------------------------------------------------------- + +function basexx.from_hex( str, ignore ) + str = ignore_set( str, ignore ) + local pos = string.find( str, "[^%x]" ) + if pos then return nil, unexpected_char_error( str, pos ) end + + return ( str:gsub( '..', function ( cc ) + return string.char( tonumber( cc, 16 ) ) + end ) ) +end + +function basexx.to_hex( str ) + return ( str:gsub( '.', function ( c ) + return string.format('%02X', string.byte( c ) ) + end ) ) +end + +-------------------------------------------------------------------------------- +-- generic function to decode and encode base32/base64 +-------------------------------------------------------------------------------- + +function basexx.from_basexx( str, alphabet, bits ) + local result = {} + for i = 1, #str do + local c = string.sub( str, i, i ) + if c ~= '=' then + local index = string.find( alphabet, c, 1, true ) + if not index then + return nil, unexpected_char_error( str, i ) + end + table.insert( result, number_to_bit( index - 1, bits ) ) + end + end + + local value = table.concat( result ) + local pad = #value % 8 + return pure_from_bit( string.sub( value, 1, #value - pad ) ) +end + +function basexx.to_basexx( str, alphabet, bits, pad ) + local bitString = basexx.to_bit( str ) + + local chunks = divide_string( bitString, bits ) + local result = {} + for _,value in ipairs( chunks ) do + if ( #value < bits ) then + value = value .. string.rep( '0', bits - #value ) + end + local pos = tonumber( value, 2 ) + 1 + table.insert( result, alphabet:sub( pos, pos ) ) + end + + table.insert( result, pad ) + return table.concat( result ) +end + +-------------------------------------------------------------------------------- +-- rfc 3548: http://www.rfc-editor.org/rfc/rfc3548.txt +-------------------------------------------------------------------------------- + +local base32Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" +local base32PadMap = { "", "======", "====", "===", "=" } + +function basexx.from_base32( str, ignore ) + str = ignore_set( str, ignore ) + return basexx.from_basexx( string.upper( str ), base32Alphabet, 5 ) +end + +function basexx.to_base32( str ) + return basexx.to_basexx( str, base32Alphabet, 5, base32PadMap[ #str % 5 + 1 ] ) +end + +-------------------------------------------------------------------------------- +-- crockford: http://www.crockford.com/wrmg/base32.html +-------------------------------------------------------------------------------- + +local crockfordAlphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" +local crockfordMap = { O = "0", I = "1", L = "1" } + +function basexx.from_crockford( str, ignore ) + str = ignore_set( str, ignore ) + str = string.upper( str ) + str = str:gsub( '[ILOU]', function( c ) return crockfordMap[ c ] end ) + return basexx.from_basexx( str, crockfordAlphabet, 5 ) +end + +function basexx.to_crockford( str ) + return basexx.to_basexx( str, crockfordAlphabet, 5, "" ) +end + +-------------------------------------------------------------------------------- +-- base64 decode and encode function +-------------------------------------------------------------------------------- + +local base64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".. + "abcdefghijklmnopqrstuvwxyz".. + "0123456789+/" +local base64PadMap = { "", "==", "=" } + +function basexx.from_base64( str, ignore ) + str = ignore_set( str, ignore ) + return basexx.from_basexx( str, base64Alphabet, 6 ) +end + +function basexx.to_base64( str ) + return basexx.to_basexx( str, base64Alphabet, 6, base64PadMap[ #str % 3 + 1 ] ) +end + +-------------------------------------------------------------------------------- +-- URL safe base64 decode and encode function +-------------------------------------------------------------------------------- + +local url64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".. + "abcdefghijklmnopqrstuvwxyz".. + "0123456789-_" + +function basexx.from_url64( str, ignore ) + str = ignore_set( str, ignore ) + return basexx.from_basexx( str, url64Alphabet, 6 ) +end + +function basexx.to_url64( str ) + return basexx.to_basexx( str, url64Alphabet, 6, "" ) +end + +-------------------------------------------------------------------------------- +-- +-------------------------------------------------------------------------------- + +local function length_error( len, d ) + return string.format( "invalid length: %d - must be a multiple of %d", len, d ) +end + +local z85Decoder = { 0x00, 0x44, 0x00, 0x54, 0x53, 0x52, 0x48, 0x00, + 0x4B, 0x4C, 0x46, 0x41, 0x00, 0x3F, 0x3E, 0x45, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x40, 0x00, 0x49, 0x42, 0x4A, 0x47, + 0x51, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, + 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, + 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, + 0x3B, 0x3C, 0x3D, 0x4D, 0x00, 0x4E, 0x43, 0x00, + 0x00, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, + 0x21, 0x22, 0x23, 0x4F, 0x00, 0x50, 0x00, 0x00 } + +function basexx.from_z85( str, ignore ) + str = ignore_set( str, ignore ) + if ( #str % 5 ) ~= 0 then + return nil, length_error( #str, 5 ) + end + + local result = {} + + local value = 0 + for i = 1, #str do + local index = string.byte( str, i ) - 31 + if index < 1 or index >= #z85Decoder then + return nil, unexpected_char_error( str, i ) + end + value = ( value * 85 ) + z85Decoder[ index ] + if ( i % 5 ) == 0 then + local divisor = 256 * 256 * 256 + while divisor ~= 0 do + local b = math.floor( value / divisor ) % 256 + table.insert( result, string.char( b ) ) + divisor = math.floor( divisor / 256 ) + end + value = 0 + end + end + + return table.concat( result ) +end + +local z85Encoder = "0123456789".. + "abcdefghijklmnopqrstuvwxyz".. + "ABCDEFGHIJKLMNOPQRSTUVWXYZ".. + ".-:+=^!/*?&<>()[]{}@%$#" + +function basexx.to_z85( str ) + if ( #str % 4 ) ~= 0 then + return nil, length_error( #str, 4 ) + end + + local result = {} + + local value = 0 + for i = 1, #str do + local b = string.byte( str, i ) + value = ( value * 256 ) + b + if ( i % 4 ) == 0 then + local divisor = 85 * 85 * 85 * 85 + while divisor ~= 0 do + local index = ( math.floor( value / divisor ) % 85 ) + 1 + table.insert( result, z85Encoder:sub( index, index ) ) + divisor = math.floor( divisor / 85 ) + end + value = 0 + end + end + + return table.concat( result ) +end + +-------------------------------------------------------------------------------- + +return basexx diff --git a/docker/nginx/libs/skynet/skylink.lua b/docker/nginx/libs/skynet/skylink.lua index f059249e..d3b61d36 100644 --- a/docker/nginx/libs/skynet/skylink.lua +++ b/docker/nginx/libs/skynet/skylink.lua @@ -1,58 +1,13 @@ local _M = {} -local function str_split(str, size) - local result = {} - for i=1, #str, size do - table.insert(result, str:sub(i, i + size - 1)) - end - return result -end - -local function dec2bin(num) - local result = "" - repeat - local halved = num / 2 - local int, frac = math.modf(halved) - num = int - result = math.ceil(frac) .. result - until num == 0 - return result -end - -local base32Alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUV" - --- this is not a full implementation of base32 decoder, it doesn't --- cover all edge cases but it works good enough for decoding skylinks --- source: https://gist.github.com/linsonder6/3202fc06832f97905aab2a8a492a80de -function decodeBase32Skylink(str) - local binary = string.upper(str):gsub(".", function (char) - if char == "=" then return "" end - local pos = string.find(base32Alphabet, char) - pos = pos - 1 - return string.format("%05u", dec2bin(pos)) - end) - - local bytes = str_split(binary, 8) - - local decoded = {} - for _, byte in pairs(bytes) do - table.insert(decoded, string.char(tonumber(byte, 2))) - end - - local concatenated = table.concat(decoded) - - -- decoded skylink is always 34 bytes, anything - -- else is leftover and should be discarded - return string.sub(concatenated, 1, 34) -end +local basexx = require("basexx") -- parse any skylink and return base64 version function _M.parse(skylink) if string.len(skylink) == 55 then - local base64 = require("ngx.base64") - local decoded = decodeBase32Skylink(skylink) + local decoded = basexx.from_basexx(string.upper(skylink), "0123456789ABCDEFGHIJKLMNOPQRSTUV", 5) - return base64.encode_base64url(decoded) + return basexx.to_url64(decoded) end return skylink diff --git a/docker/nginx/libs/skynet/skylink.spec.lua b/docker/nginx/libs/skynet/skylink.spec.lua new file mode 100644 index 00000000..b7d3733c --- /dev/null +++ b/docker/nginx/libs/skynet/skylink.spec.lua @@ -0,0 +1,14 @@ +skylink = require("skynet/skylink") + +describe("parse", function() + local base32 = "0404dsjvti046fsua4ktor9grrpe76erq9jot9cvopbhsvsu76r4r30" + local base64 = "AQBG8n_sgEM_nlEp3G0w3vLjmdvSZ46ln8ZXHn-eObZNjA" + + it("should return unchanged base64 skylink", function() + assert.is.same(skylink.parse(base64), base64) + end) + + it("should transform base32 skylink into base64", function() + assert.is.same(skylink.parse(base32), base64) + end) +end) diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index 755bd003..a6883cb8 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -75,7 +75,6 @@ http { # this runs before forking out nginx worker processes init_by_lua_block { require "cjson" - require "ngx.base64" require "resty.http" require "skynet.skylink" }