Merge pull request #1349 from SkynetLabs/base-32-lua-converter

implement base32 decoder

.github/workflows/nginx-lua-unit-tests.yml

@@ -0,0 +1,32 @@
+# Install and run unit tests with busted
+# Docs: http://olivinelabs.com/busted/
+
+name: Nginx Lua Unit Tests
+
+on:
+  pull_request:
+    paths:
+      - "docker/nginx/libs/**.lua"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+          architecture: "x64"
+
+      - name: Install Dependencies
+        run: |
+          pip install hererocks
+          hererocks env --lua=5.1 -rlatest
+          source env/bin/activate
+          luarocks install busted
+
+      - name: Unit Tests
+        run: |
+          source env/bin/activate
+          busted --verbose --pattern=spec --directory=docker/nginx/libs .

docker/nginx/Dockerfile

@@ -7,6 +7,7 @@ RUN luarocks install lua-resty-http && \
     -out /etc/ssl/local-certificate.crt
 
 COPY mo ./
+COPY libs /etc/nginx/libs
 COPY conf.d /etc/nginx/conf.d
 COPY conf.d.templates /etc/nginx/conf.d.templates
 

docker/nginx/conf.d/include/init-optional-variables

@@ -7,3 +7,6 @@ set $hns_domain '';
 
 # set only if server has been access through SERVER_DOMAIN
 set $server_alias '';
+
+# expose skylink variable so we can use it in access log
+set $skylink '';

docker/nginx/conf.d/include/location-skylink

@@ -11,6 +11,10 @@ if ($request_method = PURGE) {
 
 limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
 
+# ensure that skylink that we pass around is base64 encoded (transform base32 encoded ones)
+# this is important because we want only one format in cache keys and logs
+set_by_lua_block $skylink { return require("skynet.skylink").parse(ngx.var.skylink) }
+
 # $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was:
 # a) skylink v1 - it would not matter, no additional logic is executed
 # b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update 

docker/nginx/conf.d/server/server.dnslink

@@ -37,6 +37,7 @@ location / {
             ngx.var.skylink = cache_value
         end
 
+        ngx.var.skylink = require("skynet.skylink").parse(ngx.var.skylink)
         ngx.var.skylink_v1 = ngx.var.skylink
         ngx.var.skylink_v2 = ngx.var.skylink
     }

docker/nginx/libs/basexx.lua

@@ -0,0 +1,301 @@
+-- source: https://github.com/aiq/basexx
+-- license: MIT
+-- modified: exposed from_basexx and to_basexx generic functions
+
+--------------------------------------------------------------------------------
+-- util functions
+--------------------------------------------------------------------------------
+
+local function divide_string( str, max )
+   local result = {}
+
+   local start = 1
+   for i = 1, #str do
+      if i % max == 0 then
+         table.insert( result, str:sub( start, i ) )
+         start = i + 1
+      elseif i == #str then
+         table.insert( result, str:sub( start, i ) )
+      end
+   end
+
+   return result
+end
+ 
+local function number_to_bit( num, length )
+   local bits = {}
+
+   while num > 0 do
+      local rest = math.floor( math.fmod( num, 2 ) )
+      table.insert( bits, rest )
+      num = ( num - rest ) / 2
+   end
+
+   while #bits < length do
+      table.insert( bits, "0" )
+   end
+
+   return string.reverse( table.concat( bits ) )
+end
+
+local function ignore_set( str, set )
+   if set then
+      str = str:gsub( "["..set.."]", "" )
+   end
+   return str
+end
+
+local function pure_from_bit( str )
+   return ( str:gsub( '........', function ( cc )
+               return string.char( tonumber( cc, 2 ) )
+            end ) )
+end
+
+local function unexpected_char_error( str, pos )
+   local c = string.sub( str, pos, pos )
+   return string.format( "unexpected character at position %d: '%s'", pos, c )
+end
+
+--------------------------------------------------------------------------------
+
+local basexx = {}
+
+--------------------------------------------------------------------------------
+-- base2(bitfield) decode and encode function
+--------------------------------------------------------------------------------
+
+local bitMap = { o = "0", i = "1", l = "1" }
+
+function basexx.from_bit( str, ignore )
+   str = ignore_set( str, ignore )
+   str = string.lower( str )
+   str = str:gsub( '[ilo]', function( c ) return bitMap[ c ] end )
+   local pos = string.find( str, "[^01]" )
+   if pos then return nil, unexpected_char_error( str, pos ) end
+
+   return pure_from_bit( str )
+end
+
+function basexx.to_bit( str )
+   return ( str:gsub( '.', function ( c )
+               local byte = string.byte( c )
+               local bits = {}
+               for _ = 1,8 do
+                  table.insert( bits, byte % 2 )
+                  byte = math.floor( byte / 2 )
+               end
+               return table.concat( bits ):reverse()
+            end ) )
+end
+
+--------------------------------------------------------------------------------
+-- base16(hex) decode and encode function
+--------------------------------------------------------------------------------
+
+function basexx.from_hex( str, ignore )
+   str = ignore_set( str, ignore )
+   local pos = string.find( str, "[^%x]" )
+   if pos then return nil, unexpected_char_error( str, pos ) end
+
+   return ( str:gsub( '..', function ( cc )
+               return string.char( tonumber( cc, 16 ) )
+            end ) )
+end
+
+function basexx.to_hex( str )
+   return ( str:gsub( '.', function ( c )
+               return string.format('%02X', string.byte( c ) )
+            end ) )
+end
+
+--------------------------------------------------------------------------------
+-- generic function to decode and encode base32/base64
+--------------------------------------------------------------------------------
+
+function basexx.from_basexx( str, alphabet, bits )
+   local result = {}
+   for i = 1, #str do
+      local c = string.sub( str, i, i )
+      if c ~= '=' then
+         local index = string.find( alphabet, c, 1, true )
+         if not index then
+            return nil, unexpected_char_error( str, i )
+         end
+         table.insert( result, number_to_bit( index - 1, bits ) )
+      end
+   end
+
+   local value = table.concat( result )
+   local pad = #value % 8
+   return pure_from_bit( string.sub( value, 1, #value - pad ) )
+end
+
+function basexx.to_basexx( str, alphabet, bits, pad )
+   local bitString = basexx.to_bit( str )
+
+   local chunks = divide_string( bitString, bits )
+   local result = {}
+   for _,value in ipairs( chunks ) do
+      if ( #value < bits ) then
+         value = value .. string.rep( '0', bits - #value )
+      end
+      local pos = tonumber( value, 2 ) + 1
+      table.insert( result, alphabet:sub( pos, pos ) )
+   end
+
+   table.insert( result, pad )
+   return table.concat( result )   
+end
+
+--------------------------------------------------------------------------------
+-- rfc 3548: http://www.rfc-editor.org/rfc/rfc3548.txt
+--------------------------------------------------------------------------------
+
+local base32Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+local base32PadMap = { "", "======", "====", "===", "=" }
+
+function basexx.from_base32( str, ignore )
+   str = ignore_set( str, ignore )
+   return basexx.from_basexx( string.upper( str ), base32Alphabet, 5 )
+end
+
+function basexx.to_base32( str )
+   return basexx.to_basexx( str, base32Alphabet, 5, base32PadMap[ #str % 5 + 1 ] )
+end
+
+--------------------------------------------------------------------------------
+-- crockford: http://www.crockford.com/wrmg/base32.html
+--------------------------------------------------------------------------------
+
+local crockfordAlphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
+local crockfordMap = { O = "0", I = "1", L = "1" }
+
+function basexx.from_crockford( str, ignore )
+   str = ignore_set( str, ignore )
+   str = string.upper( str )
+   str = str:gsub( '[ILOU]', function( c ) return crockfordMap[ c ] end )
+   return basexx.from_basexx( str, crockfordAlphabet, 5 )
+end
+
+function basexx.to_crockford( str )
+   return basexx.to_basexx( str, crockfordAlphabet, 5, "" )
+end
+
+--------------------------------------------------------------------------------
+-- base64 decode and encode function
+--------------------------------------------------------------------------------
+
+local base64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"..
+                     "abcdefghijklmnopqrstuvwxyz"..
+                     "0123456789+/"
+local base64PadMap = { "", "==", "=" }
+
+function basexx.from_base64( str, ignore )
+   str = ignore_set( str, ignore )
+   return basexx.from_basexx( str, base64Alphabet, 6 )
+end
+
+function basexx.to_base64( str )
+   return basexx.to_basexx( str, base64Alphabet, 6, base64PadMap[ #str % 3 + 1 ] )
+end
+
+--------------------------------------------------------------------------------
+-- URL safe base64 decode and encode function
+--------------------------------------------------------------------------------
+
+local url64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"..
+                     "abcdefghijklmnopqrstuvwxyz"..
+                     "0123456789-_"
+
+function basexx.from_url64( str, ignore )
+   str = ignore_set( str, ignore )
+   return basexx.from_basexx( str, url64Alphabet, 6 )
+end
+
+function basexx.to_url64( str )
+   return basexx.to_basexx( str, url64Alphabet, 6, "" )
+end
+
+--------------------------------------------------------------------------------
+--
+--------------------------------------------------------------------------------
+
+local function length_error( len, d )
+   return string.format( "invalid length: %d - must be a multiple of %d", len, d )
+end
+
+local z85Decoder = { 0x00, 0x44, 0x00, 0x54, 0x53, 0x52, 0x48, 0x00,
+                     0x4B, 0x4C, 0x46, 0x41, 0x00, 0x3F, 0x3E, 0x45, 
+                     0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 
+                     0x08, 0x09, 0x40, 0x00, 0x49, 0x42, 0x4A, 0x47, 
+                     0x51, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 
+                     0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 
+                     0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 
+                     0x3B, 0x3C, 0x3D, 0x4D, 0x00, 0x4E, 0x43, 0x00, 
+                     0x00, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 
+                     0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 
+                     0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 
+                     0x21, 0x22, 0x23, 0x4F, 0x00, 0x50, 0x00, 0x00 }
+
+function basexx.from_z85( str, ignore )
+   str = ignore_set( str, ignore )
+   if ( #str % 5 ) ~= 0 then
+      return nil, length_error( #str, 5 )
+   end
+
+   local result = {}
+
+   local value = 0
+   for i = 1, #str do
+      local index = string.byte( str, i ) - 31
+      if index < 1 or index >= #z85Decoder then
+         return nil, unexpected_char_error( str, i )
+      end
+      value = ( value * 85 ) + z85Decoder[ index ]
+      if ( i % 5 ) == 0 then
+         local divisor = 256 * 256 * 256
+         while divisor ~= 0 do
+            local b = math.floor( value / divisor ) % 256
+            table.insert( result, string.char( b ) )
+            divisor = math.floor( divisor / 256 )
+         end
+         value = 0
+      end
+   end
+
+   return table.concat( result )
+end
+
+local z85Encoder = "0123456789"..
+                  "abcdefghijklmnopqrstuvwxyz"..
+                  "ABCDEFGHIJKLMNOPQRSTUVWXYZ"..
+                  ".-:+=^!/*?&<>()[]{}@%$#"
+
+function basexx.to_z85( str )
+   if ( #str % 4 ) ~= 0 then
+      return nil, length_error( #str, 4 )
+   end
+
+   local result = {}
+
+   local value = 0
+   for i = 1, #str do
+      local b = string.byte( str, i )
+      value = ( value * 256 ) + b
+      if ( i % 4 ) == 0 then
+         local divisor = 85 * 85 * 85 * 85
+         while divisor ~= 0 do
+            local index = ( math.floor( value / divisor ) % 85 ) + 1
+            table.insert( result, z85Encoder:sub( index, index ) )
+            divisor = math.floor( divisor / 85 )
+         end
+         value = 0
+      end
+   end
+
+   return table.concat( result )
+end
+
+--------------------------------------------------------------------------------
+
+return basexx

docker/nginx/libs/skynet/skylink.lua

@@ -0,0 +1,16 @@
+local _M = {}
+
+local basexx = require("basexx")
+
+-- parse any skylink and return base64 version
+function _M.parse(skylink)
+    if string.len(skylink) == 55 then
+        local decoded = basexx.from_basexx(string.upper(skylink), "0123456789ABCDEFGHIJKLMNOPQRSTUV", 5)
+
+        return basexx.to_url64(decoded)
+    end
+
+    return skylink
+end
+
+return _M

docker/nginx/libs/skynet/skylink.spec.lua

@@ -0,0 +1,14 @@
+skylink = require("skynet/skylink")
+
+describe("parse", function()
+   local base32 = "0404dsjvti046fsua4ktor9grrpe76erq9jot9cvopbhsvsu76r4r30"
+   local base64 = "AQBG8n_sgEM_nlEp3G0w3vLjmdvSZ46ln8ZXHn-eObZNjA"
+
+   it("should return unchanged base64 skylink", function()
+      assert.is.same(skylink.parse(base64), base64)
+   end)
+   
+   it("should transform base32 skylink into base64", function()
+      assert.is.same(skylink.parse(base32), base64)
+   end)
+end)

docker/nginx/nginx.conf

@@ -38,6 +38,8 @@ http {
     include       mime.types;
     default_type  application/octet-stream;
 
+    lua_package_path "/etc/nginx/libs/?.lua;;";
+
     log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
         '$status $body_bytes_sent "$http_referer" '
         '"$http_user_agent" $upstream_response_time '
@@ -45,7 +47,7 @@ http {
         '"$upstream_http_content_type" "$upstream_cache_status" '
         '"$server_alias" "$sent_http_skynet_skylink" '
         '$upstream_connect_time $upstream_header_time '
-        '$request_time "$hns_domain"';
+        '$request_time "$hns_domain" "$skylink"';
 
     access_log  logs/access.log  main;
 
@@ -74,6 +76,7 @@ http {
     init_by_lua_block { 
         require "cjson"
         require "resty.http"
+        require "skynet.skylink"
     }
 
     # include skynet-portal-api and skynet-server-api header on every request