Merge pull request #1478 from SkynetLabs/blake-hashing-lua

no delay blocklisting

.github/workflows/nginx-lua-unit-tests.yml

@@ -25,6 +25,7 @@ jobs:
           hererocks env --lua=5.1 -rlatest
           source env/bin/activate
           luarocks install busted
+          luarocks install hasher
 
       - name: Unit Tests
         run: |

docker/nginx/Dockerfile

@@ -1,6 +1,7 @@
 FROM openresty/openresty:1.19.9.1-bionic
 
 RUN luarocks install lua-resty-http && \
+    luarocks install hasher && \
     openssl req -new -newkey rsa:2048 -days 3650 -nodes -x509 \
     -subj '/CN=local-certificate' \
     -keyout /etc/ssl/local-certificate.key \

docker/nginx/conf.d/include/local-network-only

@@ -0,0 +1,3 @@
+allow 127.0.0.1/32;  # localhost
+allow 10.10.10.0/24; # docker network
+deny all;

docker/nginx/conf.d/include/location-skylink

@@ -57,6 +57,14 @@ access_by_lua_block {
         ngx.var.skynet_proof = res.headers["Skynet-Proof"]
     end
 
+    -- check if skylink v1 is present on blocklist (compare hashes)
+    if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then
+        ngx.status = ngx.HTTP_ILLEGAL
+        ngx.header["content-type"] = "text/plain"
+        ngx.say("Unavailable For Legal Reasons")
+        return ngx.exit(ngx.status)
+    end
+
     -- this block runs only when accounts are enabled
     if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end
 

docker/nginx/conf.d/include/sia-auth

@@ -1,15 +1,4 @@
 rewrite_by_lua_block {
-    local b64 = require("ngx.base64")
-    -- open apipassword file for reading (b flag is required for some reason)
-    -- (file /etc/.sia/apipassword has to be mounted from the host system)
-    local apipassword_file = io.open("/data/sia/apipassword", "rb")
-    -- read apipassword file contents and trim newline (important)
-    local apipassword = apipassword_file:read("*all"):gsub("%s+", "")
-    -- make sure to close file after reading the password
-    apipassword_file.close()
-    -- encode the user:password authorization string
-    -- (in our case user is empty so it is just :password)
-    local content = b64.encode_base64url(":" .. apipassword)
-    -- set authorization header with proper base64 encoded string
-    ngx.req.set_header("Authorization", "Basic " .. content)
+    -- set basic authorization header with base64 encoded apipassword
+    ngx.req.set_header("Authorization", require("skynet.utils").authorization_header())
 }

docker/nginx/conf.d/server.local.conf

@@ -0,0 +1,10 @@
+server {
+    # local server - do not expose this port externally
+    listen 8000;
+    listen [::]:8000;
+
+    # secure traffic by limiting to only local networks
+    include /etc/nginx/conf.d/include/local-network-only;
+
+    include /etc/nginx/conf.d/server/server.local;
+}

docker/nginx/conf.d/server/server.api

@@ -334,11 +334,8 @@ location ~ "^/file/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" {
 }
 
 location @purge {
-    allow 10.0.0.0/8;
-    allow 127.0.0.1/32;
-    allow 172.16.0.0/12;
-    allow 192.168.0.0/16;
-    deny all;
+    # secure traffic by limiting to only local networks
+    include /etc/nginx/conf.d/include/local-network-only;
 
     set $lua_purge_path "/data/nginx/cache/";
     content_by_lua_file /etc/nginx/conf.d/scripts/purge-multi.lua;

docker/nginx/conf.d/server/server.local

@@ -0,0 +1,34 @@
+include /etc/nginx/conf.d/include/init-optional-variables;
+
+location /skynet/blocklist {
+    content_by_lua_block {        
+        local httpc = require("resty.http").new()
+
+        ngx.req.read_body() -- ensure the post body data is read before using get_body_data
+
+        -- proxy blocklist update request
+        -- 10.10.10.10 points to sia service (alias not available when using resty-http)
+        local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/blocklist", {
+            method = "POST",
+            body = ngx.req.get_body_data(),
+            headers = {
+                ["Content-Type"] = "application/x-www-form-urlencoded",
+                ["Authorization"] = require("skynet.utils").authorization_header(),
+                ["User-Agent"] = "Sia-Agent",
+            }
+        })
+
+        -- print error and exit with 500 or exit with response if status is not 204
+        if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then
+            ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
+            ngx.header["content-type"] = "text/plain"
+            ngx.say(err or res.body)
+            return ngx.exit(ngx.status)
+        end
+
+        require("skynet.blocklist").reload()
+
+        ngx.status = ngx.HTTP_NO_CONTENT
+        return ngx.exit(ngx.status)
+    }
+}

docker/nginx/libs/skynet/blocklist.lua

@@ -0,0 +1,55 @@
+local _M = {}
+
+function _M.reload()
+    local httpc = require("resty.http").new()
+
+    -- fetch blocklist records (all blocked skylink hashes)
+    -- 10.10.10.10 points to sia service (alias not available when using resty-http)
+    local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/blocklist", {
+        headers = {
+            ["User-Agent"] = "Sia-Agent",
+        }
+    })
+
+    -- fail whole request in case this request failed, we want to make sure
+    -- the blocklist is pre cached before serving first skylink
+    if err or (res and res.status ~= ngx.HTTP_OK) then
+        ngx.log(ngx.ERR, "Failed skyd service request /skynet/blocklist: ", err or ("[HTTP " .. res.status .. "] " .. res.body))
+        ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
+        ngx.header["content-type"] = "text/plain"
+        ngx.say(err or res.body)
+        return ngx.exit(ngx.status)
+    elseif res and res.status == ngx.HTTP_OK then
+        local json = require('cjson')
+        local data = json.decode(res.body)
+
+        -- mark all existing entries as expired
+        ngx.shared.blocklist:flush_all()
+
+        -- set all cache entries one by one (resets expiration)
+        for i, hash in ipairs(data.blocklist) do
+            ngx.shared.blocklist:set(hash, true)
+        end
+
+        -- ensure that init flag is persisted
+        ngx.shared.blocklist:set("__init", true)
+
+        -- remove all leftover expired entries
+        ngx.shared.blocklist:flush_expired()
+    end
+end
+
+function _M.is_blocked(skylink)
+    -- make sure that blocklist has been preloaded
+    if not ngx.shared.blocklist:get("__init") then _M.reload() end
+
+    -- hash skylink before comparing it with blocklist
+    local hash = require("skynet.skylink").hash(skylink)
+
+    -- we need to use get_stale because we are expiring previous
+    -- entries when the blocklist is reloading and we still want
+    -- to block them until the reloading is finished
+    return ngx.shared.blocklist:get_stale(hash) == true
+end
+
+return _M

docker/nginx/libs/skynet/skylink.lua

@@ -1,6 +1,7 @@
 local _M = {}
 
 local basexx = require("basexx")
+local hasher = require("hasher")
 
 -- parse any skylink and return base64 version
 function _M.parse(skylink)
@@ -13,4 +14,27 @@ function _M.parse(skylink)
     return skylink
 end
 
+-- hash skylink into 32 bytes hash used in blocklist
+function _M.hash(skylink)
+    -- ensure that the skylink is base64 encoded
+    local base64Skylink = _M.parse(skylink)
+
+    -- decode skylink from base64 encoding
+    local rawSkylink = basexx.from_url64(base64Skylink)
+
+    -- drop first two bytes and leave just merkle root
+    local rawMerkleRoot = string.sub(rawSkylink, 3)
+
+    -- parse with blake2b with key length of 32
+    local blake2bHashed = hasher.blake2b(rawMerkleRoot, 32)
+    
+    -- hex encode the blake hash
+    local hexHashed = basexx.to_hex(blake2bHashed)
+
+    -- lowercase the hex encoded hash
+    local lowerHexHashed = string.lower(hexHashed)
+
+    return lowerHexHashed
+end
+
 return _M

docker/nginx/libs/skynet/skylink.spec.lua

@@ -1,14 +1,23 @@
-skylink = require("skynet/skylink")
+local skynet_skylink = require("skynet/skylink")
 
 describe("parse", function()
    local base32 = "0404dsjvti046fsua4ktor9grrpe76erq9jot9cvopbhsvsu76r4r30"
    local base64 = "AQBG8n_sgEM_nlEp3G0w3vLjmdvSZ46ln8ZXHn-eObZNjA"
 
    it("should return unchanged base64 skylink", function()
-      assert.is.same(skylink.parse(base64), base64)
+      assert.is.same(skynet_skylink.parse(base64), base64)
    end)
    
    it("should transform base32 skylink into base64", function()
-      assert.is.same(skylink.parse(base32), base64)
+      assert.is.same(skynet_skylink.parse(base32), base64)
+   end)
+end)
+
+describe("hash", function()
+   local base64 = "EADi4QZWt87sSDCSjVTcmyI5tE_YAsuC90BcCi_jEmG5NA"
+   local hash = "6cfb9996ad74e5614bbb8e7228e72f1c1bc14dd9ce8a83b3ccabdb6d8d70f330"
+
+   it("should hash skylink", function()
+      assert.is.same(hash, skynet_skylink.hash(base64))
    end)
 end)

docker/nginx/libs/skynet/utils.lua

@@ -0,0 +1,19 @@
+local _M = {}
+
+function _M.authorization_header()
+    local b64 = require("ngx.base64")
+    -- open apipassword file for reading (b flag is required for some reason)
+    -- (file /etc/.sia/apipassword has to be mounted from the host system)
+    local apipassword_file = io.open("/data/sia/apipassword", "rb")
+    -- read apipassword file contents and trim newline (important)
+    local apipassword = apipassword_file:read("*all"):gsub("%s+", "")
+    -- make sure to close file after reading the password
+    apipassword_file.close()
+    -- encode the user:password authorization string
+    -- (in our case user is empty so it is just :password)
+    local content = b64.encode_base64url(":" .. apipassword)
+    -- set authorization header with proper base64 encoded string
+    return "Basic " .. content
+end
+
+return _M

docker/nginx/nginx.conf

@@ -72,11 +72,18 @@ http {
     # proxy cache definition
     proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=50g min_free=100g inactive=48h use_temp_path=off;
 
+    # create a shared blocklist dictionary with size of 30 megabytes
+    # estimated capacity of 1 megabyte dictionary is 3500 blocklist entries
+    # that gives us capacity of around 100k entries in 30 megabyte dictionary
+    lua_shared_dict blocklist 30m;
+
     # this runs before forking out nginx worker processes
     init_by_lua_block { 
         require "cjson"
         require "resty.http"
+        require "skynet.blocklist"
         require "skynet.skylink"
+        require "skynet.utils"
     }
 
     # include skynet-portal-api and skynet-server-api header on every request