diff --git a/.github/workflows/nginx-lua-unit-tests.yml b/.github/workflows/nginx-lua-unit-tests.yml index d0c71aa1..514459fa 100644 --- a/.github/workflows/nginx-lua-unit-tests.yml +++ b/.github/workflows/nginx-lua-unit-tests.yml @@ -25,6 +25,7 @@ jobs: hererocks env --lua=5.1 -rlatest source env/bin/activate luarocks install busted + luarocks install hasher - name: Unit Tests run: | diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index 999df4b2..ba230a28 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -1,6 +1,7 @@ FROM openresty/openresty:1.19.9.1-bionic RUN luarocks install lua-resty-http && \ + luarocks install hasher && \ openssl req -new -newkey rsa:2048 -days 3650 -nodes -x509 \ -subj '/CN=local-certificate' \ -keyout /etc/ssl/local-certificate.key \ diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index 143ca108..ea1e6cd2 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -57,6 +57,14 @@ access_by_lua_block { ngx.var.skynet_proof = res.headers["Skynet-Proof"] end + -- check if skylink v1 is present on blocklist (compare hashes) + if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then + ngx.status = ngx.HTTP_ILLEGAL + ngx.header["content-type"] = "text/plain" + ngx.say("Unavailable For Legal Reasons") + return ngx.exit(ngx.status) + end + -- this block runs only when accounts are enabled if os.getenv("ACCOUNTS_ENABLED") ~= "true" then return end diff --git a/docker/nginx/conf.d/include/sia-auth b/docker/nginx/conf.d/include/sia-auth index df7df09e..d8ec3cf4 100644 --- a/docker/nginx/conf.d/include/sia-auth +++ b/docker/nginx/conf.d/include/sia-auth @@ -1,15 +1,4 @@ rewrite_by_lua_block { - local b64 = require("ngx.base64") - -- open apipassword file for reading (b flag is required for some reason) - -- (file /etc/.sia/apipassword has to be mounted from the host system) - local apipassword_file = io.open("/data/sia/apipassword", "rb") - -- read apipassword file contents and trim newline (important) - local apipassword = apipassword_file:read("*all"):gsub("%s+", "") - -- make sure to close file after reading the password - apipassword_file.close() - -- encode the user:password authorization string - -- (in our case user is empty so it is just :password) - local content = b64.encode_base64url(":" .. apipassword) - -- set authorization header with proper base64 encoded string - ngx.req.set_header("Authorization", "Basic " .. content) + -- set basic authorization header with base64 encoded apipassword + ngx.req.set_header("Authorization", require("skynet.utils").authorization_header()) } diff --git a/docker/nginx/conf.d/server.local.conf b/docker/nginx/conf.d/server.local.conf new file mode 100644 index 00000000..0fe7ecee --- /dev/null +++ b/docker/nginx/conf.d/server.local.conf @@ -0,0 +1,14 @@ +server { + # local server - no not expose this port externally + listen 8000; + listen [::]:8000; + + # secure traffic by limiting to only local networks + allow 10.0.0.0/8; + allow 127.0.0.1/32; + allow 172.16.0.0/12; + allow 192.168.0.0/16; + deny all; + + include /etc/nginx/conf.d/server/server.local; +} diff --git a/docker/nginx/conf.d/server/server.local b/docker/nginx/conf.d/server/server.local new file mode 100644 index 00000000..1de4ab72 --- /dev/null +++ b/docker/nginx/conf.d/server/server.local @@ -0,0 +1,34 @@ +include /etc/nginx/conf.d/include/init-optional-variables; + +location /skynet/blocklist { + content_by_lua_block { + local httpc = require("resty.http").new() + + ngx.req.read_body() -- ensure the post body data is read before using get_body_data + + -- proxy blocklist update request + -- 10.10.10.10 points to sia service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/blocklist", { + method = "POST", + body = ngx.req.get_body_data(), + headers = { + ["Content-Type"] = "application/x-www-form-urlencoded", + ["Authorization"] = require("skynet.utils").authorization_header(), + ["User-Agent"] = "Sia-Agent", + } + }) + + -- print error and exit with 500 or exit with response if status is not 204 + if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then + ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status + ngx.header["content-type"] = "text/plain" + ngx.say(err or res.body) + return ngx.exit(ngx.status) + end + + require("skynet.blocklist").reload() + + ngx.status = ngx.HTTP_NO_CONTENT + return ngx.exit(ngx.status) + } +} diff --git a/docker/nginx/libs/skynet/blocklist.lua b/docker/nginx/libs/skynet/blocklist.lua new file mode 100644 index 00000000..7b45fe28 --- /dev/null +++ b/docker/nginx/libs/skynet/blocklist.lua @@ -0,0 +1,54 @@ +local _M = {} + +function _M.reload() + local httpc = require("resty.http").new() + + -- fetch blocklist records (all blocked skylink hashes) + -- 10.10.10.10 points to sia service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/blocklist", { + headers = { + ["User-Agent"] = "Sia-Agent", + } + }) + + -- fail whole request in case this request failed, we want to make sure + -- the blocklist is pre cached before serving first skylink + if err or (res and res.status ~= ngx.HTTP_OK) then + ngx.log(ngx.ERR, "Failed skyd service request /skynet/blocklist: ", err or ("[HTTP " .. res.status .. "] " .. res.body)) + ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status + ngx.header["content-type"] = "text/plain" + ngx.say(err or res.body) + return ngx.exit(ngx.status) + elseif res and res.status == ngx.HTTP_OK then + local json = require('cjson') + local data = json.decode(res.body) + + -- mark all existing entries as expired + ngx.shared.blocklist:flush_all() + + -- set all cache entries one by one (resets expiration) + for i, hash in ipairs(data.blocklist) do + ngx.shared.blocklist:set(hash, true) + end + + -- ensure that init flag is persisted + ngx.shared.blocklist:set("__init", true) + + -- remove all leftover expired entries + ngx.shared.blocklist:flush_expired() + end +end + +function _M.is_blocked(skylink) + -- make sure that blocklist has been preloaded + if not ngx.shared.blocklist:get("__init") then _M.reload() end + + -- hash skylink before comparing it with blocklist + local hash = require("skynet.skylink").hash(skylink) + + -- we need to use get stale because we're using expiring when updating blocklist + -- and we want to make sure that we're blocking the skylink + return ngx.shared.blocklist:get_stale(hash) == true +end + +return _M diff --git a/docker/nginx/libs/skynet/skylink.lua b/docker/nginx/libs/skynet/skylink.lua index d3b61d36..adcf0b70 100644 --- a/docker/nginx/libs/skynet/skylink.lua +++ b/docker/nginx/libs/skynet/skylink.lua @@ -1,6 +1,7 @@ local _M = {} local basexx = require("basexx") +local hasher = require("hasher") -- parse any skylink and return base64 version function _M.parse(skylink) @@ -13,4 +14,27 @@ function _M.parse(skylink) return skylink end +-- hash skylink into 32 bytes hash used in blocklist +function _M.hash(skylink) + -- ensure that the skylink is base64 encoded + local base64Skylink = _M.parse(skylink) + + -- decode skylink from base64 encoding + local rawSkylink = basexx.from_url64(base64Skylink) + + -- drop first two bytes and leave just merkle root + local rawMerkleRoot = string.sub(rawSkylink, 3) + + -- parse with blake2b with key length of 32 + local blake2bHashed = hasher.blake2b(rawMerkleRoot, 32) + + -- hex encode the blake hash + local hexHashed = basexx.to_hex(blake2bHashed) + + -- lowercase the hex encoded hash + local lowerHexHashed = string.lower(hexHashed) + + return lowerHexHashed +end + return _M diff --git a/docker/nginx/libs/skynet/skylink.spec.lua b/docker/nginx/libs/skynet/skylink.spec.lua index b7d3733c..f45b3013 100644 --- a/docker/nginx/libs/skynet/skylink.spec.lua +++ b/docker/nginx/libs/skynet/skylink.spec.lua @@ -1,14 +1,23 @@ -skylink = require("skynet/skylink") +local skynet_skylink = require("skynet/skylink") describe("parse", function() local base32 = "0404dsjvti046fsua4ktor9grrpe76erq9jot9cvopbhsvsu76r4r30" local base64 = "AQBG8n_sgEM_nlEp3G0w3vLjmdvSZ46ln8ZXHn-eObZNjA" it("should return unchanged base64 skylink", function() - assert.is.same(skylink.parse(base64), base64) + assert.is.same(skynet_skylink.parse(base64), base64) end) it("should transform base32 skylink into base64", function() - assert.is.same(skylink.parse(base32), base64) + assert.is.same(skynet_skylink.parse(base32), base64) + end) +end) + +describe("hash", function() + local base64 = "EADi4QZWt87sSDCSjVTcmyI5tE_YAsuC90BcCi_jEmG5NA" + local hash = "6cfb9996ad74e5614bbb8e7228e72f1c1bc14dd9ce8a83b3ccabdb6d8d70f330" + + it("should hash skylink", function() + assert.is.same(hash, skynet_skylink.hash(base64)) end) end) diff --git a/docker/nginx/libs/skynet/utils.lua b/docker/nginx/libs/skynet/utils.lua new file mode 100644 index 00000000..84f4d812 --- /dev/null +++ b/docker/nginx/libs/skynet/utils.lua @@ -0,0 +1,19 @@ +local _M = {} + +function _M.authorization_header() + local b64 = require("ngx.base64") + -- open apipassword file for reading (b flag is required for some reason) + -- (file /etc/.sia/apipassword has to be mounted from the host system) + local apipassword_file = io.open("/data/sia/apipassword", "rb") + -- read apipassword file contents and trim newline (important) + local apipassword = apipassword_file:read("*all"):gsub("%s+", "") + -- make sure to close file after reading the password + apipassword_file.close() + -- encode the user:password authorization string + -- (in our case user is empty so it is just :password) + local content = b64.encode_base64url(":" .. apipassword) + -- set authorization header with proper base64 encoded string + return "Basic " .. content +end + +return _M diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index c8da6b0f..8ca7abad 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -72,11 +72,16 @@ http { # proxy cache definition proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=50g min_free=100g inactive=48h use_temp_path=off; + # create a shared blocklist dictionary + lua_shared_dict blocklist 25m; + # this runs before forking out nginx worker processes init_by_lua_block { require "cjson" require "resty.http" + require "skynet.blocklist" require "skynet.skylink" + require "skynet.utils" } # include skynet-portal-api and skynet-server-api header on every request