From 9b3e659ec57a7bec8e0a49101a4a2070d7c9d2fd Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 1 Apr 2022 16:14:55 +0200 Subject: [PATCH 1/5] lua testing and codecov v3 --- .github/workflows/nginx-lua-unit-tests.yml | 47 +- .gitignore | 4 + docker/nginx/conf.d/include/location-skylink | 16 +- .../conf.d/include/location-skynet-registry | 8 +- docker/nginx/conf.d/include/track-download | 55 -- docker/nginx/conf.d/include/track-registry | 33 -- docker/nginx/conf.d/include/track-upload | 55 -- docker/nginx/conf.d/server/server.api | 64 +- docker/nginx/libs/skynet/account.lua | 4 +- docker/nginx/libs/skynet/modules.lua | 23 + docker/nginx/libs/skynet/modules.spec.lua | 95 +++ docker/nginx/libs/skynet/scanner.lua | 26 + docker/nginx/libs/skynet/scanner.spec.lua | 121 ++++ docker/nginx/libs/skynet/tracker.lua | 91 +++ docker/nginx/libs/skynet/tracker.spec.lua | 550 ++++++++++++++++++ docker/nginx/libs/skynet/utils.lua | 13 +- docker/nginx/libs/skynet/utils.spec.lua | 65 +++ docker/nginx/libs/utils.lua | 38 ++ docker/nginx/libs/utils.spec.lua | 124 ++++ docker/nginx/testing/.luacov | 6 + docker/nginx/testing/Dockerfile | 11 + docker/nginx/testing/README.md | 3 + docker/nginx/testing/rbusted | 8 + 23 files changed, 1278 insertions(+), 182 deletions(-) delete mode 100644 docker/nginx/conf.d/include/track-download delete mode 100644 docker/nginx/conf.d/include/track-registry delete mode 100644 docker/nginx/conf.d/include/track-upload create mode 100644 docker/nginx/libs/skynet/modules.lua create mode 100644 docker/nginx/libs/skynet/modules.spec.lua create mode 100644 docker/nginx/libs/skynet/scanner.lua create mode 100644 docker/nginx/libs/skynet/scanner.spec.lua create mode 100644 docker/nginx/libs/skynet/tracker.lua create mode 100644 docker/nginx/libs/skynet/tracker.spec.lua create mode 100644 docker/nginx/libs/skynet/utils.spec.lua create mode 100644 docker/nginx/testing/.luacov create mode 100644 docker/nginx/testing/Dockerfile create mode 100644 docker/nginx/testing/README.md create mode 100755 docker/nginx/testing/rbusted diff --git a/.github/workflows/nginx-lua-unit-tests.yml b/.github/workflows/nginx-lua-unit-tests.yml index b86a4e04..6c9fba83 100644 --- a/.github/workflows/nginx-lua-unit-tests.yml +++ b/.github/workflows/nginx-lua-unit-tests.yml @@ -6,48 +6,37 @@ name: Nginx Lua Unit Tests on: push: branches: - - "master" - paths: - - ".github/workflows/nginx-lua-unit-tests.yml" - - "docker/nginx/libs/**.lua" + - master pull_request: - paths: - - ".github/workflows/nginx-lua-unit-tests.yml" - - "docker/nginx/libs/**.lua" jobs: - build: + test: runs-on: ubuntu-latest - + container: openresty/openresty:1.19.9.1-focal steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: "3.x" - architecture: "x64" + - uses: actions/checkout@v3 - name: Install Dependencies run: | - pip install hererocks - hererocks env --lua=5.1 -rlatest - source env/bin/activate + luarocks install lua-resty-http + luarocks install hasher luarocks install busted luarocks install luacov - luarocks install hasher luarocks install luacheck - - name: Lint code - run: | - source env/bin/activate - luacheck docker/nginx/libs --std ngx_lua+busted + - name: Lint Code With Luacheck + run: luacheck docker/nginx/libs --std ngx_lua+busted - - name: Unit Tests - run: | - source env/bin/activate - busted --verbose --coverage --pattern=spec --directory=docker/nginx/libs . - cd docker/nginx/libs && luacov + - name: Run Tests With Busted + # ran from root repo directory; produces luacov.stats.out file + run: docker/nginx/testing/rbusted --lpath='docker/nginx/libs/?.lua;docker/nginx/libs/?/?.lua' --verbose --coverage --pattern=spec docker/nginx/libs - - uses: codecov/codecov-action@v2 + - name: Generate Code Coverage Report With Luacov + # requires config file in cwd; produces luacov.report.out file + run: cp docker/nginx/testing/.luacov . && luacov && rm .luacov + + - uses: codecov/codecov-action@v3 with: - directory: docker/nginx/libs + root_dir: ${GITHUB_WORKSPACE} + files: ./luacov.report.out flags: nginx-lua diff --git a/.gitignore b/.gitignore index 4b85194e..8a98ee28 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,10 @@ __pycache__ /.idea/ /venv* +# Luacov file +luacov.stats.out +luacov.report.out + # Setup-script log files setup-scripts/serverload.log setup-scripts/serverload.json diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index 995a6e2d..b214e3a9 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -1,5 +1,4 @@ include /etc/nginx/conf.d/include/cors; -include /etc/nginx/conf.d/include/track-download; limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time @@ -37,3 +36,18 @@ proxy_read_timeout 600; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://sia:9980/skynet/skylink/$skylink$path$is_args$args; + +log_by_lua_block { + local skynet_account = require("skynet.account") + local skynet_modules = require("skynet.modules") + local skynet_scanner = require("skynet.scanner") + local skynet_tracker = require("skynet.tracker") + + if skynet_modules.is_enabled("a") then + skynet_tracker.track_download(ngx.header["Skynet-Skylink"], ngx.status, skynet_account.get_auth_headers(), ngx.var.body_bytes_sent) + end + + if skynet_modules.is_enabled("s") then + skynet_scanner.scan_skylink(ngx.header["Skynet-Skylink"]) + end +} diff --git a/docker/nginx/conf.d/include/location-skynet-registry b/docker/nginx/conf.d/include/location-skynet-registry index 33838f70..cd450be9 100644 --- a/docker/nginx/conf.d/include/location-skynet-registry +++ b/docker/nginx/conf.d/include/location-skynet-registry @@ -1,6 +1,5 @@ include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; -include /etc/nginx/conf.d/include/track-registry; limit_req zone=registry_access_by_ip burst=600 nodelay; limit_req zone=registry_access_by_ip_throttled burst=200 nodelay; @@ -30,3 +29,10 @@ access_by_lua_block { end end } + +log_by_lua_block { + local skynet_account = require("skynet.account") + local skynet_tracker = require("skynet.tracker") + + skynet_tracker.track_registry(ngx.status, skynet_account.get_auth_headers(), ngx.req.get_method()) +} diff --git a/docker/nginx/conf.d/include/track-download b/docker/nginx/conf.d/include/track-download deleted file mode 100644 index 4e12fd41..00000000 --- a/docker/nginx/conf.d/include/track-download +++ /dev/null @@ -1,55 +0,0 @@ -log_by_lua_block { - local skynet_account = require("skynet.account") - - -- tracking runs only when request comes from authenticated user - if skynet_account.is_authenticated() then - local function track(premature, skylink, status, body_bytes_sent, auth_headers) - if premature then return end - - local httpc = require("resty.http").new() - local query = table.concat({ "status=" .. status, "bytes=" .. body_bytes_sent }, "&") - - -- 10.10.10.70 points to accounts service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.70:3000/track/download/" .. skylink .. "?" .. query, { - method = "POST", - headers = auth_headers, - }) - - if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then - local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) - ngx.log(ngx.ERR, "Failed accounts service request /track/download/" .. skylink .. ": ", error_response) - end - end - - if ngx.header["Skynet-Skylink"] and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then - local auth_headers = skynet_account.get_auth_headers() - local ok, err = ngx.timer.at(0, track, ngx.header["Skynet-Skylink"], ngx.status, ngx.var.body_bytes_sent, auth_headers) - if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end - end - end - - -- this block runs only when scanner module is enabled - if os.getenv("PORTAL_MODULES"):match("s") then - local function scan(premature, skylink) - if premature then return end - - local httpc = require("resty.http").new() - - -- 10.10.10.101 points to malware-scanner service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.101:4000/scan/" .. skylink, { - method = "POST", - }) - - if err or (res and res.status ~= ngx.HTTP_OK) then - local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) - ngx.log(ngx.ERR, "Failed malware-scanner request /scan/" .. skylink .. ": ", error_response) - end - end - - -- scan all skylinks but make sure to only run if skylink is present (empty if request failed) - if ngx.header["Skynet-Skylink"] then - local ok, err = ngx.timer.at(0, scan, ngx.header["Skynet-Skylink"]) - if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end - end - end -} diff --git a/docker/nginx/conf.d/include/track-registry b/docker/nginx/conf.d/include/track-registry deleted file mode 100644 index 2c840491..00000000 --- a/docker/nginx/conf.d/include/track-registry +++ /dev/null @@ -1,33 +0,0 @@ -log_by_lua_block { - local skynet_account = require("skynet.account") - - -- tracking runs only when request comes from authenticated user - if skynet_account.is_authenticated() then - local function track(premature, request_method, auth_headers) - if premature then return end - - local httpc = require("resty.http").new() - - -- based on request method we assign a registry action string used - -- in track endpoint namely "read" for GET and "write" for POST - local registry_action = request_method == "GET" and "read" or "write" - - -- 10.10.10.70 points to accounts service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.70:3000/track/registry/" .. registry_action, { - method = "POST", - headers = auth_headers, - }) - - if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then - local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) - ngx.log(ngx.ERR, "Failed accounts service request /track/registry/" .. registry_action .. ": ", error_response) - end - end - - if ngx.status == ngx.HTTP_OK or ngx.status == ngx.HTTP_NOT_FOUND then - local auth_headers = skynet_account.get_auth_headers() - local ok, err = ngx.timer.at(0, track, ngx.req.get_method(), auth_headers) - if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end - end - end -} diff --git a/docker/nginx/conf.d/include/track-upload b/docker/nginx/conf.d/include/track-upload deleted file mode 100644 index 36b12b9e..00000000 --- a/docker/nginx/conf.d/include/track-upload +++ /dev/null @@ -1,55 +0,0 @@ -log_by_lua_block { - local skynet_account = require("skynet.account") - - -- tracking runs only when request comes from authenticated user - if skynet_account.is_authenticated() then - local function track(premature, skylink, auth_headers) - if premature then return end - - local httpc = require("resty.http").new() - - -- 10.10.10.70 points to accounts service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.70:3000/track/upload/" .. skylink, { - method = "POST", - headers = auth_headers, - }) - - if err or (res and res.status ~= ngx.HTTP_NO_CONTENT) then - local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) - ngx.log(ngx.ERR, "Failed accounts service request /track/upload/" .. skylink .. ": ", error_response) - end - end - - -- report all skylinks (header empty if request failed) but only if jwt is preset (user is authenticated) - if ngx.header["Skynet-Skylink"] then - local auth_headers = skynet_account.get_auth_headers() - local ok, err = ngx.timer.at(0, track, ngx.header["Skynet-Skylink"], auth_headers) - if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end - end - end - - -- this block runs only when scanner module is enabled - if os.getenv("PORTAL_MODULES"):match("s") then - local function scan(premature, skylink) - if premature then return end - - local httpc = require("resty.http").new() - - -- 10.10.10.101 points to malware-scanner service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.101:4000/scan/" .. skylink, { - method = "POST", - }) - - if err or (res and res.status ~= ngx.HTTP_OK) then - local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) - ngx.log(ngx.ERR, "Failed malware-scanner request /scan/" .. skylink .. ": ", error_response) - end - end - - -- scan all skylinks but make sure to only run if skylink is present (empty if request failed) - if ngx.header["Skynet-Skylink"] then - local ok, err = ngx.timer.at(0, scan, ngx.header["Skynet-Skylink"]) - if err then ngx.log(ngx.ERR, "Failed to create timer: ", err) end - end - end -} diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index f681cca8..84842a03 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -206,7 +206,6 @@ location /skynet/registry/subscription { location /skynet/skyfile { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; - include /etc/nginx/conf.d/include/track-upload; include /etc/nginx/conf.d/include/generate-siapath; include /etc/nginx/conf.d/include/portal-access-check; @@ -228,12 +227,26 @@ location /skynet/skyfile { # proxy this call to siad endpoint (make sure the ip is correct) proxy_pass http://sia:9980/skynet/skyfile/$dir1/$dir2/$dir3$is_args$args; + + log_by_lua_block { + local skynet_account = require("skynet.account") + local skynet_modules = require("skynet.modules") + local skynet_scanner = require("skynet.scanner") + local skynet_tracker = require("skynet.tracker") + + if skynet_modules.is_enabled("a") then + skynet_tracker.track_upload(ngx.header["Skynet-Skylink"], ngx.status, skynet_account.get_auth_headers()) + end + + if skynet_modules.is_enabled("s") then + skynet_scanner.scan_skylink(ngx.header["Skynet-Skylink"]) + end + } } # endpoint implementing resumable file uploads open protocol https://tus.io location /skynet/tus { include /etc/nginx/conf.d/include/cors-headers; # include cors headers but do not overwrite OPTIONS response - include /etc/nginx/conf.d/include/track-upload; limit_req zone=uploads_by_ip burst=10 nodelay; limit_req zone=uploads_by_ip_throttled; @@ -294,12 +307,26 @@ location /skynet/tus { end end } + + log_by_lua_block { + local skynet_account = require("skynet.account") + local skynet_modules = require("skynet.modules") + local skynet_scanner = require("skynet.scanner") + local skynet_tracker = require("skynet.tracker") + + if skynet_modules.is_enabled("a") then + skynet_tracker.track_upload(ngx.header["Skynet-Skylink"], ngx.status, skynet_account.get_auth_headers()) + end + + if skynet_modules.is_enabled("s") then + skynet_scanner.scan_skylink(ngx.header["Skynet-Skylink"]) + end + } } location /skynet/pin { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; - include /etc/nginx/conf.d/include/track-upload; include /etc/nginx/conf.d/include/generate-siapath; include /etc/nginx/conf.d/include/portal-access-check; @@ -311,6 +338,21 @@ location /skynet/pin { proxy_set_header User-Agent: Sia-Agent; proxy_pass http://sia:9980$uri?siapath=$dir1/$dir2/$dir3&$args; + + log_by_lua_block { + local skynet_account = require("skynet.account") + local skynet_modules = require("skynet.modules") + local skynet_scanner = require("skynet.scanner") + local skynet_tracker = require("skynet.tracker") + + if skynet_modules.is_enabled("a") then + skynet_tracker.track_upload(ngx.header["Skynet-Skylink"], ngx.status, skynet_account.get_auth_headers()) + end + + if skynet_modules.is_enabled("s") then + skynet_scanner.scan_skylink(ngx.header["Skynet-Skylink"]) + end + } } location /skynet/metadata { @@ -357,7 +399,6 @@ location ~ "^/file/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" { location /skynet/trustless/basesector { include /etc/nginx/conf.d/include/cors; - include /etc/nginx/conf.d/include/track-download; limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time @@ -391,6 +432,21 @@ location /skynet/trustless/basesector { proxy_set_header User-Agent: Sia-Agent; proxy_pass http://sia:9980; + + log_by_lua_block { + local skynet_account = require("skynet.account") + local skynet_modules = require("skynet.modules") + local skynet_scanner = require("skynet.scanner") + local skynet_tracker = require("skynet.tracker") + + if skynet_modules.is_enabled("a") then + skynet_tracker.track_download(ngx.header["Skynet-Skylink"], ngx.status, skynet_account.get_auth_headers(), ngx.var.body_bytes_sent) + end + + if skynet_modules.is_enabled("s") then + skynet_scanner.scan_skylink(ngx.header["Skynet-Skylink"]) + end + } } location /__internal/do/not/use/accounts { diff --git a/docker/nginx/libs/skynet/account.lua b/docker/nginx/libs/skynet/account.lua index 6fa2c4d2..56c99897 100644 --- a/docker/nginx/libs/skynet/account.lua +++ b/docker/nginx/libs/skynet/account.lua @@ -59,7 +59,9 @@ function _M.exit_access_forbidden(message) end function _M.accounts_enabled() - return os.getenv("PORTAL_MODULES"):match("a") ~= nil + local skynet_modules = require("skynet.modules") + + return skynet_modules.is_enabled("a") end function _M.get_account_limits() diff --git a/docker/nginx/libs/skynet/modules.lua b/docker/nginx/libs/skynet/modules.lua new file mode 100644 index 00000000..607e6d8e --- /dev/null +++ b/docker/nginx/libs/skynet/modules.lua @@ -0,0 +1,23 @@ +local _M = {} + +local utils = require("utils") + +function _M.is_enabled(module_abbr) + if type(module_abbr) ~= "string" or module_abbr:len() ~= 1 then + error("Module abbreviation '" .. tostring(module_abbr) .. "' should be exactly one character long string") + end + + local enabled_modules = utils.getenv("PORTAL_MODULES") + + if not enabled_modules then + return false + end + + return enabled_modules:find(module_abbr) ~= nil +end + +function _M.is_disabled(module_abbr) + return not _M.is_enabled(module_abbr) +end + +return _M diff --git a/docker/nginx/libs/skynet/modules.spec.lua b/docker/nginx/libs/skynet/modules.spec.lua new file mode 100644 index 00000000..0797a32d --- /dev/null +++ b/docker/nginx/libs/skynet/modules.spec.lua @@ -0,0 +1,95 @@ +-- luacheck: ignore os + +local skynet_modules = require("skynet.modules") + +describe("is_enabled", function() + before_each(function() + stub(os, "getenv") + end) + + after_each(function() + os.getenv:revert() + end) + + it("should return false if PORTAL_MODULES are not defined", function() + os.getenv.on_call_with("PORTAL_MODULES").returns(nil) + + assert.is_false(skynet_modules.is_enabled("a")) + end) + + it("should return false if PORTAL_MODULES are empty", function() + os.getenv.on_call_with("PORTAL_MODULES").returns("") + + assert.is_false(skynet_modules.is_enabled("a")) + end) + + it("should return false if module is not enabled", function() + os.getenv.on_call_with("PORTAL_MODULES").returns("qwerty") + + assert.is_false(skynet_modules.is_enabled("a")) + end) + + it("should return true if module is enabled", function() + os.getenv.on_call_with("PORTAL_MODULES").returns("asdfg") + + assert.is_true(skynet_modules.is_enabled("a")) + end) + + it("should throw an error for empty module", function() + assert.has_error(function() + skynet_modules.is_enabled() + end, "Module abbreviation 'nil' should be exactly one character long string") + end) + + it("should throw an error for too long module", function() + assert.has_error(function() + skynet_modules.is_enabled("gandalf") + end, "Module abbreviation 'gandalf' should be exactly one character long string") + end) +end) + +describe("is_disabled", function() + before_each(function() + stub(os, "getenv") + end) + + after_each(function() + os.getenv:revert() + end) + + it("should return true if PORTAL_MODULES are not defined", function() + os.getenv.on_call_with("PORTAL_MODULES").returns(nil) + + assert.is_true(skynet_modules.is_disabled("a")) + end) + + it("should return true if PORTAL_MODULES are empty", function() + os.getenv.on_call_with("PORTAL_MODULES").returns("") + + assert.is_true(skynet_modules.is_disabled("a")) + end) + + it("should return true if module is not enabled", function() + os.getenv.on_call_with("PORTAL_MODULES").returns("qwerty") + + assert.is_true(skynet_modules.is_disabled("a")) + end) + + it("should return false if module is enabled", function() + os.getenv.on_call_with("PORTAL_MODULES").returns("asdfg") + + assert.is_false(skynet_modules.is_disabled("a")) + end) + + it("should throw an error for empty module", function() + assert.has_error(function() + skynet_modules.is_disabled() + end, "Module abbreviation 'nil' should be exactly one character long string") + end) + + it("should throw an error for too long module", function() + assert.has_error(function() + skynet_modules.is_disabled("gandalf") + end, "Module abbreviation 'gandalf' should be exactly one character long string") + end) +end) \ No newline at end of file diff --git a/docker/nginx/libs/skynet/scanner.lua b/docker/nginx/libs/skynet/scanner.lua new file mode 100644 index 00000000..445f1ae9 --- /dev/null +++ b/docker/nginx/libs/skynet/scanner.lua @@ -0,0 +1,26 @@ +local _M = {} + +function _M.scan_skylink_timer(premature, skylink) + if premature then return end + + local httpc = require("resty.http").new() + + -- 10.10.10.101 points to malware-scanner service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.101:4000/scan/" .. skylink, { + method = "POST", + }) + + if err or (res and res.status ~= ngx.HTTP_OK) then + local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) + ngx.log(ngx.ERR, "Failed malware-scanner request /scan/" .. skylink .. ": ", error_response) + end +end + +function _M.scan_skylink(skylink) + if not skylink then return end + + local ok, err = ngx.timer.at(0, _M.scan_skylink_timer, skylink) + if not ok then ngx.log(ngx.ERR, "Failed to create timer: ", err) end +end + +return _M diff --git a/docker/nginx/libs/skynet/scanner.spec.lua b/docker/nginx/libs/skynet/scanner.spec.lua new file mode 100644 index 00000000..400c301c --- /dev/null +++ b/docker/nginx/libs/skynet/scanner.spec.lua @@ -0,0 +1,121 @@ +-- luacheck: ignore ngx + +local skynet_scanner = require("skynet.scanner") +local skylink = "AQBG8n_sgEM_nlEp3G0w3vLjmdvSZ46ln8ZXHn-eObZNjA" + +describe("scan_skylink", function() + before_each(function() + stub(ngx.timer, "at") + end) + + after_each(function() + ngx.timer.at:revert() + end) + + it("should schedule a timer when skylink is provided", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_scanner.scan_skylink(skylink) + + assert.stub(ngx.timer.at).was_called_with(0, skynet_scanner.scan_skylink_timer, skylink) + end) + + it("should log an error if timer failed to create", function() + stub(ngx, "log") + + ngx.timer.at.invokes(function() return false, "such a failure" end) + + skynet_scanner.scan_skylink(skylink) + + assert.stub(ngx.timer.at).was_called_with(0, skynet_scanner.scan_skylink_timer, skylink) + assert.stub(ngx.log).was_called_with(ngx.ERR, "Failed to create timer: ", "such a failure") + + ngx.log:revert() + end) + + it("should not schedule a timer if skylink is not provided", function() + skynet_scanner.scan_skylink() + + assert.stub(ngx.timer.at).was_not_called() + end) +end) + +describe("scan_skylink_timer", function() + before_each(function() + stub(ngx, "log") + end) + + after_each(function() + local resty_http = require("resty.http") + + ngx.log:revert() + resty_http.new:revert() + end) + + it("should exit early on premature", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 200 } -- return 200 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_scanner.scan_skylink_timer(true, skylink) + + assert.stub(request_uri).was_not_called() + assert.stub(ngx.log).was_not_called() + end) + + it("should make a post request with skylink to scanner service", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 200 } -- return 200 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_scanner.scan_skylink_timer(false, skylink) + + local uri = "http://10.10.10.101:4000/scan/" .. skylink + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST" }) + assert.stub(ngx.log).was_not_called() + end) + + it("should log message on scanner request failure with response code", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 404, body = "baz" } -- return 404 failure + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_scanner.scan_skylink_timer(false, skylink) + + local uri = "http://10.10.10.101:4000/scan/" .. skylink + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST" }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed malware-scanner request /scan/AQBG8n_sgEM_nlEp3G0w3vLjmdvSZ46ln8ZXHn-eObZNjA: ", + "[HTTP 404] baz" + ) + end) + + it("should log message on scanner request error", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return nil, "foo != bar" -- return error + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_scanner.scan_skylink_timer(false, skylink) + + local uri = "http://10.10.10.101:4000/scan/" .. skylink + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST" }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed malware-scanner request /scan/AQBG8n_sgEM_nlEp3G0w3vLjmdvSZ46ln8ZXHn-eObZNjA: ", + "foo != bar" + ) + end) +end) \ No newline at end of file diff --git a/docker/nginx/libs/skynet/tracker.lua b/docker/nginx/libs/skynet/tracker.lua new file mode 100644 index 00000000..78fa6995 --- /dev/null +++ b/docker/nginx/libs/skynet/tracker.lua @@ -0,0 +1,91 @@ +local _M = {} + +local utils = require("utils") + +function _M.track_download_timer(premature, skylink, status, auth_headers, body_bytes_sent) + if premature then return end + + local httpc = require("resty.http").new() + local query = table.concat({ "status=" .. status, "bytes=" .. body_bytes_sent }, "&") + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/track/download/" .. skylink .. "?" .. query, { + method = "POST", + headers = auth_headers, + }) + + if err or (res and res.status ~= 204) then + local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) + ngx.log(ngx.ERR, "Failed accounts service request /track/download/" .. skylink .. ": ", error_response) + end +end + +function _M.track_download(skylink, status_code, auth_headers, body_bytes_sent) + local has_auth_headers = not utils.is_table_empty(auth_headers) + local status_success = status_code >= 200 and status_code <= 299 + + if skylink and status_success and has_auth_headers then + local ok, err = ngx.timer.at(0, _M.track_download_timer, skylink, status_code, auth_headers, body_bytes_sent) + if not ok then ngx.log(ngx.ERR, "Failed to create timer: ", err) end + end +end + +function _M.track_upload_timer(premature, skylink, auth_headers) + if premature then return end + + local httpc = require("resty.http").new() + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/track/upload/" .. skylink, { + method = "POST", + headers = auth_headers, + }) + + if err or (res and res.status ~= 204) then + local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) + ngx.log(ngx.ERR, "Failed accounts service request /track/upload/" .. skylink .. ": ", error_response) + end +end + +function _M.track_upload(skylink, status_code, auth_headers) + local has_auth_headers = not utils.is_table_empty(auth_headers) + local status_success = status_code >= 200 and status_code <= 299 + + if skylink and status_success and has_auth_headers then + local ok, err = ngx.timer.at(0, _M.track_upload_timer, skylink, auth_headers) + if not ok then ngx.log(ngx.ERR, "Failed to create timer: ", err) end + end +end + +function _M.track_registry_timer(premature, auth_headers, request_method) + if premature then return end + + local httpc = require("resty.http").new() + + -- based on request method we assign a registry action string used + -- in track endpoint namely "read" for GET and "write" for POST + local registry_action = request_method == "GET" and "read" or "write" + + -- 10.10.10.70 points to accounts service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.70:3000/track/registry/" .. registry_action, { + method = "POST", + headers = auth_headers, + }) + + if err or (res and res.status ~= 204) then + local error_response = err or ("[HTTP " .. res.status .. "] " .. res.body) + ngx.log(ngx.ERR, "Failed accounts service request /track/registry/" .. registry_action .. ": ", error_response) + end +end + +function _M.track_registry(status_code, auth_headers, request_method) + local has_auth_headers = not utils.is_table_empty(auth_headers) + local tracked_status = status_code == 200 or status_code == 404 + + if tracked_status and has_auth_headers then + local ok, err = ngx.timer.at(0, _M.track_registry_timer, auth_headers, request_method) + if not ok then ngx.log(ngx.ERR, "Failed to create timer: ", err) end + end +end + +return _M diff --git a/docker/nginx/libs/skynet/tracker.spec.lua b/docker/nginx/libs/skynet/tracker.spec.lua new file mode 100644 index 00000000..41dab843 --- /dev/null +++ b/docker/nginx/libs/skynet/tracker.spec.lua @@ -0,0 +1,550 @@ +-- luacheck: ignore ngx + +local skynet_tracker = require("skynet.tracker") + +local valid_skylink = "AQBG8n_sgEM_nlEp3G0w3vLjmdvSZ46ln8ZXHn-eObZNjA" +local valid_status_code = 200 +local valid_auth_headers = { ["Skynet-Api-Key"] = "foo" } + +describe("track_download", function() + local valid_body_bytes_sent = 12345 + + before_each(function() + stub(ngx.timer, "at") + end) + + after_each(function() + ngx.timer.at:revert() + end) + + it("should schedule a timer when conditions are met", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_download(valid_skylink, valid_status_code, valid_auth_headers, valid_body_bytes_sent) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_download_timer, + valid_skylink, + valid_status_code, + valid_auth_headers, + valid_body_bytes_sent + ) + end) + + it("should not schedule a timer if skylink is empty", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_download(nil, valid_status_code, valid_auth_headers, valid_body_bytes_sent) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should not schedule a timer if status code is not in 2XX range", function() + ngx.timer.at.invokes(function() return true, nil end) + + -- couple of example of 4XX and 5XX codes + skynet_tracker.track_download(valid_skylink, 401, valid_auth_headers, valid_body_bytes_sent) + skynet_tracker.track_download(valid_skylink, 403, valid_auth_headers, valid_body_bytes_sent) + skynet_tracker.track_download(valid_skylink, 490, valid_auth_headers, valid_body_bytes_sent) + skynet_tracker.track_download(valid_skylink, 500, valid_auth_headers, valid_body_bytes_sent) + skynet_tracker.track_download(valid_skylink, 502, valid_auth_headers, valid_body_bytes_sent) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should not schedule a timer if auth headers are empty", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_download(valid_skylink, valid_status_code, {}, valid_body_bytes_sent) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should log an error if timer failed to create", function() + stub(ngx, "log") + ngx.timer.at.invokes(function() return false, "such a failure" end) + + skynet_tracker.track_download(valid_skylink, valid_status_code, valid_auth_headers, valid_body_bytes_sent) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_download_timer, + valid_skylink, + valid_status_code, + valid_auth_headers, + valid_body_bytes_sent + ) + assert.stub(ngx.log).was_called_with(ngx.ERR, "Failed to create timer: ", "such a failure") + + ngx.log:revert() + end) + + describe("track_download_timer", function() + before_each(function() + stub(ngx, "log") + end) + + after_each(function() + local resty_http = require("resty.http") + + ngx.log:revert() + resty_http.new:revert() + end) + + it("should exit early on premature", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 200 } -- return 200 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_download_timer( + true, + valid_skylink, + valid_status_code, + valid_auth_headers, + valid_body_bytes_sent + ) + + assert.stub(request_uri).was_not_called() + assert.stub(ngx.log).was_not_called() + end) + + it("should make a post request to tracker service", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 204 } -- return 204 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_download_timer( + false, + valid_skylink, + valid_status_code, + valid_auth_headers, + valid_body_bytes_sent + ) + + local uri_params = "status=" .. valid_status_code .. "&bytes=" .. valid_body_bytes_sent + local uri = "http://10.10.10.70:3000/track/download/" .. valid_skylink .. "?" .. uri_params + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_not_called() + end) + + it("should log message on tracker request failure with response code", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 404, body = "baz" } -- return 404 failure + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_download_timer( + false, + valid_skylink, + valid_status_code, + valid_auth_headers, + valid_body_bytes_sent + ) + + local uri_params = "status=" .. valid_status_code .. "&bytes=" .. valid_body_bytes_sent + local uri = "http://10.10.10.70:3000/track/download/" .. valid_skylink .. "?" .. uri_params + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed accounts service request /track/download/" .. valid_skylink .. ": ", + "[HTTP 404] baz" + ) + end) + + it("should log message on tracker request error", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return nil, "foo != bar" -- return error + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_download_timer( + false, + valid_skylink, + valid_status_code, + valid_auth_headers, + valid_body_bytes_sent + ) + + local uri_params = "status=" .. valid_status_code .. "&bytes=" .. valid_body_bytes_sent + local uri = "http://10.10.10.70:3000/track/download/" .. valid_skylink .. "?" .. uri_params + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed accounts service request /track/download/" .. valid_skylink .. ": ", + "foo != bar" + ) + end) + end) +end) + +describe("track_upload", function() + before_each(function() + stub(ngx.timer, "at") + end) + + after_each(function() + ngx.timer.at:revert() + end) + + it("should schedule a timer when conditions are met", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_upload(valid_skylink, valid_status_code, valid_auth_headers) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_upload_timer, + valid_skylink, + valid_auth_headers + ) + end) + + it("should not schedule a timer if skylink is empty", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_upload(nil, valid_status_code, valid_auth_headers) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should not schedule a timer if status code is not in 2XX range", function() + ngx.timer.at.invokes(function() return true, nil end) + + -- couple of example of 4XX and 5XX codes + skynet_tracker.track_upload(valid_skylink, 401, valid_auth_headers) + skynet_tracker.track_upload(valid_skylink, 403, valid_auth_headers) + skynet_tracker.track_upload(valid_skylink, 490, valid_auth_headers) + skynet_tracker.track_upload(valid_skylink, 500, valid_auth_headers) + skynet_tracker.track_upload(valid_skylink, 502, valid_auth_headers) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should not schedule a timer if auth headers are empty", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_upload(valid_skylink, valid_status_code, {}) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should log an error if timer failed to create", function() + stub(ngx, "log") + ngx.timer.at.invokes(function() return false, "such a failure" end) + + skynet_tracker.track_upload(valid_skylink, valid_status_code, valid_auth_headers) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_upload_timer, + valid_skylink, + valid_auth_headers + ) + assert.stub(ngx.log).was_called_with(ngx.ERR, "Failed to create timer: ", "such a failure") + + ngx.log:revert() + end) + + describe("track_upload_timer", function() + before_each(function() + stub(ngx, "log") + end) + + after_each(function() + local resty_http = require("resty.http") + + ngx.log:revert() + resty_http.new:revert() + end) + + it("should exit early on premature", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 200 } -- return 200 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_upload_timer( + true, + valid_skylink, + valid_auth_headers + ) + + assert.stub(request_uri).was_not_called() + assert.stub(ngx.log).was_not_called() + end) + + it("should make a post request to tracker service", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 204 } -- return 204 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_upload_timer( + false, + valid_skylink, + valid_auth_headers + ) + + local uri = "http://10.10.10.70:3000/track/upload/" .. valid_skylink + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_not_called() + end) + + it("should log message on tracker request failure with response code", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 404, body = "baz" } -- return 404 failure + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_upload_timer( + false, + valid_skylink, + valid_auth_headers + ) + + local uri = "http://10.10.10.70:3000/track/upload/" .. valid_skylink + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed accounts service request /track/upload/" .. valid_skylink .. ": ", + "[HTTP 404] baz" + ) + end) + + it("should log message on tracker request error", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return nil, "foo != bar" -- return error + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_upload_timer( + false, + valid_skylink, + valid_auth_headers + ) + + local uri = "http://10.10.10.70:3000/track/upload/" .. valid_skylink + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed accounts service request /track/upload/" .. valid_skylink .. ": ", + "foo != bar" + ) + end) + end) +end) + +describe("track_registry", function() + local status_code_ok = 200 + local status_code_not_found = 404 + local request_method_write = "POST" + local request_method_read = "GET" + + before_each(function() + stub(ngx.timer, "at") + end) + + after_each(function() + ngx.timer.at:revert() + end) + + it("should schedule a timer when status code was 200", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_registry(status_code_ok, valid_auth_headers, request_method_write) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_registry_timer, + valid_auth_headers, + request_method_write + ) + end) + + it("should schedule a timer when status code was 404", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_registry(status_code_not_found, valid_auth_headers, request_method_write) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_registry_timer, + valid_auth_headers, + request_method_write + ) + end) + + it("should not schedule a timer if status code is not in 200 or 404", function() + ngx.timer.at.invokes(function() return true, nil end) + + -- couple of example of invalid 2XX, 4XX and 5XX codes + skynet_tracker.track_registry(204, valid_auth_headers, request_method_write) + skynet_tracker.track_registry(206, valid_auth_headers, request_method_write) + skynet_tracker.track_registry(401, valid_auth_headers, request_method_write) + skynet_tracker.track_registry(403, valid_auth_headers, request_method_write) + skynet_tracker.track_registry(490, valid_auth_headers, request_method_write) + skynet_tracker.track_registry(500, valid_auth_headers, request_method_write) + skynet_tracker.track_registry(502, valid_auth_headers, request_method_write) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should not schedule a timer if auth headers are empty", function() + ngx.timer.at.invokes(function() return true, nil end) + + skynet_tracker.track_registry(status_code_ok, {}, request_method_write) + + assert.stub(ngx.timer.at).was_not_called() + end) + + it("should log an error if timer failed to create", function() + stub(ngx, "log") + ngx.timer.at.invokes(function() return false, "such a failure" end) + + skynet_tracker.track_registry(status_code_ok, valid_auth_headers, request_method_write) + + assert.stub(ngx.timer.at).was_called_with( + 0, + skynet_tracker.track_registry_timer, + valid_auth_headers, + request_method_write + ) + assert.stub(ngx.log).was_called_with(ngx.ERR, "Failed to create timer: ", "such a failure") + + ngx.log:revert() + end) + + describe("track_registry_timer", function() + before_each(function() + stub(ngx, "log") + end) + + after_each(function() + local resty_http = require("resty.http") + + ngx.log:revert() + resty_http.new:revert() + end) + + it("should exit early on premature", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 200 } -- return 200 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_registry_timer( + true, + valid_auth_headers, + request_method_write + ) + + assert.stub(request_uri).was_not_called() + assert.stub(ngx.log).was_not_called() + end) + + it("should make a post request to registry write tracker service", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 204 } -- return 204 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_registry_timer( + false, + valid_auth_headers, + request_method_write + ) + + local uri = "http://10.10.10.70:3000/track/registry/write" + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_not_called() + end) + + it("should make a post request to registry read tracker service", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 204 } -- return 204 success + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_registry_timer( + false, + valid_auth_headers, + request_method_read + ) + + local uri = "http://10.10.10.70:3000/track/registry/read" + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_not_called() + end) + + it("should log message on tracker request failure with response code", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return { status = 404, body = "baz" } -- return 404 failure + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_registry_timer( + false, + valid_auth_headers, + request_method_write + ) + + local uri = "http://10.10.10.70:3000/track/registry/write" + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed accounts service request /track/registry/write: ", + "[HTTP 404] baz" + ) + end) + + it("should log message on tracker request error", function() + local resty_http = require("resty.http") + local request_uri = spy.new(function() + return nil, "foo != bar" -- return error + end) + local httpc = mock({ request_uri = request_uri }) + stub(resty_http, "new").returns(httpc) + + skynet_tracker.track_registry_timer( + false, + valid_auth_headers, + request_method_write + ) + + local uri = "http://10.10.10.70:3000/track/registry/write" + assert.stub(request_uri).was_called_with(httpc, uri, { method = "POST", headers = valid_auth_headers }) + assert.stub(ngx.log).was_called_with( + ngx.ERR, + "Failed accounts service request /track/registry/write: ", + "foo != bar" + ) + end) + end) +end) \ No newline at end of file diff --git a/docker/nginx/libs/skynet/utils.lua b/docker/nginx/libs/skynet/utils.lua index adee23b2..05755f7b 100644 --- a/docker/nginx/libs/skynet/utils.lua +++ b/docker/nginx/libs/skynet/utils.lua @@ -1,13 +1,20 @@ local _M = {} +local ngx_base64 = require("ngx.base64") +local utils = require("utils") + function _M.authorization_header() -- read api password from env variable - local apipassword = os.getenv("SIA_API_PASSWORD") + local apipassword = utils.getenv("SIA_API_PASSWORD") -- if api password is not available as env variable, read it from disk - if apipassword == nil or apipassword == "" then + if not apipassword then -- open apipassword file for reading (b flag is required for some reason) -- (file /etc/.sia/apipassword has to be mounted from the host system) local apipassword_file = io.open("/data/sia/apipassword", "rb") + -- make sure to throw a meaningful error if apipassword file does not exist + if not apipassword_file then + error("Error reading /data/sia/apipassword file") + end -- read apipassword file contents and trim newline (important) apipassword = apipassword_file:read("*all"):gsub("%s+", "") -- make sure to close file after reading the password @@ -15,7 +22,7 @@ function _M.authorization_header() end -- encode the user:password authorization string -- (in our case user is empty so it is just :password) - local content = require("ngx.base64").encode_base64url(":" .. apipassword) + local content = ngx_base64.encode_base64url(":" .. apipassword) -- set authorization header with proper base64 encoded string return "Basic " .. content end diff --git a/docker/nginx/libs/skynet/utils.spec.lua b/docker/nginx/libs/skynet/utils.spec.lua new file mode 100644 index 00000000..171be4fa --- /dev/null +++ b/docker/nginx/libs/skynet/utils.spec.lua @@ -0,0 +1,65 @@ +-- luacheck: ignore io + +local utils = require('utils') +local skynet_utils = require('skynet.utils') + +describe("authorization_header", function() + local apipassword = "ddd0c1430fbf2708" + local expected_header = "Basic OmRkZDBjMTQzMGZiZjI3MDg" + + it("reads SIA_API_PASSWORD from env variable and returns a header", function() + -- stub getenv on SIA_API_PASSWORD + stub(utils, "getenv") + utils.getenv.on_call_with("SIA_API_PASSWORD").returns(apipassword) + + local header = skynet_utils.authorization_header() + + assert.is_equal(header, expected_header) + + -- revert stub to original function + utils.getenv:revert() + end) + + it("uses /data/sia/apipassword file if SIA_API_PASSWORD env var is missing", function() + -- stub /data/sia/apipassword file + stub(io, "open") + io.open.on_call_with("/data/sia/apipassword", "rb").returns(mock({ + read = spy.new(function() return apipassword end), + close = spy.new() + })) + + local header = skynet_utils.authorization_header() + + assert.is_equal(header, expected_header) + + -- revert stub to original function + io.open:revert() + end) + + it("should choose env variable over file if both are available", function() + -- stub getenv on SIA_API_PASSWORD + stub(utils, "getenv") + utils.getenv.on_call_with("SIA_API_PASSWORD").returns(apipassword) + + -- stub /data/sia/apipassword file + stub(io, "open") + io.open.on_call_with("/data/sia/apipassword", "rb").returns(mock({ + read = spy.new(function() return "foooooooooooooo" end), + close = spy.new() + })) + + local header = skynet_utils.authorization_header() + + assert.is_equal(header, "Basic OmRkZDBjMTQzMGZiZjI3MDg") + + -- revert stubs to original function + utils.getenv:revert() + io.open:revert() + end) + + it("should error out if neither env variable is available nor file exists", function() + assert.has_error(function() + skynet_utils.authorization_header() + end, "Error reading /data/sia/apipassword file") + end) +end) diff --git a/docker/nginx/libs/utils.lua b/docker/nginx/libs/utils.lua index 4330c94c..3b9b2592 100644 --- a/docker/nginx/libs/utils.lua +++ b/docker/nginx/libs/utils.lua @@ -42,4 +42,42 @@ function _M.extract_cookie_value(cookie_string, name_matcher) return string.sub(cookie, value_start) end +-- utility function that builds on os.getenv to get environment variable value +-- * will always return nil for both unset and empty env vars +-- * parse "boolean": "true" and "1" as true, "false" and "0" as false, throws for others +-- * parse "integer": any numerical string gets converted, otherwise returns nil +function _M.getenv(name, parse) + local value = os.getenv(name) + + -- treat empty string value as nil to simplify comparisons + if value == nil or value == "" then + return nil + end + + -- do not parse the value + if parse == nil then + return value + end + + -- try to parse as boolean + if parse == "boolean" then + if value == "true" or value == "1" then + return true + end + + if value == "false" or value == "0" then + return false + end + + error("utils.getenv: Parsing value '" .. tostring(value) .. "' to boolean is not supported") + end + + -- try to parse as integer + if parse == "integer" then + return tonumber(value) + end + + error("utils.getenv: Parsing to '" .. parse .. "' is not supported") +end + return _M diff --git a/docker/nginx/libs/utils.spec.lua b/docker/nginx/libs/utils.spec.lua index c853c8cd..af43e898 100644 --- a/docker/nginx/libs/utils.spec.lua +++ b/docker/nginx/libs/utils.spec.lua @@ -1,3 +1,5 @@ +-- luacheck: ignore os + local utils = require('utils') describe("is_table_empty", function() @@ -77,3 +79,125 @@ describe("extract_cookie_value", function() assert.are.equals(value, "MTY0NzUyr8jD-ytiWtspm0tGabKfooxeIDuWcXhJ3lnY0eEw==") end) end) + +describe("getenv", function() + before_each(function() + stub(os, "getenv") + end) + + after_each(function() + os.getenv:revert() + end) + + it("should return nil for not existing env var", function() + os.getenv.on_call_with("foo").returns(nil) + + assert.is_nil(utils.getenv("foo")) + end) + + it("should return nil for env var that is an empty string", function() + os.getenv.on_call_with("foo").returns("") + + assert.is_nil(utils.getenv("foo")) + end) + + it("should return the value as is when it is non empty string", function() + os.getenv.on_call_with("foo").returns("bar") + + assert.is_equal(utils.getenv("foo"), "bar") + end) + + describe("parse", function() + it("should throw on not supported parser", function() + os.getenv.on_call_with("foo").returns("test") + + assert.has_error(function() + utils.getenv("foo", "starwars") + end, "utils.getenv: Parsing to 'starwars' is not supported") + end) + + describe("as boolean", function() + it("should return nil for not existing env var", function() + os.getenv.on_call_with("foo").returns(nil) + + assert.is_nil(utils.getenv("foo", "boolean")) + end) + + it("should return nil for env var that is an empty string", function() + os.getenv.on_call_with("foo").returns("") + + assert.is_nil(utils.getenv("foo", "boolean")) + end) + + it("should parse 'true' string as true", function() + os.getenv.on_call_with("foo").returns("true") + + assert.is_true(utils.getenv("foo", "boolean")) + end) + + it("should parse '1' string as true", function() + os.getenv.on_call_with("foo").returns("1") + + assert.is_true(utils.getenv("foo", "boolean")) + end) + + it("should parse 'false' string as false", function() + os.getenv.on_call_with("foo").returns("false") + + assert.is_false(utils.getenv("foo", "boolean")) + end) + + it("should parse '0' string as false", function() + os.getenv.on_call_with("foo").returns("0") + + assert.is_false(utils.getenv("foo", "boolean")) + end) + + it("should throw an error for not supported string", function() + os.getenv.on_call_with("foo").returns("mandalorian") + + assert.has_error(function() + utils.getenv("foo", "boolean") + end, "utils.getenv: Parsing value 'mandalorian' to boolean is not supported") + end) + end) + + describe("as integer", function() + it("should return nil for not existing env var", function() + os.getenv.on_call_with("foo").returns(nil) + + assert.is_nil(utils.getenv("foo", "integer")) + end) + + it("should return nil for env var that is an empty string", function() + os.getenv.on_call_with("foo").returns("") + + assert.is_nil(utils.getenv("foo", "integer")) + end) + + it("should parse '0' string as 0", function() + os.getenv.on_call_with("foo").returns("0") + + assert.equals(utils.getenv("foo", "integer"), 0) + end) + + it("should parse '1' string as 1", function() + os.getenv.on_call_with("foo").returns("1") + + assert.equals(utils.getenv("foo", "integer"), 1) + end) + + it("should parse '-1' string as 1", function() + os.getenv.on_call_with("foo").returns("-1") + + assert.equals(utils.getenv("foo", "integer"), -1) + end) + + it("should return nil for non numerical string", function() + os.getenv.on_call_with("foo").returns("test") + + assert.is_nil(utils.getenv("foo", "integer")) + end) + end) + end) +end) diff --git a/docker/nginx/testing/.luacov b/docker/nginx/testing/.luacov new file mode 100644 index 00000000..2c55f3da --- /dev/null +++ b/docker/nginx/testing/.luacov @@ -0,0 +1,6 @@ +exclude = { + "/usr/local/openresty", -- internal openresty libraries + "rbusted", -- busted executable + "basexx", -- external library https://github.com/aiq/basexx +} +includeuntestedfiles = true diff --git a/docker/nginx/testing/Dockerfile b/docker/nginx/testing/Dockerfile new file mode 100644 index 00000000..ae2cd78f --- /dev/null +++ b/docker/nginx/testing/Dockerfile @@ -0,0 +1,11 @@ +FROM openresty/openresty:1.19.9.1-focal + +WORKDIR /etc/nginx + +RUN luarocks install lua-resty-http && \ + luarocks install hasher && \ + luarocks install busted + +COPY rbusted /etc/nginx/ + +CMD /etc/nginx/rbusted --verbose --pattern=spec /usr/local/openresty/site/lualib diff --git a/docker/nginx/testing/README.md b/docker/nginx/testing/README.md new file mode 100644 index 00000000..f40e8d95 --- /dev/null +++ b/docker/nginx/testing/README.md @@ -0,0 +1,3 @@ +# Running tests locally + +`docker run -v $(pwd)/docker/nginx/libs:/usr/local/openresty/site/lualib --rm -it $(docker build -q docker/nginx/testing)` diff --git a/docker/nginx/testing/rbusted b/docker/nginx/testing/rbusted new file mode 100755 index 00000000..94149350 --- /dev/null +++ b/docker/nginx/testing/rbusted @@ -0,0 +1,8 @@ +#!/usr/bin/env resty + +setmetatable(_G, nil) + +pcall(require, "luarocks.loader") + +-- Busted command-line runner +require "busted.runner"({ standalone = false }) From 1c3fcf69998e5ed1c3403451eff806614e6e5770 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 8 Apr 2022 11:51:53 +0200 Subject: [PATCH 2/5] add newlines on eof --- docker/nginx/libs/skynet/modules.spec.lua | 2 +- docker/nginx/libs/skynet/scanner.spec.lua | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/nginx/libs/skynet/modules.spec.lua b/docker/nginx/libs/skynet/modules.spec.lua index 0797a32d..0eaaf081 100644 --- a/docker/nginx/libs/skynet/modules.spec.lua +++ b/docker/nginx/libs/skynet/modules.spec.lua @@ -92,4 +92,4 @@ describe("is_disabled", function() skynet_modules.is_disabled("gandalf") end, "Module abbreviation 'gandalf' should be exactly one character long string") end) -end) \ No newline at end of file +end) diff --git a/docker/nginx/libs/skynet/scanner.spec.lua b/docker/nginx/libs/skynet/scanner.spec.lua index 400c301c..584f07ae 100644 --- a/docker/nginx/libs/skynet/scanner.spec.lua +++ b/docker/nginx/libs/skynet/scanner.spec.lua @@ -118,4 +118,4 @@ describe("scan_skylink_timer", function() "foo != bar" ) end) -end) \ No newline at end of file +end) From 4e64ddd798fdb629b577a0ade16004ce9acb0c46 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 8 Apr 2022 13:20:05 +0200 Subject: [PATCH 3/5] match lowercase true/false --- docker/nginx/libs/utils.lua | 4 ++-- docker/nginx/libs/utils.spec.lua | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docker/nginx/libs/utils.lua b/docker/nginx/libs/utils.lua index 3b9b2592..8b77d802 100644 --- a/docker/nginx/libs/utils.lua +++ b/docker/nginx/libs/utils.lua @@ -61,11 +61,11 @@ function _M.getenv(name, parse) -- try to parse as boolean if parse == "boolean" then - if value == "true" or value == "1" then + if string.lower(value) == "true" or value == "1" then return true end - if value == "false" or value == "0" then + if string.lower(value) == "false" or value == "0" then return false end diff --git a/docker/nginx/libs/utils.spec.lua b/docker/nginx/libs/utils.spec.lua index af43e898..78aa0833 100644 --- a/docker/nginx/libs/utils.spec.lua +++ b/docker/nginx/libs/utils.spec.lua @@ -135,6 +135,12 @@ describe("getenv", function() assert.is_true(utils.getenv("foo", "boolean")) end) + it("should parse 'True' string as true", function() + os.getenv.on_call_with("foo").returns("True") + + assert.is_true(utils.getenv("foo", "boolean")) + end) + it("should parse '1' string as true", function() os.getenv.on_call_with("foo").returns("1") @@ -147,6 +153,12 @@ describe("getenv", function() assert.is_false(utils.getenv("foo", "boolean")) end) + it("should parse 'False' string as false", function() + os.getenv.on_call_with("foo").returns("False") + + assert.is_false(utils.getenv("foo", "boolean")) + end) + it("should parse '0' string as false", function() os.getenv.on_call_with("foo").returns("0") From a0693ac6b66a2c709fb098e99725c2de0859e295 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 8 Apr 2022 13:20:48 +0200 Subject: [PATCH 4/5] fix bad test title --- docker/nginx/libs/utils.spec.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/libs/utils.spec.lua b/docker/nginx/libs/utils.spec.lua index 78aa0833..71ef086c 100644 --- a/docker/nginx/libs/utils.spec.lua +++ b/docker/nginx/libs/utils.spec.lua @@ -199,7 +199,7 @@ describe("getenv", function() assert.equals(utils.getenv("foo", "integer"), 1) end) - it("should parse '-1' string as 1", function() + it("should parse '-1' string as -1", function() os.getenv.on_call_with("foo").returns("-1") assert.equals(utils.getenv("foo", "integer"), -1) From 2e39fb47362fe3a2f13c5903efad400e2745b930 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 8 Apr 2022 13:24:35 +0200 Subject: [PATCH 5/5] remove spy stub that will never be called --- docker/nginx/libs/skynet/scanner.spec.lua | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/nginx/libs/skynet/scanner.spec.lua b/docker/nginx/libs/skynet/scanner.spec.lua index 584f07ae..533ef44c 100644 --- a/docker/nginx/libs/skynet/scanner.spec.lua +++ b/docker/nginx/libs/skynet/scanner.spec.lua @@ -54,9 +54,7 @@ describe("scan_skylink_timer", function() it("should exit early on premature", function() local resty_http = require("resty.http") - local request_uri = spy.new(function() - return { status = 200 } -- return 200 success - end) + local request_uri = spy.new() local httpc = mock({ request_uri = request_uri }) stub(resty_http, "new").returns(httpc)