From 9241979ca5cc19fb578760bbbae45e1aa34f6658 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 23 Feb 2022 17:40:57 +0100 Subject: [PATCH 1/3] clean up nginx cache specific code --- docker/nginx/conf.d/include/location-skylink | 88 +++++++++++--------- 1 file changed, 47 insertions(+), 41 deletions(-) diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index cf250cea..db3a7b86 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -25,41 +25,44 @@ set $skynet_proof ''; set $limit_rate 0; access_by_lua_block { - local httpc = require("resty.http").new() + -- the block below only makes sense if we are using nginx cache + if not ngx.var.skyd_disk_cache_enabled then + local httpc = require("resty.http").new() - -- detect whether requested skylink is v2 - local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04" - local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ" - - if isBase32v2 or isBase64v2 then - -- 10.10.10.10 points to sia service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/resolve/" .. ngx.var.skylink_v2, { - headers = { ["User-Agent"] = "Sia-Agent" } - }) + -- detect whether requested skylink is v2 + local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04" + local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ" + + if isBase32v2 or isBase64v2 then + -- 10.10.10.10 points to sia service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/resolve/" .. ngx.var.skylink_v2, { + headers = { ["User-Agent"] = "Sia-Agent" } + }) - -- print error and exit with 500 or exit with response if status is not 200 - if err or (res and res.status ~= ngx.HTTP_OK) then - ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status - ngx.header["content-type"] = "text/plain" - ngx.say(err or res.body) - return ngx.exit(ngx.status) + -- print error and exit with 500 or exit with response if status is not 200 + if err or (res and res.status ~= ngx.HTTP_OK) then + ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status + ngx.header["content-type"] = "text/plain" + ngx.say(err or res.body) + return ngx.exit(ngx.status) + end + + local json = require('cjson') + local resolve = json.decode(res.body) + ngx.var.skylink_v1 = resolve.skylink + ngx.var.skynet_proof = res.headers["Skynet-Proof"] end - local json = require('cjson') - local resolve = json.decode(res.body) - ngx.var.skylink_v1 = resolve.skylink - ngx.var.skynet_proof = res.headers["Skynet-Proof"] - end + -- check if skylink v1 is present on blocklist (compare hashes) + if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then + return require("skynet.blocklist").exit_illegal() + end - -- check if skylink v1 is present on blocklist (compare hashes) - if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then - return require("skynet.blocklist").exit_illegal() - end - - -- if skylink is found on nocache list then set internal nocache variable - -- to tell nginx that it should not try and cache this file (too large) - if ngx.shared.nocache:get(ngx.var.skylink_v1) then - ngx.var.nocache = "1" + -- if skylink is found on nocache list then set internal nocache variable + -- to tell nginx that it should not try and cache this file (too large) + if ngx.shared.nocache:get(ngx.var.skylink_v1) then + ngx.var.nocache = "1" + end end if require("skynet.account").accounts_enabled() then @@ -85,18 +88,21 @@ header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN") ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN") - -- not empty skynet_proof means this is a skylink v2 request - -- so we should replace the Skynet-Proof header with the one - -- we got from /skynet/resolve/ endpoint, otherwise we would - -- be serving cached empty v1 skylink Skynet-Proof header - if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then - ngx.header["Skynet-Proof"] = ngx.var.skynet_proof - end + -- the block below only makes sense if we are using nginx cache + if not ngx.var.skyd_disk_cache_enabled then + -- not empty skynet_proof means this is a skylink v2 request + -- so we should replace the Skynet-Proof header with the one + -- we got from /skynet/resolve/ endpoint, otherwise we would + -- be serving cached empty v1 skylink Skynet-Proof header + if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then + ngx.header["Skynet-Proof"] = ngx.var.skynet_proof + end - -- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold - -- (content length can be nil for already cached files - we can ignore them) - if ngx.header["Content-Length"] and tonumber(ngx.header["Content-Length"]) > 1e+9 then - ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"]) + -- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold + -- (content length can be nil for already cached files - we can ignore them) + if ngx.header["Content-Length"] and tonumber(ngx.header["Content-Length"]) > 1e+9 then + ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"]) + end end } From 3942e3fa0e72d31ff0f53934ccadeb8dbc676650 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 25 Feb 2022 00:28:50 +0100 Subject: [PATCH 2/3] switch accounts service to docker image --- docker-compose.accounts.yml | 6 +----- docker/accounts/Dockerfile | 22 ---------------------- 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 docker/accounts/Dockerfile diff --git a/docker-compose.accounts.yml b/docker-compose.accounts.yml index a3941f6b..7c3ed921 100644 --- a/docker-compose.accounts.yml +++ b/docker-compose.accounts.yml @@ -20,11 +20,7 @@ services: - ACCOUNTS_LIMIT_ACCESS=${ACCOUNTS_LIMIT_ACCESS:-authenticated} # default to authenticated access only accounts: - build: - context: ./docker/accounts - dockerfile: Dockerfile - args: - branch: main + image: skynetlabs/skynet-accounts container_name: accounts restart: unless-stopped logging: *default-logging diff --git a/docker/accounts/Dockerfile b/docker/accounts/Dockerfile deleted file mode 100644 index 5cbf359a..00000000 --- a/docker/accounts/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM golang:1.16.7 -LABEL maintainer="SkynetLabs " - -ENV GOOS linux -ENV GOARCH amd64 - -ARG branch=main - -WORKDIR /root - -RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/skynet-accounts.git && \ - cd skynet-accounts && \ - go mod download && \ - make release - -ENV SKYNET_DB_HOST="localhost" -ENV SKYNET_DB_PORT="27017" -ENV SKYNET_DB_USER="username" -ENV SKYNET_DB_PASS="password" -ENV SKYNET_ACCOUNTS_PORT=3000 - -ENTRYPOINT ["skynet-accounts"] From b6dd4c5ef6de17566da14399e68cbc70c24f361d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Wypch=C5=82o?= Date: Fri, 25 Feb 2022 09:57:55 +0100 Subject: [PATCH 3/3] drop ipv6 support (#1768) --- docker/nginx/conf.d/server.dnslink.conf | 2 -- docker/nginx/conf.d/server.local.conf | 1 - docker/nginx/conf.d/server/server.account | 1 - docker/nginx/conf.d/server/server.api | 1 - docker/nginx/conf.d/server/server.hns | 1 - docker/nginx/conf.d/server/server.http | 1 - docker/nginx/conf.d/server/server.skylink | 1 - 7 files changed, 8 deletions(-) diff --git a/docker/nginx/conf.d/server.dnslink.conf b/docker/nginx/conf.d/server.dnslink.conf index 491bc389..c35536ea 100644 --- a/docker/nginx/conf.d/server.dnslink.conf +++ b/docker/nginx/conf.d/server.dnslink.conf @@ -2,14 +2,12 @@ lua_shared_dict dnslink 10m; server { listen 80 default_server; - listen [::]:80 default_server; include /etc/nginx/conf.d/server/server.dnslink; } server { listen 443 default_server; - listen [::]:443 default_server; ssl_certificate /etc/ssl/local-certificate.crt; ssl_certificate_key /etc/ssl/local-certificate.key; diff --git a/docker/nginx/conf.d/server.local.conf b/docker/nginx/conf.d/server.local.conf index 6c5af504..8a487a53 100644 --- a/docker/nginx/conf.d/server.local.conf +++ b/docker/nginx/conf.d/server.local.conf @@ -1,7 +1,6 @@ server { # local server - do not expose this port externally listen 8000; - listen [::]:8000; # secure traffic by limiting to only local networks include /etc/nginx/conf.d/include/local-network-only; diff --git a/docker/nginx/conf.d/server/server.account b/docker/nginx/conf.d/server/server.account index 2fb5551d..debfe572 100644 --- a/docker/nginx/conf.d/server/server.account +++ b/docker/nginx/conf.d/server/server.account @@ -1,5 +1,4 @@ listen 443 ssl http2; -listen [::]:443 ssl http2; include /etc/nginx/conf.d/include/ssl-settings; include /etc/nginx/conf.d/include/init-optional-variables; diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index e8fc0743..58648a9b 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -1,5 +1,4 @@ listen 443 ssl http2; -listen [::]:443 ssl http2; include /etc/nginx/conf.d/include/ssl-settings; include /etc/nginx/conf.d/include/init-optional-variables; diff --git a/docker/nginx/conf.d/server/server.hns b/docker/nginx/conf.d/server/server.hns index 3daa167f..9e68dc0b 100644 --- a/docker/nginx/conf.d/server/server.hns +++ b/docker/nginx/conf.d/server/server.hns @@ -1,5 +1,4 @@ listen 443 ssl http2; -listen [::]:443 ssl http2; include /etc/nginx/conf.d/include/ssl-settings; include /etc/nginx/conf.d/include/init-optional-variables; diff --git a/docker/nginx/conf.d/server/server.http b/docker/nginx/conf.d/server/server.http index 77cce00a..22ec6f30 100644 --- a/docker/nginx/conf.d/server/server.http +++ b/docker/nginx/conf.d/server/server.http @@ -1,5 +1,4 @@ listen 80; -listen [::]:80; include /etc/nginx/conf.d/include/init-optional-variables; diff --git a/docker/nginx/conf.d/server/server.skylink b/docker/nginx/conf.d/server/server.skylink index a8f659f1..7f628989 100644 --- a/docker/nginx/conf.d/server/server.skylink +++ b/docker/nginx/conf.d/server/server.skylink @@ -1,5 +1,4 @@ listen 443 ssl http2; -listen [::]:443 ssl http2; include /etc/nginx/conf.d/include/ssl-settings; include /etc/nginx/conf.d/include/init-optional-variables;