From 9241979ca5cc19fb578760bbbae45e1aa34f6658 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 23 Feb 2022 17:40:57 +0100 Subject: [PATCH 1/7] clean up nginx cache specific code --- docker/nginx/conf.d/include/location-skylink | 88 +++++++++++--------- 1 file changed, 47 insertions(+), 41 deletions(-) diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index cf250cea..db3a7b86 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -25,41 +25,44 @@ set $skynet_proof ''; set $limit_rate 0; access_by_lua_block { - local httpc = require("resty.http").new() + -- the block below only makes sense if we are using nginx cache + if not ngx.var.skyd_disk_cache_enabled then + local httpc = require("resty.http").new() - -- detect whether requested skylink is v2 - local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04" - local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ" - - if isBase32v2 or isBase64v2 then - -- 10.10.10.10 points to sia service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/resolve/" .. ngx.var.skylink_v2, { - headers = { ["User-Agent"] = "Sia-Agent" } - }) + -- detect whether requested skylink is v2 + local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04" + local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ" + + if isBase32v2 or isBase64v2 then + -- 10.10.10.10 points to sia service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/resolve/" .. ngx.var.skylink_v2, { + headers = { ["User-Agent"] = "Sia-Agent" } + }) - -- print error and exit with 500 or exit with response if status is not 200 - if err or (res and res.status ~= ngx.HTTP_OK) then - ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status - ngx.header["content-type"] = "text/plain" - ngx.say(err or res.body) - return ngx.exit(ngx.status) + -- print error and exit with 500 or exit with response if status is not 200 + if err or (res and res.status ~= ngx.HTTP_OK) then + ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status + ngx.header["content-type"] = "text/plain" + ngx.say(err or res.body) + return ngx.exit(ngx.status) + end + + local json = require('cjson') + local resolve = json.decode(res.body) + ngx.var.skylink_v1 = resolve.skylink + ngx.var.skynet_proof = res.headers["Skynet-Proof"] end - local json = require('cjson') - local resolve = json.decode(res.body) - ngx.var.skylink_v1 = resolve.skylink - ngx.var.skynet_proof = res.headers["Skynet-Proof"] - end + -- check if skylink v1 is present on blocklist (compare hashes) + if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then + return require("skynet.blocklist").exit_illegal() + end - -- check if skylink v1 is present on blocklist (compare hashes) - if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then - return require("skynet.blocklist").exit_illegal() - end - - -- if skylink is found on nocache list then set internal nocache variable - -- to tell nginx that it should not try and cache this file (too large) - if ngx.shared.nocache:get(ngx.var.skylink_v1) then - ngx.var.nocache = "1" + -- if skylink is found on nocache list then set internal nocache variable + -- to tell nginx that it should not try and cache this file (too large) + if ngx.shared.nocache:get(ngx.var.skylink_v1) then + ngx.var.nocache = "1" + end end if require("skynet.account").accounts_enabled() then @@ -85,18 +88,21 @@ header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN") ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN") - -- not empty skynet_proof means this is a skylink v2 request - -- so we should replace the Skynet-Proof header with the one - -- we got from /skynet/resolve/ endpoint, otherwise we would - -- be serving cached empty v1 skylink Skynet-Proof header - if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then - ngx.header["Skynet-Proof"] = ngx.var.skynet_proof - end + -- the block below only makes sense if we are using nginx cache + if not ngx.var.skyd_disk_cache_enabled then + -- not empty skynet_proof means this is a skylink v2 request + -- so we should replace the Skynet-Proof header with the one + -- we got from /skynet/resolve/ endpoint, otherwise we would + -- be serving cached empty v1 skylink Skynet-Proof header + if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then + ngx.header["Skynet-Proof"] = ngx.var.skynet_proof + end - -- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold - -- (content length can be nil for already cached files - we can ignore them) - if ngx.header["Content-Length"] and tonumber(ngx.header["Content-Length"]) > 1e+9 then - ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"]) + -- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold + -- (content length can be nil for already cached files - we can ignore them) + if ngx.header["Content-Length"] and tonumber(ngx.header["Content-Length"]) > 1e+9 then + ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"]) + end end } From ccb271a3d7d4da8cbf796fd03451f63f472e7760 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Thu, 24 Feb 2022 10:57:08 +0100 Subject: [PATCH 2/7] switch malware-scanner to image --- docker-compose.malware-scanner.yml | 10 +++------- docker/malware-scanner/Dockerfile | 23 ----------------------- 2 files changed, 3 insertions(+), 30 deletions(-) delete mode 100644 docker/malware-scanner/Dockerfile diff --git a/docker-compose.malware-scanner.yml b/docker-compose.malware-scanner.yml index 946e8c5c..9fc68374 100644 --- a/docker-compose.malware-scanner.yml +++ b/docker-compose.malware-scanner.yml @@ -26,19 +26,15 @@ services: ipv4_address: 10.10.10.100 malware-scanner: - build: - context: ./docker/malware-scanner - dockerfile: Dockerfile - args: - branch: main + image: skynetlabs/malware-scanner container_name: malware-scanner restart: unless-stopped logging: *default-logging env_file: - .env environment: - - CLAMAV_IP=${CLAMAV_IP:-10.10.10.100} - - CLAMAV_PORT=${CLAMAV_PORT:-3310} + - CLAMAV_IP=10.10.10.100 + - CLAMAV_PORT=3310 - BLOCKER_IP=10.10.10.110 - BLOCKER_PORT=4000 expose: diff --git a/docker/malware-scanner/Dockerfile b/docker/malware-scanner/Dockerfile deleted file mode 100644 index 2466e48b..00000000 --- a/docker/malware-scanner/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM golang:1.17.3 -LABEL maintainer="SkynetLabs " - -ENV GOOS linux -ENV GOARCH amd64 - -ARG branch=main - -WORKDIR /root - -RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/malware-scanner.git && \ - cd malware-scanner && \ - go mod download && \ - make release - -ENV SKYNET_DB_HOST="localhost" -ENV SKYNET_DB_PORT="27017" -ENV SKYNET_DB_USER="username" -ENV SKYNET_DB_PASS="password" -ENV CLAMAV_IP=127.0.0.1 -ENV CLAMAV_PORT=3310 - -ENTRYPOINT ["malware-scanner"] From 69e2b593229848ce22c53bf62a063977eb044b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Wypch=C5=82o?= Date: Thu, 24 Feb 2022 13:16:30 +0100 Subject: [PATCH 3/7] switch abuse-scanner to image (#1766) --- dc | 4 ++-- ...abuse.yml => docker-compose.abuse-scanner.yml | 8 +++----- docker/abuse/Dockerfile | 16 ---------------- 3 files changed, 5 insertions(+), 23 deletions(-) rename docker-compose.abuse.yml => docker-compose.abuse-scanner.yml (88%) delete mode 100644 docker/abuse/Dockerfile diff --git a/dc b/dc index b3e68df4..d54bbf2d 100755 --- a/dc +++ b/dc @@ -41,9 +41,9 @@ for i in $(seq 1 ${#PORTAL_MODULES}); do COMPOSE_FILES+=" -f docker-compose.mongodb.yml" fi - # abuse module - alias "u" + # abuse-scanner module - alias "u" if [[ ${PORTAL_MODULES:i-1:1} == "u" ]]; then - COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.blocker.yml -f docker-compose.abuse.yml" + COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.blocker.yml -f docker-compose.abuse-scanner.yml" fi done diff --git a/docker-compose.abuse.yml b/docker-compose.abuse-scanner.yml similarity index 88% rename from docker-compose.abuse.yml rename to docker-compose.abuse-scanner.yml index e3f32750..f655b0ea 100644 --- a/docker-compose.abuse.yml +++ b/docker-compose.abuse-scanner.yml @@ -7,11 +7,9 @@ x-logging: &default-logging max-file: "3" services: - abuse: - build: - context: ./docker/abuse - dockerfile: Dockerfile - container_name: abuse + abuse-scanner: + image: skynetlabs/abuse-scanner + container_name: abuse-scanner restart: unless-stopped logging: *default-logging env_file: diff --git a/docker/abuse/Dockerfile b/docker/abuse/Dockerfile deleted file mode 100644 index f27fb769..00000000 --- a/docker/abuse/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM golang:1.16.7 -LABEL maintainer="SkynetLabs " - -ENV GOOS linux -ENV GOARCH amd64 - -ARG branch=main - -WORKDIR /root - -RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/abuse-scanner.git && \ - cd abuse-scanner && \ - go mod download && \ - make release - -ENTRYPOINT ["abuse-scanner"] From b0ce090ef5d497d1b10aada131e54f67a498f510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Wypch=C5=82o?= Date: Thu, 24 Feb 2022 13:16:54 +0100 Subject: [PATCH 4/7] remove unused docker-compose.nginx.yml file (#1765) --- docker-compose.uploads.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 docker-compose.uploads.yml diff --git a/docker-compose.uploads.yml b/docker-compose.uploads.yml deleted file mode 100644 index c7d3043a..00000000 --- a/docker-compose.uploads.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: "3.7" - -services: - nginx: - build: - context: ./docker/nginx - dockerfile: Dockerfile.bionic - args: - RESTY_ADD_PACKAGE_BUILDDEPS: git - RESTY_EVAL_PRE_CONFIGURE: git clone https://github.com/fdintino/nginx-upload-module /tmp/nginx-upload-module - RESTY_CONFIG_OPTIONS_MORE: --add-module=/tmp/nginx-upload-module - RESTY_EVAL_POST_MAKE: /usr/local/openresty/luajit/bin/luarocks install luasocket From 368eaa0fbe242504fc6ac7c60635fd42831f3c14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Wypch=C5=82o?= Date: Thu, 24 Feb 2022 13:18:37 +0100 Subject: [PATCH 5/7] switch blocker to image (#1764) --- docker-compose.blocker.yml | 4 +--- docker/blocker/Dockerfile | 16 ---------------- 2 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 docker/blocker/Dockerfile diff --git a/docker-compose.blocker.yml b/docker-compose.blocker.yml index 845bdc6a..3c1deeaa 100644 --- a/docker-compose.blocker.yml +++ b/docker-compose.blocker.yml @@ -13,9 +13,7 @@ services: - BLOCKER_PORT=4000 blocker: - build: - context: ./docker/blocker - dockerfile: Dockerfile + image: skynetlabs/blocker container_name: blocker restart: unless-stopped logging: *default-logging diff --git a/docker/blocker/Dockerfile b/docker/blocker/Dockerfile deleted file mode 100644 index 70d572b9..00000000 --- a/docker/blocker/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM golang:1.16.7 -LABEL maintainer="SkynetLabs " - -ENV GOOS linux -ENV GOARCH amd64 - -ARG branch=main - -WORKDIR /root - -RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/blocker.git && \ - cd blocker && \ - go mod download && \ - make release - -ENTRYPOINT ["blocker"] From 3942e3fa0e72d31ff0f53934ccadeb8dbc676650 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 25 Feb 2022 00:28:50 +0100 Subject: [PATCH 6/7] switch accounts service to docker image --- docker-compose.accounts.yml | 6 +----- docker/accounts/Dockerfile | 22 ---------------------- 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 docker/accounts/Dockerfile diff --git a/docker-compose.accounts.yml b/docker-compose.accounts.yml index a3941f6b..7c3ed921 100644 --- a/docker-compose.accounts.yml +++ b/docker-compose.accounts.yml @@ -20,11 +20,7 @@ services: - ACCOUNTS_LIMIT_ACCESS=${ACCOUNTS_LIMIT_ACCESS:-authenticated} # default to authenticated access only accounts: - build: - context: ./docker/accounts - dockerfile: Dockerfile - args: - branch: main + image: skynetlabs/skynet-accounts container_name: accounts restart: unless-stopped logging: *default-logging diff --git a/docker/accounts/Dockerfile b/docker/accounts/Dockerfile deleted file mode 100644 index 5cbf359a..00000000 --- a/docker/accounts/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM golang:1.16.7 -LABEL maintainer="SkynetLabs " - -ENV GOOS linux -ENV GOARCH amd64 - -ARG branch=main - -WORKDIR /root - -RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/skynet-accounts.git && \ - cd skynet-accounts && \ - go mod download && \ - make release - -ENV SKYNET_DB_HOST="localhost" -ENV SKYNET_DB_PORT="27017" -ENV SKYNET_DB_USER="username" -ENV SKYNET_DB_PASS="password" -ENV SKYNET_ACCOUNTS_PORT=3000 - -ENTRYPOINT ["skynet-accounts"] From b6dd4c5ef6de17566da14399e68cbc70c24f361d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Wypch=C5=82o?= Date: Fri, 25 Feb 2022 09:57:55 +0100 Subject: [PATCH 7/7] drop ipv6 support (#1768) --- docker/nginx/conf.d/server.dnslink.conf | 2 -- docker/nginx/conf.d/server.local.conf | 1 - docker/nginx/conf.d/server/server.account | 1 - docker/nginx/conf.d/server/server.api | 1 - docker/nginx/conf.d/server/server.hns | 1 - docker/nginx/conf.d/server/server.http | 1 - docker/nginx/conf.d/server/server.skylink | 1 - 7 files changed, 8 deletions(-) diff --git a/docker/nginx/conf.d/server.dnslink.conf b/docker/nginx/conf.d/server.dnslink.conf index 491bc389..c35536ea 100644 --- a/docker/nginx/conf.d/server.dnslink.conf +++ b/docker/nginx/conf.d/server.dnslink.conf @@ -2,14 +2,12 @@ lua_shared_dict dnslink 10m; server { listen 80 default_server; - listen [::]:80 default_server; include /etc/nginx/conf.d/server/server.dnslink; } server { listen 443 default_server; - listen [::]:443 default_server; ssl_certificate /etc/ssl/local-certificate.crt; ssl_certificate_key /etc/ssl/local-certificate.key; diff --git a/docker/nginx/conf.d/server.local.conf b/docker/nginx/conf.d/server.local.conf index 6c5af504..8a487a53 100644 --- a/docker/nginx/conf.d/server.local.conf +++ b/docker/nginx/conf.d/server.local.conf @@ -1,7 +1,6 @@ server { # local server - do not expose this port externally listen 8000; - listen [::]:8000; # secure traffic by limiting to only local networks include /etc/nginx/conf.d/include/local-network-only; diff --git a/docker/nginx/conf.d/server/server.account b/docker/nginx/conf.d/server/server.account index 2fb5551d..debfe572 100644 --- a/docker/nginx/conf.d/server/server.account +++ b/docker/nginx/conf.d/server/server.account @@ -1,5 +1,4 @@ listen 443 ssl http2; -listen [::]:443 ssl http2; include /etc/nginx/conf.d/include/ssl-settings; include /etc/nginx/conf.d/include/init-optional-variables; diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index e8fc0743..58648a9b 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -1,5 +1,4 @@ listen 443 ssl http2; -listen [::]:443 ssl http2; include /etc/nginx/conf.d/include/ssl-settings; include /etc/nginx/conf.d/include/init-optional-variables; diff --git a/docker/nginx/conf.d/server/server.hns b/docker/nginx/conf.d/server/server.hns index 3daa167f..9e68dc0b 100644 --- a/docker/nginx/conf.d/server/server.hns +++ b/docker/nginx/conf.d/server/server.hns @@ -1,5 +1,4 @@ listen 443 ssl http2; -listen [::]:443 ssl http2; include /etc/nginx/conf.d/include/ssl-settings; include /etc/nginx/conf.d/include/init-optional-variables; diff --git a/docker/nginx/conf.d/server/server.http b/docker/nginx/conf.d/server/server.http index 77cce00a..22ec6f30 100644 --- a/docker/nginx/conf.d/server/server.http +++ b/docker/nginx/conf.d/server/server.http @@ -1,5 +1,4 @@ listen 80; -listen [::]:80; include /etc/nginx/conf.d/include/init-optional-variables; diff --git a/docker/nginx/conf.d/server/server.skylink b/docker/nginx/conf.d/server/server.skylink index a8f659f1..7f628989 100644 --- a/docker/nginx/conf.d/server/server.skylink +++ b/docker/nginx/conf.d/server/server.skylink @@ -1,5 +1,4 @@ listen 443 ssl http2; -listen [::]:443 ssl http2; include /etc/nginx/conf.d/include/ssl-settings; include /etc/nginx/conf.d/include/init-optional-variables;