Merge branch 'master' into add-nginx-stats-endpoint

dc

@@ -41,9 +41,9 @@ for i in $(seq 1 ${#PORTAL_MODULES}); do
     COMPOSE_FILES+=" -f docker-compose.mongodb.yml"
   fi
 
-  # abuse module - alias "u"
+  # abuse-scanner module - alias "u"
   if [[ ${PORTAL_MODULES:i-1:1} == "u" ]]; then
-    COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.blocker.yml -f docker-compose.abuse.yml"
+    COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.blocker.yml -f docker-compose.abuse-scanner.yml"
   fi
 done
 

docker-compose.abuse-scanner.yml

@@ -7,11 +7,9 @@ x-logging: &default-logging
     max-file: "3"
 
 services:
-  abuse:
+  abuse-scanner:
-    build:
+    image: skynetlabs/abuse-scanner
-      context: ./docker/abuse
+    container_name: abuse-scanner
-      dockerfile: Dockerfile
-    container_name: abuse
     restart: unless-stopped
     logging: *default-logging
     env_file:

docker-compose.accounts.yml

@@ -20,11 +20,7 @@ services:
       - ACCOUNTS_LIMIT_ACCESS=${ACCOUNTS_LIMIT_ACCESS:-authenticated} # default to authenticated access only
 
   accounts:
-    build:
+    image: skynetlabs/skynet-accounts
-      context: ./docker/accounts
-      dockerfile: Dockerfile
-      args:
-        branch: main
     container_name: accounts
     restart: unless-stopped
     logging: *default-logging

docker-compose.blocker.yml

@@ -13,9 +13,7 @@ services:
       - BLOCKER_PORT=4000
 
   blocker:
-    build:
+    image: skynetlabs/blocker
-      context: ./docker/blocker
-      dockerfile: Dockerfile
     container_name: blocker
     restart: unless-stopped
     logging: *default-logging

docker-compose.malware-scanner.yml

@@ -26,19 +26,15 @@ services:
         ipv4_address: 10.10.10.100
 
   malware-scanner:
-    build:
+    image: skynetlabs/malware-scanner
-      context: ./docker/malware-scanner
-      dockerfile: Dockerfile
-      args:
-        branch: main
     container_name: malware-scanner
     restart: unless-stopped
     logging: *default-logging
     env_file:
       - .env
     environment:
-      - CLAMAV_IP=${CLAMAV_IP:-10.10.10.100}
+      - CLAMAV_IP=10.10.10.100
-      - CLAMAV_PORT=${CLAMAV_PORT:-3310}
+      - CLAMAV_PORT=3310
       - BLOCKER_IP=10.10.10.110
       - BLOCKER_PORT=4000
     expose:

docker-compose.uploads.yml

@@ -1,12 +0,0 @@
-version: "3.7"
-
-services:
-  nginx:
-    build:
-      context: ./docker/nginx
-      dockerfile: Dockerfile.bionic
-      args:
-        RESTY_ADD_PACKAGE_BUILDDEPS: git
-        RESTY_EVAL_PRE_CONFIGURE: git clone https://github.com/fdintino/nginx-upload-module /tmp/nginx-upload-module
-        RESTY_CONFIG_OPTIONS_MORE: --add-module=/tmp/nginx-upload-module
-        RESTY_EVAL_POST_MAKE: /usr/local/openresty/luajit/bin/luarocks install luasocket

docker/abuse/Dockerfile

@@ -1,16 +0,0 @@
-FROM golang:1.16.7
-LABEL maintainer="SkynetLabs <devs@siasky.net>"
-
-ENV GOOS linux
-ENV GOARCH amd64
-
-ARG branch=main
-
-WORKDIR /root
-
-RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/abuse-scanner.git && \
-    cd abuse-scanner && \
-    go mod download && \
-    make release
-
-ENTRYPOINT ["abuse-scanner"]

docker/accounts/Dockerfile

@@ -1,22 +0,0 @@
-FROM golang:1.16.7
-LABEL maintainer="SkynetLabs <devs@siasky.net>"
-
-ENV GOOS linux
-ENV GOARCH amd64
-
-ARG branch=main
-
-WORKDIR /root
-
-RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/skynet-accounts.git && \
-    cd skynet-accounts && \
-    go mod download && \
-    make release
-
-ENV SKYNET_DB_HOST="localhost"
-ENV SKYNET_DB_PORT="27017"
-ENV SKYNET_DB_USER="username"
-ENV SKYNET_DB_PASS="password"
-ENV SKYNET_ACCOUNTS_PORT=3000
-
-ENTRYPOINT ["skynet-accounts"]

docker/blocker/Dockerfile

@@ -1,16 +0,0 @@
-FROM golang:1.16.7
-LABEL maintainer="SkynetLabs <devs@siasky.net>"
-
-ENV GOOS linux
-ENV GOARCH amd64
-
-ARG branch=main
-
-WORKDIR /root
-
-RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/blocker.git && \
-    cd blocker && \
-    go mod download && \
-    make release
-
-ENTRYPOINT ["blocker"]

docker/malware-scanner/Dockerfile

@@ -1,23 +0,0 @@
-FROM golang:1.17.3
-LABEL maintainer="SkynetLabs <devs@siasky.net>"
-
-ENV GOOS linux
-ENV GOARCH amd64
-
-ARG branch=main
-
-WORKDIR /root
-
-RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/malware-scanner.git && \
-    cd malware-scanner && \
-    go mod download && \
-    make release
-
-ENV SKYNET_DB_HOST="localhost"
-ENV SKYNET_DB_PORT="27017"
-ENV SKYNET_DB_USER="username"
-ENV SKYNET_DB_PASS="password"
-ENV CLAMAV_IP=127.0.0.1
-ENV CLAMAV_PORT=3310
-
-ENTRYPOINT ["malware-scanner"]

docker/nginx/conf.d/include/location-skylink

@@ -25,41 +25,44 @@ set $skynet_proof '';
 set $limit_rate 0;
 
 access_by_lua_block {
-    local httpc = require("resty.http").new()
+    -- the block below only makes sense if we are using nginx cache
+    if not ngx.var.skyd_disk_cache_enabled then
+        local httpc = require("resty.http").new()
 
-    -- detect whether requested skylink is v2
+        -- detect whether requested skylink is v2
-    local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04"
+        local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04"
-    local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ"
+        local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ"
-    
+        
-    if isBase32v2 or isBase64v2 then
+        if isBase32v2 or isBase64v2 then
-        -- 10.10.10.10 points to sia service (alias not available when using resty-http)
+            -- 10.10.10.10 points to sia service (alias not available when using resty-http)
-        local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/resolve/" .. ngx.var.skylink_v2, {
+            local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/resolve/" .. ngx.var.skylink_v2, {
-            headers = { ["User-Agent"] = "Sia-Agent" }
+                headers = { ["User-Agent"] = "Sia-Agent" }
-        })
+            })
 
-        -- print error and exit with 500 or exit with response if status is not 200
+            -- print error and exit with 500 or exit with response if status is not 200
-        if err or (res and res.status ~= ngx.HTTP_OK) then
+            if err or (res and res.status ~= ngx.HTTP_OK) then
-            ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
+                ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
-            ngx.header["content-type"] = "text/plain"
+                ngx.header["content-type"] = "text/plain"
-            ngx.say(err or res.body)
+                ngx.say(err or res.body)
-            return ngx.exit(ngx.status)
+                return ngx.exit(ngx.status)
+            end
+
+            local json = require('cjson')
+            local resolve = json.decode(res.body)
+            ngx.var.skylink_v1 = resolve.skylink
+            ngx.var.skynet_proof = res.headers["Skynet-Proof"]
         end
 
-        local json = require('cjson')
+        -- check if skylink v1 is present on blocklist (compare hashes)
-        local resolve = json.decode(res.body)
+        if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then
-        ngx.var.skylink_v1 = resolve.skylink
+            return require("skynet.blocklist").exit_illegal()
-        ngx.var.skynet_proof = res.headers["Skynet-Proof"]
+        end
-    end
 
-    -- check if skylink v1 is present on blocklist (compare hashes)
+        -- if skylink is found on nocache list then set internal nocache variable
-    if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then
+        -- to tell nginx that it should not try and cache this file (too large)
-        return require("skynet.blocklist").exit_illegal()
+        if ngx.shared.nocache:get(ngx.var.skylink_v1) then
-    end
+            ngx.var.nocache = "1"
-
+        end
-    -- if skylink is found on nocache list then set internal nocache variable
-    -- to tell nginx that it should not try and cache this file (too large)
-    if ngx.shared.nocache:get(ngx.var.skylink_v1) then
-        ngx.var.nocache = "1"
     end
 
     if require("skynet.account").accounts_enabled() then
@@ -85,18 +88,21 @@ header_filter_by_lua_block {
     ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN")
     ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN")
 
-    -- not empty skynet_proof means this is a skylink v2 request
+    -- the block below only makes sense if we are using nginx cache
-    -- so we should replace the Skynet-Proof header with the one
+    if not ngx.var.skyd_disk_cache_enabled then
-    -- we got from /skynet/resolve/ endpoint, otherwise we would
+        -- not empty skynet_proof means this is a skylink v2 request
-    -- be serving cached empty v1 skylink Skynet-Proof header
+        -- so we should replace the Skynet-Proof header with the one
-    if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then
+        -- we got from /skynet/resolve/ endpoint, otherwise we would
-        ngx.header["Skynet-Proof"] = ngx.var.skynet_proof
+        -- be serving cached empty v1 skylink Skynet-Proof header
-    end
+        if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then
+            ngx.header["Skynet-Proof"] = ngx.var.skynet_proof
+        end
 
-    -- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold
+        -- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold
-    -- (content length can be nil for already cached files - we can ignore them)
+        -- (content length can be nil for already cached files - we can ignore them)
-    if ngx.header["Content-Length"] and tonumber(ngx.header["Content-Length"]) > 1e+9 then
+        if ngx.header["Content-Length"] and tonumber(ngx.header["Content-Length"]) > 1e+9 then
-        ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"])
+            ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"])
+        end
     end
 }
 

docker/nginx/conf.d/server.dnslink.conf

@@ -2,14 +2,12 @@ lua_shared_dict dnslink 10m;
 
 server {
     listen 80 default_server;
-    listen [::]:80 default_server;
     
     include /etc/nginx/conf.d/server/server.dnslink;
 }
 
 server {
     listen 443 default_server;
-    listen [::]:443 default_server;
 
     ssl_certificate     /etc/ssl/local-certificate.crt;
     ssl_certificate_key /etc/ssl/local-certificate.key;

docker/nginx/conf.d/server.local.conf

@@ -1,7 +1,6 @@
 server {
     # local server - do not expose this port externally
     listen 8000;
-    listen [::]:8000;
 
     # secure traffic by limiting to only local networks
     include /etc/nginx/conf.d/include/local-network-only;

docker/nginx/conf.d/server/server.account

@@ -1,5 +1,4 @@
 listen 443 ssl http2;
-listen [::]:443 ssl http2;
 
 include /etc/nginx/conf.d/include/ssl-settings;
 include /etc/nginx/conf.d/include/init-optional-variables;

docker/nginx/conf.d/server/server.api

@@ -1,5 +1,4 @@
 listen 443 ssl http2;
-listen [::]:443 ssl http2;
 
 include /etc/nginx/conf.d/include/ssl-settings;
 include /etc/nginx/conf.d/include/init-optional-variables;

docker/nginx/conf.d/server/server.hns

@@ -1,5 +1,4 @@
 listen 443 ssl http2;
-listen [::]:443 ssl http2;
 
 include /etc/nginx/conf.d/include/ssl-settings;
 include /etc/nginx/conf.d/include/init-optional-variables;

docker/nginx/conf.d/server/server.http

@@ -1,5 +1,4 @@
 listen 80;
-listen [::]:80;
 
 include /etc/nginx/conf.d/include/init-optional-variables;
 

docker/nginx/conf.d/server/server.skylink

@@ -1,5 +1,4 @@
 listen 443 ssl http2;
-listen [::]:443 ssl http2;
 
 include /etc/nginx/conf.d/include/ssl-settings;
 include /etc/nginx/conf.d/include/init-optional-variables;