LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into add-nginx-stats-endpoint

This commit is contained in:
Karol Wypchło 2022-02-25 13:12:24 +01:00 committed by GitHub
parent 21e234b1c7 e5894b1b3c
commit 05fe41f75c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 57 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dc
View File

@ -41,9 +41,9 @@ for i in $(seq 1 ${#PORTAL_MODULES}); do
COMPOSE_FILES+=" -f docker-compose.mongodb.yml"
fi
# abuse module - alias "u"
# abuse-scanner module - alias "u"
if [[ ${PORTAL_MODULES:i-1:1} == "u" ]]; then
COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.blocker.yml -f docker-compose.abuse.yml"
COMPOSE_FILES+=" -f docker-compose.mongodb.yml -f docker-compose.blocker.yml -f docker-compose.abuse-scanner.yml"
fi
done

8
docker-compose.abuse.yml → docker-compose.abuse-scanner.yml
View File

@ -7,11 +7,9 @@ x-logging: &default-logging
max-file: "3"
services:
abuse:
build:
context: ./docker/abuse
dockerfile: Dockerfile
container_name: abuse
abuse-scanner:
image: skynetlabs/abuse-scanner
container_name: abuse-scanner
restart: unless-stopped
logging: *default-logging
env_file:

6
docker-compose.accounts.yml
View File

@ -20,11 +20,7 @@ services:
- ACCOUNTS_LIMIT_ACCESS=${ACCOUNTS_LIMIT_ACCESS:-authenticated} # default to authenticated access only
accounts:
build:
context: ./docker/accounts
dockerfile: Dockerfile
args:
branch: main
image: skynetlabs/skynet-accounts
container_name: accounts
restart: unless-stopped
logging: *default-logging

4
docker-compose.blocker.yml
View File

@ -13,9 +13,7 @@ services:
- BLOCKER_PORT=4000
blocker:
build:
context: ./docker/blocker
dockerfile: Dockerfile
image: skynetlabs/blocker
container_name: blocker
restart: unless-stopped
logging: *default-logging

10
docker-compose.malware-scanner.yml
View File

@ -26,19 +26,15 @@ services:
ipv4_address: 10.10.10.100
malware-scanner:
build:
context: ./docker/malware-scanner
dockerfile: Dockerfile
args:
branch: main
image: skynetlabs/malware-scanner
container_name: malware-scanner
restart: unless-stopped
logging: *default-logging
env_file:
- .env
environment:
- CLAMAV_IP=${CLAMAV_IP:-10.10.10.100}
- CLAMAV_PORT=${CLAMAV_PORT:-3310}
- CLAMAV_IP=10.10.10.100
- CLAMAV_PORT=3310
- BLOCKER_IP=10.10.10.110
- BLOCKER_PORT=4000
expose:

12
docker-compose.uploads.yml
View File

@ -1,12 +0,0 @@
version: "3.7"
services:
nginx:
build:
context: ./docker/nginx
dockerfile: Dockerfile.bionic
args:
RESTY_ADD_PACKAGE_BUILDDEPS: git
RESTY_EVAL_PRE_CONFIGURE: git clone https://github.com/fdintino/nginx-upload-module /tmp/nginx-upload-module
RESTY_CONFIG_OPTIONS_MORE: --add-module=/tmp/nginx-upload-module
RESTY_EVAL_POST_MAKE: /usr/local/openresty/luajit/bin/luarocks install luasocket

16
docker/abuse/Dockerfile
View File

@ -1,16 +0,0 @@
FROM golang:1.16.7
LABEL maintainer="SkynetLabs <devs@siasky.net>"
ENV GOOS linux
ENV GOARCH amd64
ARG branch=main
WORKDIR /root
RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/abuse-scanner.git && \
cd abuse-scanner && \
go mod download && \
make release
ENTRYPOINT ["abuse-scanner"]

22
docker/accounts/Dockerfile
View File

@ -1,22 +0,0 @@
FROM golang:1.16.7
LABEL maintainer="SkynetLabs <devs@siasky.net>"
ENV GOOS linux
ENV GOARCH amd64
ARG branch=main
WORKDIR /root
RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/skynet-accounts.git && \
cd skynet-accounts && \
go mod download && \
make release
ENV SKYNET_DB_HOST="localhost"
ENV SKYNET_DB_PORT="27017"
ENV SKYNET_DB_USER="username"
ENV SKYNET_DB_PASS="password"
ENV SKYNET_ACCOUNTS_PORT=3000
ENTRYPOINT ["skynet-accounts"]

16
docker/blocker/Dockerfile
View File

@ -1,16 +0,0 @@
FROM golang:1.16.7
LABEL maintainer="SkynetLabs <devs@siasky.net>"
ENV GOOS linux
ENV GOARCH amd64
ARG branch=main
WORKDIR /root
RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/blocker.git && \
cd blocker && \
go mod download && \
make release
ENTRYPOINT ["blocker"]

23
docker/malware-scanner/Dockerfile
View File

@ -1,23 +0,0 @@
FROM golang:1.17.3
LABEL maintainer="SkynetLabs <devs@siasky.net>"
ENV GOOS linux
ENV GOARCH amd64
ARG branch=main
WORKDIR /root
RUN git clone --single-branch --branch ${branch} https://github.com/SkynetLabs/malware-scanner.git && \
cd malware-scanner && \
go mod download && \
make release
ENV SKYNET_DB_HOST="localhost"
ENV SKYNET_DB_PORT="27017"
ENV SKYNET_DB_USER="username"
ENV SKYNET_DB_PASS="password"
ENV CLAMAV_IP=127.0.0.1
ENV CLAMAV_PORT=3310
ENTRYPOINT ["malware-scanner"]

86
docker/nginx/conf.d/include/location-skylink
View File

@ -25,41 +25,44 @@ set $skynet_proof '';
set $limit_rate 0;
access_by_lua_block {
local httpc = require("resty.http").new()
-- the block below only makes sense if we are using nginx cache
if not ngx.var.skyd_disk_cache_enabled then
local httpc = require("resty.http").new()
-- detect whether requested skylink is v2
local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04"
local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ"
-- detect whether requested skylink is v2
local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04"
local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ"
if isBase32v2 or isBase64v2 then
-- 10.10.10.10 points to sia service (alias not available when using resty-http)
local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/resolve/" .. ngx.var.skylink_v2, {
headers = { ["User-Agent"] = "Sia-Agent" }
})
if isBase32v2 or isBase64v2 then
-- 10.10.10.10 points to sia service (alias not available when using resty-http)
local res, err = httpc:request_uri("http://10.10.10.10:9980/skynet/resolve/" .. ngx.var.skylink_v2, {
headers = { ["User-Agent"] = "Sia-Agent" }
})
-- print error and exit with 500 or exit with response if status is not 200
if err or (res and res.status ~= ngx.HTTP_OK) then
ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
ngx.header["content-type"] = "text/plain"
ngx.say(err or res.body)
return ngx.exit(ngx.status)
-- print error and exit with 500 or exit with response if status is not 200
if err or (res and res.status ~= ngx.HTTP_OK) then
ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
ngx.header["content-type"] = "text/plain"
ngx.say(err or res.body)
return ngx.exit(ngx.status)
end
local json = require('cjson')
local resolve = json.decode(res.body)
ngx.var.skylink_v1 = resolve.skylink
ngx.var.skynet_proof = res.headers["Skynet-Proof"]
end
local json = require('cjson')
local resolve = json.decode(res.body)
ngx.var.skylink_v1 = resolve.skylink
ngx.var.skynet_proof = res.headers["Skynet-Proof"]
end
-- check if skylink v1 is present on blocklist (compare hashes)
if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then
return require("skynet.blocklist").exit_illegal()
end
-- check if skylink v1 is present on blocklist (compare hashes)
if require("skynet.blocklist").is_blocked(ngx.var.skylink_v1) then
return require("skynet.blocklist").exit_illegal()
end
-- if skylink is found on nocache list then set internal nocache variable
-- to tell nginx that it should not try and cache this file (too large)
if ngx.shared.nocache:get(ngx.var.skylink_v1) then
ngx.var.nocache = "1"
-- if skylink is found on nocache list then set internal nocache variable
-- to tell nginx that it should not try and cache this file (too large)
if ngx.shared.nocache:get(ngx.var.skylink_v1) then
ngx.var.nocache = "1"
end
end
if require("skynet.account").accounts_enabled() then
@ -85,18 +88,21 @@ header_filter_by_lua_block {
ngx.header["Skynet-Portal-Api"] = ngx.var.scheme .. "://" .. os.getenv("PORTAL_DOMAIN")
ngx.header["Skynet-Server-Api"] = ngx.var.scheme .. "://" .. os.getenv("SERVER_DOMAIN")
-- not empty skynet_proof means this is a skylink v2 request
-- so we should replace the Skynet-Proof header with the one
-- we got from /skynet/resolve/ endpoint, otherwise we would
-- be serving cached empty v1 skylink Skynet-Proof header
if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then
ngx.header["Skynet-Proof"] = ngx.var.skynet_proof
end
-- the block below only makes sense if we are using nginx cache
if not ngx.var.skyd_disk_cache_enabled then
-- not empty skynet_proof means this is a skylink v2 request
-- so we should replace the Skynet-Proof header with the one
-- we got from /skynet/resolve/ endpoint, otherwise we would
-- be serving cached empty v1 skylink Skynet-Proof header
if ngx.var.skynet_proof and ngx.var.skynet_proof ~= "" then
ngx.header["Skynet-Proof"] = ngx.var.skynet_proof
end
-- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold
-- (content length can be nil for already cached files - we can ignore them)
if ngx.header["Content-Length"] and tonumber(ngx.header["Content-Length"]) > 1e+9 then
ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"])
-- add skylink to nocache list if it exceeds 1GB (1e+9 bytes) threshold
-- (content length can be nil for already cached files - we can ignore them)
if ngx.header["Content-Length"] and tonumber(ngx.header["Content-Length"]) > 1e+9 then
ngx.shared.nocache:set(ngx.var.skylink_v1, ngx.header["Content-Length"])
end
end
}

2
docker/nginx/conf.d/server.dnslink.conf
View File

@ -2,14 +2,12 @@ lua_shared_dict dnslink 10m;
server {
listen 80 default_server;
listen [::]:80 default_server;
include /etc/nginx/conf.d/server/server.dnslink;
}
server {
listen 443 default_server;
listen [::]:443 default_server;
ssl_certificate /etc/ssl/local-certificate.crt;
ssl_certificate_key /etc/ssl/local-certificate.key;

1
docker/nginx/conf.d/server.local.conf
View File

@ -1,7 +1,6 @@
server {
# local server - do not expose this port externally
listen 8000;
listen [::]:8000;
# secure traffic by limiting to only local networks
include /etc/nginx/conf.d/include/local-network-only;

1
docker/nginx/conf.d/server/server.account
View File

@ -1,5 +1,4 @@
listen 443 ssl http2;
listen [::]:443 ssl http2;
include /etc/nginx/conf.d/include/ssl-settings;
include /etc/nginx/conf.d/include/init-optional-variables;

1
docker/nginx/conf.d/server/server.api
View File

@ -1,5 +1,4 @@
listen 443 ssl http2;
listen [::]:443 ssl http2;
include /etc/nginx/conf.d/include/ssl-settings;
include /etc/nginx/conf.d/include/init-optional-variables;

1
docker/nginx/conf.d/server/server.hns
View File

@ -1,5 +1,4 @@
listen 443 ssl http2;
listen [::]:443 ssl http2;
include /etc/nginx/conf.d/include/ssl-settings;
include /etc/nginx/conf.d/include/init-optional-variables;

1
docker/nginx/conf.d/server/server.http
View File

@ -1,5 +1,4 @@
listen 80;
listen [::]:80;
include /etc/nginx/conf.d/include/init-optional-variables;

1
docker/nginx/conf.d/server/server.skylink
View File

@ -1,5 +1,4 @@
listen 443 ssl http2;
listen [::]:443 ssl http2;
include /etc/nginx/conf.d/include/ssl-settings;
include /etc/nginx/conf.d/include/init-optional-variables;