diff --git a/docker-compose.yml b/docker-compose.yml index 3c364523..124b7c6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,8 +17,8 @@ services: caddy: build: - context: . - dockerfile: ./docker/caddy/Dockerfile + context: ./docker/caddy + dockerfile: Dockerfile container_name: caddy restart: on-failure env_file: @@ -37,10 +37,15 @@ services: - nginx nginx: - image: openresty/openresty:1.15.8.3-2-xenial + build: + context: . + dockerfile: ./docker/nginx/Dockerfile container_name: nginx restart: on-failure + env_file: + - .env volumes: + - ./docker/nginx/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf:ro - ./docker/nginx/conf.d:/etc/nginx/conf.d:ro - ./docker/data/nginx/cache:/data/nginx/cache - ./docker/data/nginx/logs:/usr/local/openresty/nginx/logs @@ -57,6 +62,7 @@ services: dockerfile: Dockerfile container_name: handshake restart: on-failure + command: hnd --log-console=false environment: - HSD_HTTP_HOST=0.0.0.0 - HSD_NETWORK=main @@ -90,7 +96,7 @@ services: - 3100 depends_on: - handshake - - caddy + - nginx health-check: build: @@ -104,11 +110,9 @@ services: networks: - shared environment: - - PORTAL_URL=caddy - - HOSTNAME=health-check - - NODE_TLS_REJECT_UNAUTHORIZED=0 + - PORTAL_URL=nginx expose: - 3100 depends_on: - docker-host - - caddy + - nginx diff --git a/docker/caddy/Caddyfile b/docker/caddy/Caddyfile index 38909093..77485c4e 100644 --- a/docker/caddy/Caddyfile +++ b/docker/caddy/Caddyfile @@ -1,91 +1,16 @@ -(webserver) { - root * /home/user/public_html - file_server - encode zstd gzip - - @skylink { - path_regexp skylink ^/([a-zA-Z0-9-_]{46}(/.*)?)$ - } - - @skylink_file { - path_regexp skylink_file ^/file/([a-zA-Z0-9-_]{46}(/.*)?)$ - } - - @options { - method OPTIONS - } - - @blacklist { - method GET - path /blacklist - } - - @portals { - method GET - path /portals - } - - # OPTIONS headers to allow CORS https://enable-cors.org - handle @options { - header { - Access-Control-Allow-Origin * - Access-Control-Allow-Methods GET,POST,OPTIONS - Access-Control-Allow-Headers DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range - Access-Control-Max-Age 1728000 - } - respond 204 - } - - reverse_proxy /health-check health-check:3100 - reverse_proxy /hns/* handshake-api:3100 - reverse_proxy /hnsres/* handshake-api:3100 - - reverse_proxy @blacklist nginx:80 { - header_up User-Agent Sia-Agent - header_down Access-Control-Allow-Origin * - } - reverse_proxy @portals nginx:80 { - header_up User-Agent Sia-Agent - header_down Access-Control-Allow-Origin * - } - reverse_proxy /stats nginx:80 { - header_up User-Agent Sia-Agent - header_down Access-Control-Allow-Origin * - } - reverse_proxy /statsdown nginx:80 { - header_up User-Agent Sia-Agent - header_down Access-Control-Allow-Origin * - } - reverse_proxy @skylink nginx:80 { - header_up User-Agent Sia-Agent - header_down Access-Control-Expose-Headers skynet-file-metadata - header_down Access-Control-Allow-Origin * - } - reverse_proxy @skylink_file nginx:80 { - header_up User-Agent Sia-Agent - header_down Access-Control-Expose-Headers skynet-file-metadata - header_down Access-Control-Allow-Origin * - } - reverse_proxy /skynet/skyfile* nginx:80 { - header_up User-Agent Sia-Agent - header_up Authorization "Basic {env.SIA_API_AUTHORIZATION}" - header_down Access-Control-Allow-Origin * - } -} - (custom.domain) { {$DOMAIN_NAME} { tls {$EMAIL_ADDRESS} - import webserver + reverse_proxy nginx:80 } } (siasky.net) { siasky.net, *.siasky.net { tls { - dns cloudflare {env.CLOUDFLARE_AUTH_TOKEN} + dns route53 } - import webserver + reverse_proxy nginx:80 } } @@ -94,7 +19,7 @@ tls internal { on_demand } - import webserver + reverse_proxy nginx:80 } } diff --git a/docker/caddy/Dockerfile b/docker/caddy/Dockerfile index c6356699..d0f22045 100644 --- a/docker/caddy/Dockerfile +++ b/docker/caddy/Dockerfile @@ -1,20 +1,7 @@ -FROM node:12.18.0 AS client-builder - -COPY src ./src -COPY static ./static -COPY gatsby-config.js . -COPY package.json . -COPY yarn.lock . - -ENV CYPRESS_INSTALL_BINARY 0 -RUN yarn --frozen-lockfile -RUN yarn build - FROM caddy:2.1.1-builder AS caddy-builder -RUN caddy-builder github.com/caddy-dns/cloudflare +RUN caddy-builder github.com/caddy-dns/cloudflare github.com/caddy-dns/route53 FROM caddy:2.1.1 -COPY --from=client-builder /public /home/user/public_html COPY --from=caddy-builder /usr/bin/caddy /usr/bin/caddy diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile new file mode 100644 index 00000000..ec521767 --- /dev/null +++ b/docker/nginx/Dockerfile @@ -0,0 +1,15 @@ +FROM node:14.5 AS webportal-builder + +COPY src ./src +COPY static ./static +COPY gatsby-config.js . +COPY package.json . +COPY yarn.lock . + +ENV CYPRESS_INSTALL_BINARY 0 +RUN yarn --frozen-lockfile +RUN yarn build + +FROM openresty/openresty:1.15.8.3-2-xenial + +COPY --from=webportal-builder /public /var/www/webportal diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 262db21e..9cead372 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -21,7 +21,7 @@ real_ip_header X-Forwarded-For; # The upload siad can be run in normal mode. Set the port to '9980' if # you do not want to run your portal in the double siad setup. upstream siad-upload { - server docker-host:9970; + server docker-host:9980; } upstream siad-download { @@ -41,39 +41,103 @@ server { client_body_buffer_size 128k; client_max_body_size 128k; - rewrite "^(/([a-zA-Z0-9-_]{46})[^/]{0})$" $1/ permanent; - rewrite "^(/hns/[^/]+)[^/]{0}$" $1/ permanent; + location / { + include /etc/nginx/conf.d/include/cors; + + root /var/www/webportal; + } location /blacklist { + include /etc/nginx/conf.d/include/cors; + proxy_cache skynet; proxy_cache_valid any 1m; # cache blacklist for 1 minute + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://siad-download/skynet/blacklist; + } + + location /skynet/blacklist { + include /etc/nginx/conf.d/include/cors; + + proxy_cache skynet; + proxy_cache_valid any 1m; # cache blacklist for 1 minute + proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-download/skynet/blacklist; } location /portals { + include /etc/nginx/conf.d/include/cors; + proxy_cache skynet; proxy_cache_valid any 1m; # cache portals for 1 minute + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://siad-download/skynet/portals; + } + + location /skynet/portals { + include /etc/nginx/conf.d/include/cors; + + proxy_cache skynet; + proxy_cache_valid any 1m; # cache portals for 1 minute + proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-download/skynet/portals; } location /stats { + include /etc/nginx/conf.d/include/cors; + proxy_cache skynet; proxy_cache_valid any 1m; # cache stats for 1 minute + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://siad-upload/skynet/stats; + } + + location /skynet/stats { + include /etc/nginx/conf.d/include/cors; + + proxy_cache skynet; + proxy_cache_valid any 1m; # cache stats for 1 minute + proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-upload/skynet/stats; } location /statsdown { + include /etc/nginx/conf.d/include/cors; + proxy_cache skynet; proxy_cache_valid any 1m; # cache stats for 1 minute + proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-download/skynet/stats; } + location /health-check { + include /etc/nginx/conf.d/include/cors; + + proxy_pass http://health-check:3100; + } + + location /hns { + include /etc/nginx/conf.d/include/cors; + + proxy_pass http://handshake-api:3100; + } + + location /hnsres { + include /etc/nginx/conf.d/include/cors; + + proxy_pass http://handshake-api:3100; + } + location /skynet/skyfile { + include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/sia-auth; + limit_conn uploads_by_ip 10; # ddos protection: max 10 uploads at a time client_max_body_size 1000M; # make sure to limit the size of upload to a sane value proxy_read_timeout 600; proxy_request_buffering off; # stream uploaded files through the proxy as it comes in proxy_set_header Expect $http_expect; + proxy_set_header User-Agent: Sia-Agent; # Extract 3 sets of 2 characters from $request_id and assign to $dir1, $dir2, $dir3 # respectfully. The rest of the $request_id is going to be assigned to $dir4. @@ -93,11 +157,15 @@ server { } location ~ "/skynet/skyfile/(.+)" { + include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/sia-auth; + limit_conn uploads_by_ip 10; # ddos protection: max 10 uploads at a time client_max_body_size 1000M; # make sure to limit the size of upload to a sane value proxy_read_timeout 600; proxy_request_buffering off; # stream uploaded files through the proxy as it comes in proxy_set_header Expect $http_expect; + proxy_set_header User-Agent: Sia-Agent; # we need to explicitly use set directive here because $1 will contain the siapath with # decoded whitespaces and set will re-encode it for us before passing it to proxy_pass @@ -108,6 +176,8 @@ server { } location ~ "^/([a-zA-Z0-9-_]{46}(/.*)?)$" { + include /etc/nginx/conf.d/include/cors; + limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time # we need to explicitly use set directive here because $1 will contain the skylink with @@ -115,6 +185,7 @@ server { set $skylink $1; proxy_read_timeout 600; + proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) proxy_pass http://siad-download/skynet/skylink/$skylink$is_args$args; @@ -130,6 +201,8 @@ server { } location ~ "^/file/([a-zA-Z0-9-_]{46}(/.*)?)$" { + include /etc/nginx/conf.d/include/cors; + limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time # we need to explicitly use set directive here because $1 will contain the skylink with @@ -137,6 +210,7 @@ server { set $skylink $1; proxy_read_timeout 600; + proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) # this alias also adds attachment=true url param to force download the file proxy_pass http://siad-download/skynet/skylink/$skylink?attachment=true&$args; diff --git a/docker/nginx/conf.d/cors.conf b/docker/nginx/conf.d/cors.conf new file mode 100644 index 00000000..6c8cff35 --- /dev/null +++ b/docker/nginx/conf.d/cors.conf @@ -0,0 +1,4 @@ +more_set_headers 'Access-Control-Allow-Origin: *'; +more_set_headers 'Access-Control-Allow-Methods: GET, POST, OPTIONS, PUT, DELETE'; +more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; +more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range'; diff --git a/docker/nginx/conf.d/gzip.conf b/docker/nginx/conf.d/gzip.conf new file mode 100644 index 00000000..7d7a962c --- /dev/null +++ b/docker/nginx/conf.d/gzip.conf @@ -0,0 +1,18 @@ +# enables gzip compression +gzip on; + +# set the gzip compression level (1-9) +gzip_comp_level 6; + +# tells proxies to cache both gzipped and regular versions of a resource +gzip_vary on; + +# informs NGINX to not compress anything smaller than the defined size +gzip_min_length 256; + +# compress data even for clients that are connecting via proxies if a response header includes +# the "expired", "no-cache", "no-store", "private", and "Authorization" parameters +gzip_proxied expired no-cache no-store private auth; + +# enables the types of files that can be compressed +gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/vnd.ms-fontobject application/x-font-ttf font/opentype image/svg+xml image/x-icon; diff --git a/docker/nginx/conf.d/include/cors b/docker/nginx/conf.d/include/cors new file mode 100644 index 00000000..1490624b --- /dev/null +++ b/docker/nginx/conf.d/include/cors @@ -0,0 +1,9 @@ +if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Max-Age' 1728000; # valid for 20 days + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 204; +} diff --git a/docker/nginx/conf.d/include/sia-auth b/docker/nginx/conf.d/include/sia-auth new file mode 100644 index 00000000..8983bb1f --- /dev/null +++ b/docker/nginx/conf.d/include/sia-auth @@ -0,0 +1,10 @@ +rewrite_by_lua_block { + -- local b64 = require("ngx.base64") + -- pull apipassword from SIA_API_AUTHORIZATION environment variable + -- local apipassword = os.getenv("SIA_API_AUTHORIZATION") + -- encode the user:password authorization string + -- (in our case user is empty so it is just :password) + -- local content = b64.encode_base64url(":" .. apipassword) + -- set authorization header with proper base64 encoded string + ngx.req.set_header("Authorization", "Basic " .. os.getenv("SIA_API_AUTHORIZATION")) +} diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf new file mode 100644 index 00000000..1fb6a5f0 --- /dev/null +++ b/docker/nginx/nginx.conf @@ -0,0 +1,64 @@ +# nginx.conf -- docker-openresty +# +# This file is installed to: +# `/usr/local/openresty/nginx/conf/nginx.conf` +# and is the file loaded by nginx at startup, +# unless the user specifies otherwise. +# +# It tracks the upstream OpenResty's `nginx.conf`, but removes the `server` +# section and adds this directive: +# `include /etc/nginx/conf.d/*.conf;` +# +# The `docker-openresty` file `nginx.vh.default.conf` is copied to +# `/etc/nginx/conf.d/default.conf`. It contains the `server section +# of the upstream `nginx.conf`. +# +# See https://github.com/openresty/docker-openresty/blob/master/README.md#nginx-config-files +# + +# expose environment variables +env SIA_API_AUTHORIZATION; + +#user nobody; +worker_processes 1; + +#error_log logs/error.log; +#error_log logs/error.log notice; +#error_log logs/error.log info; + +#pid logs/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include mime.types; + default_type application/octet-stream; + + #log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + # '$status $body_bytes_sent "$http_referer" ' + # '"$http_user_agent" "$http_x_forwarded_for"'; + + #access_log logs/access.log main; + + # See Move default writable paths to a dedicated directory (#119) + # https://github.com/openresty/docker-openresty/issues/119 + client_body_temp_path /var/run/openresty/nginx-client-body; + proxy_temp_path /var/run/openresty/nginx-proxy; + fastcgi_temp_path /var/run/openresty/nginx-fastcgi; + uwsgi_temp_path /var/run/openresty/nginx-uwsgi; + scgi_temp_path /var/run/openresty/nginx-scgi; + + sendfile on; + #tcp_nopush on; + + #keepalive_timeout 0; + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/health-check/checks.js b/health-check/checks.js index 51d78506..8e29b3dc 100644 --- a/health-check/checks.js +++ b/health-check/checks.js @@ -5,7 +5,7 @@ async function uploadCheck(done) { const time = process.hrtime(); superagent - .post(`https://${process.env.PORTAL_URL}/skynet/skyfile`) + .post(`http://${process.env.PORTAL_URL}/skynet/skyfile`) .attach("file", "package.json", "package.json") .end((err, res) => { const statusCode = (res && res.statusCode) || (err && err.statusCode) || null; @@ -22,7 +22,7 @@ async function uploadCheck(done) { function downloadCheck(done) { const time = process.hrtime(); - superagent.get(`https://${process.env.PORTAL_URL}`).end((err, res) => { + superagent.get(`http://${process.env.PORTAL_URL}`).end((err, res) => { const statusCode = (res && res.statusCode) || (err && err.statusCode) || null; done({ diff --git a/health-check/index.js b/health-check/index.js index 18d2ad79..c34d3770 100644 --- a/health-check/index.js +++ b/health-check/index.js @@ -9,7 +9,7 @@ const bodyparser = require("body-parser"); require("./schedule"); -const host = process.env.HOSTNAME || "localhost"; +const host = process.env.HOSTNAME || "0.0.0.0"; const port = process.env.PORT || 3100; const server = express(); diff --git a/setup-scripts/setup-docker-services.sh b/setup-scripts/setup-docker-services.sh index 26cd6c95..92525f07 100755 --- a/setup-scripts/setup-docker-services.sh +++ b/setup-scripts/setup-docker-services.sh @@ -24,7 +24,7 @@ docker-compose --version # sanity check # CLOUDFLARE_AUTH_TOKEN - cloudflare auth token for ssl generation (just for siasky.net) if ! [ -f /home/user/skynet-webportal/.env ]; then HSD_API_KEY=$(openssl rand -base64 32) # generate safe random key for handshake - printf "DOMAIN_NAME=example.com\nEMAIL_ADDRESS=email@example.com\nSIA_API_AUTHORIZATION=\nCLOUDFLARE_AUTH_TOKEN=\nHSD_API_KEY=${HSD_API_KEY}\n" > /home/user/skynet-webportal/.env + printf "DOMAIN_NAME=example.com\nEMAIL_ADDRESS=email@example.com\nSIA_API_AUTHORIZATION=\nCLOUDFLARE_AUTH_TOKEN=\nHSD_API_KEY=${HSD_API_KEY}\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\n" > /home/user/skynet-webportal/.env fi # Start docker container with nginx and client