From 025d8f5d94bbabcae3eb9634be727a9da580f287 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 27 Jul 2020 15:40:26 +0200 Subject: [PATCH 01/13] purge caddy more or less --- docker-compose.yml | 17 ++++--- docker/caddy/Caddyfile | 81 ++------------------------------- docker/caddy/Dockerfile | 15 +----- docker/nginx/Dockerfile | 15 ++++++ docker/nginx/conf.d/client.conf | 54 ++++++++++++++++++++-- docker/nginx/conf.d/gzip.conf | 18 ++++++++ 6 files changed, 99 insertions(+), 101 deletions(-) create mode 100644 docker/nginx/Dockerfile create mode 100644 docker/nginx/conf.d/gzip.conf diff --git a/docker-compose.yml b/docker-compose.yml index 3c364523..5a4a579a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,8 +17,8 @@ services: caddy: build: - context: . - dockerfile: ./docker/caddy/Dockerfile + context: ./docker/caddy + dockerfile: Dockerfile container_name: caddy restart: on-failure env_file: @@ -37,9 +37,13 @@ services: - nginx nginx: - image: openresty/openresty:1.15.8.3-2-xenial + build: + context: . + dockerfile: ./docker/nginx/Dockerfile container_name: nginx restart: on-failure + env_file: + - .env volumes: - ./docker/nginx/conf.d:/etc/nginx/conf.d:ro - ./docker/data/nginx/cache:/data/nginx/cache @@ -57,6 +61,7 @@ services: dockerfile: Dockerfile container_name: handshake restart: on-failure + command: hnd --log-console=false environment: - HSD_HTTP_HOST=0.0.0.0 - HSD_NETWORK=main @@ -90,7 +95,7 @@ services: - 3100 depends_on: - handshake - - caddy + - nginx health-check: build: @@ -104,11 +109,11 @@ services: networks: - shared environment: - - PORTAL_URL=caddy + - PORTAL_URL=nginx - HOSTNAME=health-check - NODE_TLS_REJECT_UNAUTHORIZED=0 expose: - 3100 depends_on: - docker-host - - caddy + - nginx diff --git a/docker/caddy/Caddyfile b/docker/caddy/Caddyfile index 9a0aacd7..6e3e0ae6 100644 --- a/docker/caddy/Caddyfile +++ b/docker/caddy/Caddyfile @@ -1,82 +1,7 @@ -(webserver) { - root * /home/user/public_html - file_server - encode zstd gzip - - @skylink { - path_regexp skylink ^/([a-zA-Z0-9-_]{46}(/.*)?)$ - } - - @skylink_file { - path_regexp skylink_file ^/file/([a-zA-Z0-9-_]{46}(/.*)?)$ - } - - @options { - method OPTIONS - } - - @blacklist { - method GET - path /blacklist - } - - @portals { - method GET - path /portals - } - - # OPTIONS headers to allow CORS https://enable-cors.org - handle @options { - header { - Access-Control-Allow-Origin * - Access-Control-Allow-Methods GET,POST,OPTIONS - Access-Control-Allow-Headers DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range - Access-Control-Max-Age 1728000 - } - respond 204 - } - - reverse_proxy /health-check health-check:3100 - reverse_proxy /hns/* handshake-api:3100 - reverse_proxy /hnsres/* handshake-api:3100 - - reverse_proxy @blacklist nginx:80 { - header_up User-Agent Sia-Agent - header_down Access-Control-Allow-Origin * - } - reverse_proxy @portals nginx:80 { - header_up User-Agent Sia-Agent - header_down Access-Control-Allow-Origin * - } - reverse_proxy /stats nginx:80 { - header_up User-Agent Sia-Agent - header_down Access-Control-Allow-Origin * - } - reverse_proxy /statsdown nginx:80 { - header_up User-Agent Sia-Agent - header_down Access-Control-Allow-Origin * - } - reverse_proxy @skylink nginx:80 { - header_up User-Agent Sia-Agent - header_up Access-Control-Expose-Headers skynet-file-metadata - header_down Access-Control-Allow-Origin * - } - reverse_proxy @skylink_file nginx:80 { - header_up User-Agent Sia-Agent - header_up Access-Control-Expose-Headers skynet-file-metadata - header_down Access-Control-Allow-Origin * - } - reverse_proxy /skynet/skyfile* nginx:80 { - header_up User-Agent Sia-Agent - header_up Authorization "Basic {env.SIA_API_AUTHORIZATION}" - header_down Access-Control-Allow-Origin * - } -} - (custom.domain) { {$DOMAIN_NAME} { tls {$EMAIL_ADDRESS} - import webserver + reverse_proxy nginx:80 } } @@ -85,7 +10,7 @@ tls { dns cloudflare {env.CLOUDFLARE_AUTH_TOKEN} } - import webserver + reverse_proxy nginx:80 } } @@ -94,7 +19,7 @@ tls internal { on_demand } - import webserver + reverse_proxy nginx:80 } } diff --git a/docker/caddy/Dockerfile b/docker/caddy/Dockerfile index c6356699..d0f22045 100644 --- a/docker/caddy/Dockerfile +++ b/docker/caddy/Dockerfile @@ -1,20 +1,7 @@ -FROM node:12.18.0 AS client-builder - -COPY src ./src -COPY static ./static -COPY gatsby-config.js . -COPY package.json . -COPY yarn.lock . - -ENV CYPRESS_INSTALL_BINARY 0 -RUN yarn --frozen-lockfile -RUN yarn build - FROM caddy:2.1.1-builder AS caddy-builder -RUN caddy-builder github.com/caddy-dns/cloudflare +RUN caddy-builder github.com/caddy-dns/cloudflare github.com/caddy-dns/route53 FROM caddy:2.1.1 -COPY --from=client-builder /public /home/user/public_html COPY --from=caddy-builder /usr/bin/caddy /usr/bin/caddy diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile new file mode 100644 index 00000000..ec521767 --- /dev/null +++ b/docker/nginx/Dockerfile @@ -0,0 +1,15 @@ +FROM node:14.5 AS webportal-builder + +COPY src ./src +COPY static ./static +COPY gatsby-config.js . +COPY package.json . +COPY yarn.lock . + +ENV CYPRESS_INSTALL_BINARY 0 +RUN yarn --frozen-lockfile +RUN yarn build + +FROM openresty/openresty:1.15.8.3-2-xenial + +COPY --from=webportal-builder /public /var/www/webportal diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 262db21e..0a922497 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -21,7 +21,7 @@ real_ip_header X-Forwarded-For; # The upload siad can be run in normal mode. Set the port to '9980' if # you do not want to run your portal in the double siad setup. upstream siad-upload { - server docker-host:9970; + server docker-host:9980; } upstream siad-download { @@ -41,39 +41,65 @@ server { client_body_buffer_size 128k; client_max_body_size 128k; - rewrite "^(/([a-zA-Z0-9-_]{46})[^/]{0})$" $1/ permanent; - rewrite "^(/hns/[^/]+)[^/]{0}$" $1/ permanent; + location / { + root /var/www/webportal; + } location /blacklist { proxy_cache skynet; proxy_cache_valid any 1m; # cache blacklist for 1 minute + proxy_set_header Access-Control-Allow-Origin: *; + proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-download/skynet/blacklist; } location /portals { proxy_cache skynet; proxy_cache_valid any 1m; # cache portals for 1 minute + proxy_set_header Access-Control-Allow-Origin: *; + proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-download/skynet/portals; } location /stats { proxy_cache skynet; proxy_cache_valid any 1m; # cache stats for 1 minute + proxy_set_header Access-Control-Allow-Origin: *; + proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-upload/skynet/stats; } location /statsdown { proxy_cache skynet; proxy_cache_valid any 1m; # cache stats for 1 minute + proxy_set_header Access-Control-Allow-Origin: *; + proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-download/skynet/stats; } + location /health-check { + proxy_set_header Access-Control-Allow-Origin: *; + proxy_pass http://health-check:3100; + } + + location /hns { + proxy_set_header Access-Control-Allow-Origin: *; + proxy_pass http://handshake-api:3100; + } + + location /hnsres { + proxy_set_header Access-Control-Allow-Origin: *; + proxy_pass http://handshake-api:3100; + } + location /skynet/skyfile { limit_conn uploads_by_ip 10; # ddos protection: max 10 uploads at a time client_max_body_size 1000M; # make sure to limit the size of upload to a sane value proxy_read_timeout 600; proxy_request_buffering off; # stream uploaded files through the proxy as it comes in proxy_set_header Expect $http_expect; + proxy_set_header Access-Control-Allow-Origin: *; + proxy_set_header User-Agent: Sia-Agent; # Extract 3 sets of 2 characters from $request_id and assign to $dir1, $dir2, $dir3 # respectfully. The rest of the $request_id is going to be assigned to $dir4. @@ -90,6 +116,22 @@ server { # proxy this call to siad endpoint (make sure the ip is correct) proxy_pass http://siad-upload/skynet/skyfile/$dir1/$dir2/$dir3/$dir4$is_args$args; + + # rewrite_by_lua_block { + # local b64 = require("ngx.base64") + # -- open apipassword file for reading (b flag is required for some reason) + # -- (file /etc/.sia/apipassword has to be mounted from the host system) + # local apipassword_file = io.open("/etc/.sia/apipassword", "rb") + # -- read apipassword file contents and trim newline (important) + # local apipassword = apipassword_file:read("*all"):gsub("%s+", "") + # -- make sure to close file after reading the password + # apipassword_file.close() + # -- encode the user:password authorization string + # -- (in our case user is empty so it is just :password) + # local content = b64.encode_base64url(":" .. apipassword) + # -- set authorization header with proper base64 encoded string + # ngx.req.set_header("Authorization", "Basic " .. content) + # } } location ~ "/skynet/skyfile/(.+)" { @@ -98,6 +140,8 @@ server { proxy_read_timeout 600; proxy_request_buffering off; # stream uploaded files through the proxy as it comes in proxy_set_header Expect $http_expect; + proxy_set_header Access-Control-Allow-Origin: *; + proxy_set_header User-Agent: Sia-Agent; # we need to explicitly use set directive here because $1 will contain the siapath with # decoded whitespaces and set will re-encode it for us before passing it to proxy_pass @@ -115,6 +159,8 @@ server { set $skylink $1; proxy_read_timeout 600; + proxy_set_header Access-Control-Allow-Origin: *; + proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) proxy_pass http://siad-download/skynet/skylink/$skylink$is_args$args; @@ -137,6 +183,8 @@ server { set $skylink $1; proxy_read_timeout 600; + proxy_set_header Access-Control-Allow-Origin: *; + proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) # this alias also adds attachment=true url param to force download the file proxy_pass http://siad-download/skynet/skylink/$skylink?attachment=true&$args; diff --git a/docker/nginx/conf.d/gzip.conf b/docker/nginx/conf.d/gzip.conf new file mode 100644 index 00000000..7d7a962c --- /dev/null +++ b/docker/nginx/conf.d/gzip.conf @@ -0,0 +1,18 @@ +# enables gzip compression +gzip on; + +# set the gzip compression level (1-9) +gzip_comp_level 6; + +# tells proxies to cache both gzipped and regular versions of a resource +gzip_vary on; + +# informs NGINX to not compress anything smaller than the defined size +gzip_min_length 256; + +# compress data even for clients that are connecting via proxies if a response header includes +# the "expired", "no-cache", "no-store", "private", and "Authorization" parameters +gzip_proxied expired no-cache no-store private auth; + +# enables the types of files that can be compressed +gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/vnd.ms-fontobject application/x-font-ttf font/opentype image/svg+xml image/x-icon; From af1f4896f0d71987fc6c8b3f2ec45ffdcd81a838 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 27 Jul 2020 15:55:08 +0200 Subject: [PATCH 02/13] route53 --- docker/caddy/Caddyfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/caddy/Caddyfile b/docker/caddy/Caddyfile index 6e3e0ae6..77485c4e 100644 --- a/docker/caddy/Caddyfile +++ b/docker/caddy/Caddyfile @@ -8,7 +8,7 @@ (siasky.net) { siasky.net, *.siasky.net { tls { - dns cloudflare {env.CLOUDFLARE_AUTH_TOKEN} + dns route53 } reverse_proxy nginx:80 } From 1dbc842c5575bf166b24fb4729be9753d88b7c29 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 27 Jul 2020 17:04:21 +0200 Subject: [PATCH 03/13] cors --- docker/nginx/conf.d/client.conf | 24 +++++++++++++++++++++++ docker/nginx/conf.d/include/cors | 27 ++++++++++++++++++++++++++ setup-scripts/setup-docker-services.sh | 2 +- 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 docker/nginx/conf.d/include/cors diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 0a922497..d67b07bf 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -42,10 +42,14 @@ server { client_max_body_size 128k; location / { + include /etc/nginx/conf.d/include/cors; + root /var/www/webportal; } location /blacklist { + include /etc/nginx/conf.d/include/cors; + proxy_cache skynet; proxy_cache_valid any 1m; # cache blacklist for 1 minute proxy_set_header Access-Control-Allow-Origin: *; @@ -54,6 +58,8 @@ server { } location /portals { + include /etc/nginx/conf.d/include/cors; + proxy_cache skynet; proxy_cache_valid any 1m; # cache portals for 1 minute proxy_set_header Access-Control-Allow-Origin: *; @@ -62,6 +68,8 @@ server { } location /stats { + include /etc/nginx/conf.d/include/cors; + proxy_cache skynet; proxy_cache_valid any 1m; # cache stats for 1 minute proxy_set_header Access-Control-Allow-Origin: *; @@ -70,6 +78,8 @@ server { } location /statsdown { + include /etc/nginx/conf.d/include/cors; + proxy_cache skynet; proxy_cache_valid any 1m; # cache stats for 1 minute proxy_set_header Access-Control-Allow-Origin: *; @@ -78,21 +88,29 @@ server { } location /health-check { + include /etc/nginx/conf.d/include/cors; + proxy_set_header Access-Control-Allow-Origin: *; proxy_pass http://health-check:3100; } location /hns { + include /etc/nginx/conf.d/include/cors; + proxy_set_header Access-Control-Allow-Origin: *; proxy_pass http://handshake-api:3100; } location /hnsres { + include /etc/nginx/conf.d/include/cors; + proxy_set_header Access-Control-Allow-Origin: *; proxy_pass http://handshake-api:3100; } location /skynet/skyfile { + include /etc/nginx/conf.d/include/cors; + limit_conn uploads_by_ip 10; # ddos protection: max 10 uploads at a time client_max_body_size 1000M; # make sure to limit the size of upload to a sane value proxy_read_timeout 600; @@ -135,6 +153,8 @@ server { } location ~ "/skynet/skyfile/(.+)" { + include /etc/nginx/conf.d/include/cors; + limit_conn uploads_by_ip 10; # ddos protection: max 10 uploads at a time client_max_body_size 1000M; # make sure to limit the size of upload to a sane value proxy_read_timeout 600; @@ -152,6 +172,8 @@ server { } location ~ "^/([a-zA-Z0-9-_]{46}(/.*)?)$" { + include /etc/nginx/conf.d/include/cors; + limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time # we need to explicitly use set directive here because $1 will contain the skylink with @@ -176,6 +198,8 @@ server { } location ~ "^/file/([a-zA-Z0-9-_]{46}(/.*)?)$" { + include /etc/nginx/conf.d/include/cors; + limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time # we need to explicitly use set directive here because $1 will contain the skylink with diff --git a/docker/nginx/conf.d/include/cors b/docker/nginx/conf.d/include/cors new file mode 100644 index 00000000..33bd2050 --- /dev/null +++ b/docker/nginx/conf.d/include/cors @@ -0,0 +1,27 @@ +if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + # + # Custom headers and headers various browsers *should* be OK with but aren't + # + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + # + # Tell client that this pre-flight info is valid for 20 days + # + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 204; +} +if ($request_method = 'POST') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; +} +if ($request_method = 'GET') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; +} \ No newline at end of file diff --git a/setup-scripts/setup-docker-services.sh b/setup-scripts/setup-docker-services.sh index 26cd6c95..92525f07 100755 --- a/setup-scripts/setup-docker-services.sh +++ b/setup-scripts/setup-docker-services.sh @@ -24,7 +24,7 @@ docker-compose --version # sanity check # CLOUDFLARE_AUTH_TOKEN - cloudflare auth token for ssl generation (just for siasky.net) if ! [ -f /home/user/skynet-webportal/.env ]; then HSD_API_KEY=$(openssl rand -base64 32) # generate safe random key for handshake - printf "DOMAIN_NAME=example.com\nEMAIL_ADDRESS=email@example.com\nSIA_API_AUTHORIZATION=\nCLOUDFLARE_AUTH_TOKEN=\nHSD_API_KEY=${HSD_API_KEY}\n" > /home/user/skynet-webportal/.env + printf "DOMAIN_NAME=example.com\nEMAIL_ADDRESS=email@example.com\nSIA_API_AUTHORIZATION=\nCLOUDFLARE_AUTH_TOKEN=\nHSD_API_KEY=${HSD_API_KEY}\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\n" > /home/user/skynet-webportal/.env fi # Start docker container with nginx and client From 7090edc5e74a84aab386d8eab97b185c829fd729 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 27 Jul 2020 17:16:17 +0200 Subject: [PATCH 04/13] cors --- docker/nginx/conf.d/cors.conf | 4 ++++ docker/nginx/conf.d/include/cors | 12 ------------ 2 files changed, 4 insertions(+), 12 deletions(-) create mode 100644 docker/nginx/conf.d/cors.conf diff --git a/docker/nginx/conf.d/cors.conf b/docker/nginx/conf.d/cors.conf new file mode 100644 index 00000000..7fbce3ba --- /dev/null +++ b/docker/nginx/conf.d/cors.conf @@ -0,0 +1,4 @@ +more_set_headers 'Access-Control-Allow-Origin: *'; +more_set_headers 'Access-Control-Allow-Methods: GET, POST, OPTIONS, PUT, DELETE'; +more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; +more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range'; \ No newline at end of file diff --git a/docker/nginx/conf.d/include/cors b/docker/nginx/conf.d/include/cors index 33bd2050..fef6d135 100644 --- a/docker/nginx/conf.d/include/cors +++ b/docker/nginx/conf.d/include/cors @@ -12,16 +12,4 @@ if ($request_method = 'OPTIONS') { add_header 'Content-Type' 'text/plain; charset=utf-8'; add_header 'Content-Length' 0; return 204; -} -if ($request_method = 'POST') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; -} -if ($request_method = 'GET') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; } \ No newline at end of file From 18ae4360870a6767d077aa846b4bcb6e4f97745a Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 27 Jul 2020 17:18:09 +0200 Subject: [PATCH 05/13] new line --- docker/nginx/conf.d/cors.conf | 2 +- docker/nginx/conf.d/include/cors | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/nginx/conf.d/cors.conf b/docker/nginx/conf.d/cors.conf index 7fbce3ba..6c8cff35 100644 --- a/docker/nginx/conf.d/cors.conf +++ b/docker/nginx/conf.d/cors.conf @@ -1,4 +1,4 @@ more_set_headers 'Access-Control-Allow-Origin: *'; more_set_headers 'Access-Control-Allow-Methods: GET, POST, OPTIONS, PUT, DELETE'; more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; -more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range'; \ No newline at end of file +more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range'; diff --git a/docker/nginx/conf.d/include/cors b/docker/nginx/conf.d/include/cors index fef6d135..1ebe6617 100644 --- a/docker/nginx/conf.d/include/cors +++ b/docker/nginx/conf.d/include/cors @@ -12,4 +12,4 @@ if ($request_method = 'OPTIONS') { add_header 'Content-Type' 'text/plain; charset=utf-8'; add_header 'Content-Length' 0; return 204; -} \ No newline at end of file +} From 3b75226e0b582d22ab315268b8389c1939626b24 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 27 Jul 2020 17:30:05 +0200 Subject: [PATCH 06/13] apipassword --- docker/nginx/conf.d/client.conf | 36 +++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index d67b07bf..f534b08d 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -135,21 +135,16 @@ server { # proxy this call to siad endpoint (make sure the ip is correct) proxy_pass http://siad-upload/skynet/skyfile/$dir1/$dir2/$dir3/$dir4$is_args$args; - # rewrite_by_lua_block { - # local b64 = require("ngx.base64") - # -- open apipassword file for reading (b flag is required for some reason) - # -- (file /etc/.sia/apipassword has to be mounted from the host system) - # local apipassword_file = io.open("/etc/.sia/apipassword", "rb") - # -- read apipassword file contents and trim newline (important) - # local apipassword = apipassword_file:read("*all"):gsub("%s+", "") - # -- make sure to close file after reading the password - # apipassword_file.close() - # -- encode the user:password authorization string - # -- (in our case user is empty so it is just :password) - # local content = b64.encode_base64url(":" .. apipassword) - # -- set authorization header with proper base64 encoded string - # ngx.req.set_header("Authorization", "Basic " .. content) - # } + rewrite_by_lua_block { + local b64 = require("ngx.base64") + -- put your apipassword here + local apipassword = "" + -- encode the user:password authorization string + -- (in our case user is empty so it is just :password) + local content = b64.encode_base64url(":" .. apipassword) + -- set authorization header with proper base64 encoded string + ngx.req.set_header("Authorization", "Basic " .. content) + } } location ~ "/skynet/skyfile/(.+)" { @@ -169,6 +164,17 @@ server { # proxy this call to siad endpoint (make sure the ip is correct) proxy_pass http://siad-upload/skynet/skyfile/$siapath$is_args$args; + + rewrite_by_lua_block { + local b64 = require("ngx.base64") + -- put your apipassword here + local apipassword = "" + -- encode the user:password authorization string + -- (in our case user is empty so it is just :password) + local content = b64.encode_base64url(":" .. apipassword) + -- set authorization header with proper base64 encoded string + ngx.req.set_header("Authorization", "Basic " .. content) + } } location ~ "^/([a-zA-Z0-9-_]{46}(/.*)?)$" { From 2a64b33cc9f1096033d9badf978632bf06891d4c Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 27 Jul 2020 17:34:30 +0200 Subject: [PATCH 07/13] ?skynet stats --- docker/nginx/conf.d/client.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index f534b08d..e473c2e3 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -67,7 +67,7 @@ server { proxy_pass http://siad-download/skynet/portals; } - location /stats { + location ~ "(/skynet)?/stats" { include /etc/nginx/conf.d/include/cors; proxy_cache skynet; From dd53f31171cd2930edcc7526c5770bbfaca8e3ec Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 27 Jul 2020 17:37:17 +0200 Subject: [PATCH 08/13] skynet endpoints --- docker/nginx/conf.d/client.conf | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index e473c2e3..66d9d7ad 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -57,6 +57,16 @@ server { proxy_pass http://siad-download/skynet/blacklist; } + location /skynet/blacklist { + include /etc/nginx/conf.d/include/cors; + + proxy_cache skynet; + proxy_cache_valid any 1m; # cache blacklist for 1 minute + proxy_set_header Access-Control-Allow-Origin: *; + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://siad-download/skynet/blacklist; + } + location /portals { include /etc/nginx/conf.d/include/cors; @@ -67,7 +77,27 @@ server { proxy_pass http://siad-download/skynet/portals; } - location ~ "(/skynet)?/stats" { + location /skynet/portals { + include /etc/nginx/conf.d/include/cors; + + proxy_cache skynet; + proxy_cache_valid any 1m; # cache portals for 1 minute + proxy_set_header Access-Control-Allow-Origin: *; + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://siad-download/skynet/portals; + } + + location /stats { + include /etc/nginx/conf.d/include/cors; + + proxy_cache skynet; + proxy_cache_valid any 1m; # cache stats for 1 minute + proxy_set_header Access-Control-Allow-Origin: *; + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://siad-upload/skynet/stats; + } + + location /skynet/stats { include /etc/nginx/conf.d/include/cors; proxy_cache skynet; From 5890b5e8ef237eddb94dea3ed25b22719051499a Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 27 Jul 2020 17:48:45 +0200 Subject: [PATCH 09/13] fix cors --- docker/nginx/conf.d/client.conf | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 66d9d7ad..52f4ed36 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -52,7 +52,6 @@ server { proxy_cache skynet; proxy_cache_valid any 1m; # cache blacklist for 1 minute - proxy_set_header Access-Control-Allow-Origin: *; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-download/skynet/blacklist; } @@ -62,7 +61,6 @@ server { proxy_cache skynet; proxy_cache_valid any 1m; # cache blacklist for 1 minute - proxy_set_header Access-Control-Allow-Origin: *; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-download/skynet/blacklist; } @@ -72,7 +70,6 @@ server { proxy_cache skynet; proxy_cache_valid any 1m; # cache portals for 1 minute - proxy_set_header Access-Control-Allow-Origin: *; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-download/skynet/portals; } @@ -82,7 +79,6 @@ server { proxy_cache skynet; proxy_cache_valid any 1m; # cache portals for 1 minute - proxy_set_header Access-Control-Allow-Origin: *; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-download/skynet/portals; } @@ -92,7 +88,6 @@ server { proxy_cache skynet; proxy_cache_valid any 1m; # cache stats for 1 minute - proxy_set_header Access-Control-Allow-Origin: *; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-upload/skynet/stats; } @@ -102,7 +97,6 @@ server { proxy_cache skynet; proxy_cache_valid any 1m; # cache stats for 1 minute - proxy_set_header Access-Control-Allow-Origin: *; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-upload/skynet/stats; } @@ -112,7 +106,6 @@ server { proxy_cache skynet; proxy_cache_valid any 1m; # cache stats for 1 minute - proxy_set_header Access-Control-Allow-Origin: *; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad-download/skynet/stats; } @@ -120,21 +113,18 @@ server { location /health-check { include /etc/nginx/conf.d/include/cors; - proxy_set_header Access-Control-Allow-Origin: *; proxy_pass http://health-check:3100; } location /hns { include /etc/nginx/conf.d/include/cors; - proxy_set_header Access-Control-Allow-Origin: *; proxy_pass http://handshake-api:3100; } location /hnsres { include /etc/nginx/conf.d/include/cors; - proxy_set_header Access-Control-Allow-Origin: *; proxy_pass http://handshake-api:3100; } @@ -146,7 +136,6 @@ server { proxy_read_timeout 600; proxy_request_buffering off; # stream uploaded files through the proxy as it comes in proxy_set_header Expect $http_expect; - proxy_set_header Access-Control-Allow-Origin: *; proxy_set_header User-Agent: Sia-Agent; # Extract 3 sets of 2 characters from $request_id and assign to $dir1, $dir2, $dir3 @@ -185,7 +174,6 @@ server { proxy_read_timeout 600; proxy_request_buffering off; # stream uploaded files through the proxy as it comes in proxy_set_header Expect $http_expect; - proxy_set_header Access-Control-Allow-Origin: *; proxy_set_header User-Agent: Sia-Agent; # we need to explicitly use set directive here because $1 will contain the siapath with @@ -217,7 +205,6 @@ server { set $skylink $1; proxy_read_timeout 600; - proxy_set_header Access-Control-Allow-Origin: *; proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) proxy_pass http://siad-download/skynet/skylink/$skylink$is_args$args; @@ -243,7 +230,6 @@ server { set $skylink $1; proxy_read_timeout 600; - proxy_set_header Access-Control-Allow-Origin: *; proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) # this alias also adds attachment=true url param to force download the file From 6a6feade0ca2bbc32414b8690f1c0b9b913fe881 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 27 Jul 2020 18:58:53 +0200 Subject: [PATCH 10/13] fix healthcheck --- docker-compose.yml | 1 - health-check/checks.js | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5a4a579a..1ce9002a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -111,7 +111,6 @@ services: environment: - PORTAL_URL=nginx - HOSTNAME=health-check - - NODE_TLS_REJECT_UNAUTHORIZED=0 expose: - 3100 depends_on: diff --git a/health-check/checks.js b/health-check/checks.js index 51d78506..0ab96f81 100644 --- a/health-check/checks.js +++ b/health-check/checks.js @@ -5,7 +5,7 @@ async function uploadCheck(done) { const time = process.hrtime(); superagent - .post(`https://${process.env.PORTAL_URL}/skynet/skyfile`) + .post(`${process.env.PORTAL_URL}/skynet/skyfile`) .attach("file", "package.json", "package.json") .end((err, res) => { const statusCode = (res && res.statusCode) || (err && err.statusCode) || null; @@ -22,7 +22,7 @@ async function uploadCheck(done) { function downloadCheck(done) { const time = process.hrtime(); - superagent.get(`https://${process.env.PORTAL_URL}`).end((err, res) => { + superagent.get(`${process.env.PORTAL_URL}`).end((err, res) => { const statusCode = (res && res.statusCode) || (err && err.statusCode) || null; done({ From 60fcf4b8b82741f17c2c49d0a9d97410ce50dae3 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 27 Jul 2020 18:59:32 +0200 Subject: [PATCH 11/13] fix healthcheck --- health-check/checks.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/health-check/checks.js b/health-check/checks.js index 0ab96f81..8e29b3dc 100644 --- a/health-check/checks.js +++ b/health-check/checks.js @@ -5,7 +5,7 @@ async function uploadCheck(done) { const time = process.hrtime(); superagent - .post(`${process.env.PORTAL_URL}/skynet/skyfile`) + .post(`http://${process.env.PORTAL_URL}/skynet/skyfile`) .attach("file", "package.json", "package.json") .end((err, res) => { const statusCode = (res && res.statusCode) || (err && err.statusCode) || null; @@ -22,7 +22,7 @@ async function uploadCheck(done) { function downloadCheck(done) { const time = process.hrtime(); - superagent.get(`${process.env.PORTAL_URL}`).end((err, res) => { + superagent.get(`http://${process.env.PORTAL_URL}`).end((err, res) => { const statusCode = (res && res.statusCode) || (err && err.statusCode) || null; done({ From 8f245d2b0e1388a4201ad56b9a5cf825470286f2 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 27 Jul 2020 19:03:05 +0200 Subject: [PATCH 12/13] fix healthcheck --- docker-compose.yml | 1 - health-check/index.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1ce9002a..a5c60631 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -110,7 +110,6 @@ services: - shared environment: - PORTAL_URL=nginx - - HOSTNAME=health-check expose: - 3100 depends_on: diff --git a/health-check/index.js b/health-check/index.js index 18d2ad79..c34d3770 100644 --- a/health-check/index.js +++ b/health-check/index.js @@ -9,7 +9,7 @@ const bodyparser = require("body-parser"); require("./schedule"); -const host = process.env.HOSTNAME || "localhost"; +const host = process.env.HOSTNAME || "0.0.0.0"; const port = process.env.PORT || 3100; const server = express(); From 919edb439c8ad4652fa6248633b0290536c22409 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 28 Jul 2020 11:19:03 +0200 Subject: [PATCH 13/13] auth uploads --- docker-compose.yml | 1 + docker/nginx/conf.d/client.conf | 24 +---------- docker/nginx/conf.d/include/cors | 8 +--- docker/nginx/conf.d/include/sia-auth | 10 +++++ docker/nginx/nginx.conf | 64 ++++++++++++++++++++++++++++ 5 files changed, 78 insertions(+), 29 deletions(-) create mode 100644 docker/nginx/conf.d/include/sia-auth create mode 100644 docker/nginx/nginx.conf diff --git a/docker-compose.yml b/docker-compose.yml index a5c60631..124b7c6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,6 +45,7 @@ services: env_file: - .env volumes: + - ./docker/nginx/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf:ro - ./docker/nginx/conf.d:/etc/nginx/conf.d:ro - ./docker/data/nginx/cache:/data/nginx/cache - ./docker/data/nginx/logs:/usr/local/openresty/nginx/logs diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 52f4ed36..9cead372 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -130,6 +130,7 @@ server { location /skynet/skyfile { include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/sia-auth; limit_conn uploads_by_ip 10; # ddos protection: max 10 uploads at a time client_max_body_size 1000M; # make sure to limit the size of upload to a sane value @@ -153,21 +154,11 @@ server { # proxy this call to siad endpoint (make sure the ip is correct) proxy_pass http://siad-upload/skynet/skyfile/$dir1/$dir2/$dir3/$dir4$is_args$args; - - rewrite_by_lua_block { - local b64 = require("ngx.base64") - -- put your apipassword here - local apipassword = "" - -- encode the user:password authorization string - -- (in our case user is empty so it is just :password) - local content = b64.encode_base64url(":" .. apipassword) - -- set authorization header with proper base64 encoded string - ngx.req.set_header("Authorization", "Basic " .. content) - } } location ~ "/skynet/skyfile/(.+)" { include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/sia-auth; limit_conn uploads_by_ip 10; # ddos protection: max 10 uploads at a time client_max_body_size 1000M; # make sure to limit the size of upload to a sane value @@ -182,17 +173,6 @@ server { # proxy this call to siad endpoint (make sure the ip is correct) proxy_pass http://siad-upload/skynet/skyfile/$siapath$is_args$args; - - rewrite_by_lua_block { - local b64 = require("ngx.base64") - -- put your apipassword here - local apipassword = "" - -- encode the user:password authorization string - -- (in our case user is empty so it is just :password) - local content = b64.encode_base64url(":" .. apipassword) - -- set authorization header with proper base64 encoded string - ngx.req.set_header("Authorization", "Basic " .. content) - } } location ~ "^/([a-zA-Z0-9-_]{46}(/.*)?)$" { diff --git a/docker/nginx/conf.d/include/cors b/docker/nginx/conf.d/include/cors index 1ebe6617..1490624b 100644 --- a/docker/nginx/conf.d/include/cors +++ b/docker/nginx/conf.d/include/cors @@ -1,14 +1,8 @@ if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - # - # Custom headers and headers various browsers *should* be OK with but aren't - # add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - # - # Tell client that this pre-flight info is valid for 20 days - # - add_header 'Access-Control-Max-Age' 1728000; + add_header 'Access-Control-Max-Age' 1728000; # valid for 20 days add_header 'Content-Type' 'text/plain; charset=utf-8'; add_header 'Content-Length' 0; return 204; diff --git a/docker/nginx/conf.d/include/sia-auth b/docker/nginx/conf.d/include/sia-auth new file mode 100644 index 00000000..8983bb1f --- /dev/null +++ b/docker/nginx/conf.d/include/sia-auth @@ -0,0 +1,10 @@ +rewrite_by_lua_block { + -- local b64 = require("ngx.base64") + -- pull apipassword from SIA_API_AUTHORIZATION environment variable + -- local apipassword = os.getenv("SIA_API_AUTHORIZATION") + -- encode the user:password authorization string + -- (in our case user is empty so it is just :password) + -- local content = b64.encode_base64url(":" .. apipassword) + -- set authorization header with proper base64 encoded string + ngx.req.set_header("Authorization", "Basic " .. os.getenv("SIA_API_AUTHORIZATION")) +} diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf new file mode 100644 index 00000000..1fb6a5f0 --- /dev/null +++ b/docker/nginx/nginx.conf @@ -0,0 +1,64 @@ +# nginx.conf -- docker-openresty +# +# This file is installed to: +# `/usr/local/openresty/nginx/conf/nginx.conf` +# and is the file loaded by nginx at startup, +# unless the user specifies otherwise. +# +# It tracks the upstream OpenResty's `nginx.conf`, but removes the `server` +# section and adds this directive: +# `include /etc/nginx/conf.d/*.conf;` +# +# The `docker-openresty` file `nginx.vh.default.conf` is copied to +# `/etc/nginx/conf.d/default.conf`. It contains the `server section +# of the upstream `nginx.conf`. +# +# See https://github.com/openresty/docker-openresty/blob/master/README.md#nginx-config-files +# + +# expose environment variables +env SIA_API_AUTHORIZATION; + +#user nobody; +worker_processes 1; + +#error_log logs/error.log; +#error_log logs/error.log notice; +#error_log logs/error.log info; + +#pid logs/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include mime.types; + default_type application/octet-stream; + + #log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + # '$status $body_bytes_sent "$http_referer" ' + # '"$http_user_agent" "$http_x_forwarded_for"'; + + #access_log logs/access.log main; + + # See Move default writable paths to a dedicated directory (#119) + # https://github.com/openresty/docker-openresty/issues/119 + client_body_temp_path /var/run/openresty/nginx-client-body; + proxy_temp_path /var/run/openresty/nginx-proxy; + fastcgi_temp_path /var/run/openresty/nginx-fastcgi; + uwsgi_temp_path /var/run/openresty/nginx-uwsgi; + scgi_temp_path /var/run/openresty/nginx-scgi; + + sendfile on; + #tcp_nopush on; + + #keepalive_timeout 0; + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +}