From 6053cb3eb27e6d95a897e4b58c528a098944c884 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 13 Jan 2021 15:13:34 +0100 Subject: [PATCH 1/6] add new nginx metrics to measure performance --- docker/nginx/nginx.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index a266943d..1c225e72 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -40,7 +40,10 @@ http { '"$http_user_agent" $upstream_response_time ' '$upstream_bytes_sent $upstream_bytes_received ' '"$upstream_http_content_type" "$upstream_cache_status" ' - '"$portal_domain" "$upstream_http_skynet_skylink"'; + '"$portal_domain" "$upstream_http_skynet_skylink" ' + 'request_time=$request_time ' + 'upstream_connect_time=$upstream_connect_time ' + 'upstream_header_time=$upstream_header_time'; access_log logs/access.log main; From 79235060524f00dde123ea83820b58124f331455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Wypch=C5=82o?= Date: Wed, 13 Jan 2021 17:49:24 +0100 Subject: [PATCH 2/6] add hns_domain and remove named variables --- docker/nginx/nginx.conf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index 1c225e72..4cff1f70 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -41,9 +41,8 @@ http { '$upstream_bytes_sent $upstream_bytes_received ' '"$upstream_http_content_type" "$upstream_cache_status" ' '"$portal_domain" "$upstream_http_skynet_skylink" ' - 'request_time=$request_time ' - 'upstream_connect_time=$upstream_connect_time ' - 'upstream_header_time=$upstream_header_time'; + '$upstream_connect_time $upstream_header_time ' + '$request_time "$hns_domain"'; access_log logs/access.log main; From be0545197e1876c2a8789a7ea89919f65b3fc89f Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Thu, 14 Jan 2021 14:07:33 +0100 Subject: [PATCH 3/6] add new siasky.net servers --- scripts/blocklist-skylink.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh index 1f7dabfb..17e0ae66 100755 --- a/scripts/blocklist-skylink.sh +++ b/scripts/blocklist-skylink.sh @@ -32,7 +32,7 @@ fi ######################################################################### # iterate through all servers, block the skylinks and purge it from cache ######################################################################### -for server in "germany.siasky.net" "helsinki.siasky.net" "us-west.siasky.net" "us-va-1.siasky.net" "us-pa-1.siasky.net" "us-pa-2.siasky.net" "siasky.xyz"; +for server in "eu-ger-1.siasky.net" "eu-ger-2.siasky.net" "eu-fin-1.siasky.net" "us-or-1.siasky.net" "us-or-2.siasky.net" "us-va-1.siasky.net" "us-pa-1.siasky.net" "us-pa-2.siasky.net" "siasky.xyz"; do for skylink in "${skylinks[@]}"; do From fb1b41a52ce0e713196070e8820745869d25840b Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 15 Jan 2021 12:34:46 +0100 Subject: [PATCH 4/6] expose skynet skylink header --- docker/nginx/conf.d/include/cors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/include/cors b/docker/nginx/conf.d/include/cors index 818ac28d..add176d2 100644 --- a/docker/nginx/conf.d/include/cors +++ b/docker/nginx/conf.d/include/cors @@ -11,4 +11,4 @@ if ($request_method = 'OPTIONS') { more_set_headers 'Access-Control-Allow-Origin: *'; more_set_headers 'Access-Control-Allow-Methods: GET, POST, OPTIONS, PUT, DELETE'; more_set_headers 'Access-Control-Allow-Headers: DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; -more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range,Skynet-File-Metadata'; +more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range,Skynet-File-Metadata,Skynet-Skylink'; From a7cc85acd459c4f12d73cd55473b90a0f3a6aa7d Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 15 Jan 2021 16:11:06 +0100 Subject: [PATCH 5/6] improve nginx cache --- docker/nginx/conf.d/client.conf | 2 +- docker/nginx/conf.d/include/proxy-cache-downloads | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 4d76586a..a10220a1 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -1,4 +1,4 @@ -proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=10g use_temp_path=off; +proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=10g inactive=24h use_temp_path=off; # ratelimit specified IPs geo $limit { diff --git a/docker/nginx/conf.d/include/proxy-cache-downloads b/docker/nginx/conf.d/include/proxy-cache-downloads index f3f82a5f..b1eb83f3 100644 --- a/docker/nginx/conf.d/include/proxy-cache-downloads +++ b/docker/nginx/conf.d/include/proxy-cache-downloads @@ -5,5 +5,6 @@ proxy_set_header Range $slice_range; # pass slice range to proxy proxy_cache_key $uri$arg_format$arg_attachment$slice_range; # use just the uri path, format and attachment args and slice range proxy_cache_min_uses 3; # cache responses after 3 requests of the same file proxy_cache_valid 200 206 24h; # cache 200 and 206 responses for 24 hours +proxy_cache_lock on; # queue cache requests for the same resource until it is fully cached proxy_cache_bypass $cookie_nocache $arg_nocache; # add cache bypass option add_header X-Proxy-Cache $upstream_cache_status; # add response header to indicate cache hits and misses From c4a6e71260ada2e07e85547bc6a005e962bb2475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Wypch=C5=82o?= Date: Mon, 18 Jan 2021 12:33:17 +0100 Subject: [PATCH 6/6] reduce directory tree nesting size (#583) --- docker/nginx/conf.d/client.conf | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index a10220a1..db69542f 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -275,21 +275,20 @@ server { proxy_set_header Expect $http_expect; proxy_set_header User-Agent: Sia-Agent; - # Extract 3 sets of 2 characters from $request_id and assign to $dir1, $dir2, $dir3 - # respectfully. The rest of the $request_id is going to be assigned to $dir4. + # Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2 + # respectfully. The rest of the $request_id is going to be assigned to $dir3. # We use those variables to automatically generate a unique path for the uploaded file. # This ensures that not all uploaded files end up in the same directory, which is something # that causes performance issues in the renter. - # Example path result: /af/24/9b/c5ec894920ccc45634dc9a8065 - if ($request_id ~* "(\w{2})(\w{2})(\w{2})(\w+)") { + # Example path result: /af/24/9bc5ec894920ccc45634dc9a8065 + if ($request_id ~* "(\w{2})(\w{2})(\w+)") { set $dir1 $1; set $dir2 $2; set $dir3 $3; - set $dir4 $4; } # proxy this call to siad endpoint (make sure the ip is correct) - proxy_pass http://siad/skynet/skyfile/$dir1/$dir2/$dir3/$dir4$is_args$args; + proxy_pass http://siad/skynet/skyfile/$dir1/$dir2/$dir3$is_args$args; } location ~ "^/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" {