From ece202556199b76bf0684518986f64f5c352ced6 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 2 Jun 2021 17:56:35 +0200 Subject: [PATCH 1/6] /skynet/pin endpoint --- docker/nginx/conf.d/client.conf | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 632a0c13..6d79c497 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -402,6 +402,25 @@ server { proxy_redirect https://siad/ https://$domain.$tld/; } + location /skynet/pin { + include /etc/nginx/conf.d/include/cors; + + # Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2 + # respectfully. The rest of the $request_id is going to be assigned to $dir3. + # We use those variables to automatically generate a unique path for the uploaded file. + # This ensures that not all uploaded files end up in the same directory, which is something + # that causes performance issues in the renter. + # Example path result: /af/24/9bc5ec894920ccc45634dc9a8065 + if ($request_id ~* "(\w{2})(\w{2})(\w+)") { + set $dir1 $1; + set $dir2 $2; + set $dir3 $3; + } + + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://siad$uri?siapath=$dir/$dir2/$dir3&$args; + } + location /skynet/metadata { include /etc/nginx/conf.d/include/cors; From 2b9d469f1e7f6eb55e43ece2e6c03471b56f7e53 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 2 Jun 2021 18:03:35 +0200 Subject: [PATCH 2/6] fix dir variable name --- docker/nginx/conf.d/client.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 6d79c497..2ae32288 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -418,7 +418,7 @@ server { } proxy_set_header User-Agent: Sia-Agent; - proxy_pass http://siad$uri?siapath=$dir/$dir2/$dir3&$args; + proxy_pass http://siad$uri?siapath=$dir1/$dir2/$dir3&$args; } location /skynet/metadata { From c84723b1592cc1e6cf7aa4eba25db8151c1c8922 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 2 Jun 2021 18:15:04 +0200 Subject: [PATCH 3/6] add sia-auth --- docker/nginx/conf.d/client.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 2ae32288..17c67bd3 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -404,6 +404,7 @@ server { location /skynet/pin { include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/sia-auth; # Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2 # respectfully. The rest of the $request_id is going to be assigned to $dir3. From ed7c1f71c7b4f839210708a23fadf3bc76102ecb Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 4 Jun 2021 12:33:37 +0200 Subject: [PATCH 4/6] add /skynet/pin endpoint and clean up config --- docker/nginx/conf.d/client.conf | 68 ++------------------ docker/nginx/conf.d/include/generate-siapath | 11 ++++ docker/nginx/conf.d/include/track-download | 19 ++++++ docker/nginx/conf.d/include/track-upload | 18 ++++++ 4 files changed, 53 insertions(+), 63 deletions(-) create mode 100644 docker/nginx/conf.d/include/generate-siapath create mode 100644 docker/nginx/conf.d/include/track-download create mode 100644 docker/nginx/conf.d/include/track-upload diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 17c67bd3..3798e43c 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -325,6 +325,8 @@ server { location /skynet/skyfile { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; + include /etc/nginx/conf.d/include/track-upload; + include /etc/nginx/conf.d/include/generate-siapath; limit_req zone=uploads_by_ip burst=100 nodelay; limit_req zone=uploads_by_ip_throttled; @@ -338,18 +340,6 @@ server { proxy_set_header Expect $http_expect; proxy_set_header User-Agent: Sia-Agent; - # Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2 - # respectfully. The rest of the $request_id is going to be assigned to $dir3. - # We use those variables to automatically generate a unique path for the uploaded file. - # This ensures that not all uploaded files end up in the same directory, which is something - # that causes performance issues in the renter. - # Example path result: /af/24/9bc5ec894920ccc45634dc9a8065 - if ($request_id ~* "(\w{2})(\w{2})(\w+)") { - set $dir1 $1; - set $dir2 $2; - set $dir3 $3; - } - # access_by_lua_block { # -- this block runs only when accounts are enabled # if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end @@ -365,25 +355,6 @@ server { # proxy this call to siad endpoint (make sure the ip is correct) proxy_pass http://siad/skynet/skyfile/$dir1/$dir2/$dir3$is_args$args; - - # register the upload in accounts service (cookies should contain jwt) - log_by_lua_block { - -- this block runs only when accounts are enabled - if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end - - local skylink = ngx.header["Skynet-Skylink"] - if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then - local http = require("socket.http") - local ok, statusCode, headers, statusText = http.request { - url = "http://accounts:3000/track/upload/" .. skylink, - method = "POST", - headers = ngx.req.get_headers() - } - if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then - ngx.log(ngx.ERR, "accounts endpoint /track/upload/" .. skylink .. " failed with error " .. statusCode) - end - end - } } # endpoing implementing resumable file uploads open protocol https://tus.io @@ -405,18 +376,8 @@ server { location /skynet/pin { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; - - # Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2 - # respectfully. The rest of the $request_id is going to be assigned to $dir3. - # We use those variables to automatically generate a unique path for the uploaded file. - # This ensures that not all uploaded files end up in the same directory, which is something - # that causes performance issues in the renter. - # Example path result: /af/24/9bc5ec894920ccc45634dc9a8065 - if ($request_id ~* "(\w{2})(\w{2})(\w+)") { - set $dir1 $1; - set $dir2 $2; - set $dir3 $3; - } + include /etc/nginx/conf.d/include/track-upload; + include /etc/nginx/conf.d/include/generate-siapath; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad$uri?siapath=$dir1/$dir2/$dir3&$args; @@ -433,6 +394,7 @@ server { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/proxy-buffer; include /etc/nginx/conf.d/include/proxy-cache-downloads; + include /etc/nginx/conf.d/include/track-download; # redirect purge calls to separate location error_page 462 = @purge; @@ -467,26 +429,6 @@ server { end } - # register the download in accounts service (cookies should contain jwt) - log_by_lua_block { - -- this block runs only when accounts are enabled - if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end - - local skylink = ngx.header["Skynet-Skylink"] - if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then - local http = require("socket.http") - local query = table.concat({ "status=" .. ngx.status, "bytes=" .. ngx.var.body_bytes_sent }, "&") - local ok, statusCode, headers, statusText = http.request { - url = "http://accounts:3000/track/download/" .. skylink .. "?" .. query, - method = "POST", - headers = ngx.req.get_headers() - } - if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then - ngx.log(ngx.ERR, "accounts endpoint /track/download/" .. skylink .. " failed with error " .. statusCode) - end - end - } - proxy_read_timeout 600; proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) diff --git a/docker/nginx/conf.d/include/generate-siapath b/docker/nginx/conf.d/include/generate-siapath new file mode 100644 index 00000000..a25c9c15 --- /dev/null +++ b/docker/nginx/conf.d/include/generate-siapath @@ -0,0 +1,11 @@ +# Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2 +# respectfully. The rest of the $request_id is going to be assigned to $dir3. +# We use those variables to automatically generate a unique path for the uploaded file. +# This ensures that not all uploaded files end up in the same directory, which is something +# that causes performance issues in the renter. +# Example path result: /af/24/9bc5ec894920ccc45634dc9a8065 +if ($request_id ~* "(\w{2})(\w{2})(\w+)") { + set $dir1 $1; + set $dir2 $2; + set $dir3 $3; +} \ No newline at end of file diff --git a/docker/nginx/conf.d/include/track-download b/docker/nginx/conf.d/include/track-download new file mode 100644 index 00000000..c767eb8a --- /dev/null +++ b/docker/nginx/conf.d/include/track-download @@ -0,0 +1,19 @@ +# register the download in accounts service (cookies should contain jwt) +log_by_lua_block { + -- this block runs only when accounts are enabled + if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end + + local skylink = ngx.header["Skynet-Skylink"] + if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then + local http = require("socket.http") + local query = table.concat({ "status=" .. ngx.status, "bytes=" .. ngx.var.body_bytes_sent }, "&") + local ok, statusCode, headers, statusText = http.request { + url = "http://accounts:3000/track/download/" .. skylink .. "?" .. query, + method = "POST", + headers = ngx.req.get_headers() + } + if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then + ngx.log(ngx.ERR, "accounts endpoint /track/download/" .. skylink .. " failed with error " .. statusCode) + end + end +} \ No newline at end of file diff --git a/docker/nginx/conf.d/include/track-upload b/docker/nginx/conf.d/include/track-upload new file mode 100644 index 00000000..c6a8ef4d --- /dev/null +++ b/docker/nginx/conf.d/include/track-upload @@ -0,0 +1,18 @@ +# register the upload in accounts service (cookies should contain jwt) +log_by_lua_block { + -- this block runs only when accounts are enabled + if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end + + local skylink = ngx.header["Skynet-Skylink"] + if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then + local http = require("socket.http") + local ok, statusCode, headers, statusText = http.request { + url = "http://accounts:3000/track/upload/" .. skylink, + method = "POST", + headers = ngx.req.get_headers() + } + if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then + ngx.log(ngx.ERR, "accounts endpoint /track/upload/" .. skylink .. " failed with error " .. statusCode) + end + end +} \ No newline at end of file From aa34c1f0887e07fe53cab23acb171ced9cda2482 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 4 Jun 2021 12:35:16 +0200 Subject: [PATCH 5/6] add new lines on eof --- docker/nginx/conf.d/include/generate-siapath | 2 +- docker/nginx/conf.d/include/track-download | 2 +- docker/nginx/conf.d/include/track-upload | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/nginx/conf.d/include/generate-siapath b/docker/nginx/conf.d/include/generate-siapath index a25c9c15..51ef8d84 100644 --- a/docker/nginx/conf.d/include/generate-siapath +++ b/docker/nginx/conf.d/include/generate-siapath @@ -8,4 +8,4 @@ if ($request_id ~* "(\w{2})(\w{2})(\w+)") { set $dir1 $1; set $dir2 $2; set $dir3 $3; -} \ No newline at end of file +} diff --git a/docker/nginx/conf.d/include/track-download b/docker/nginx/conf.d/include/track-download index c767eb8a..376568c9 100644 --- a/docker/nginx/conf.d/include/track-download +++ b/docker/nginx/conf.d/include/track-download @@ -16,4 +16,4 @@ log_by_lua_block { ngx.log(ngx.ERR, "accounts endpoint /track/download/" .. skylink .. " failed with error " .. statusCode) end end -} \ No newline at end of file +} diff --git a/docker/nginx/conf.d/include/track-upload b/docker/nginx/conf.d/include/track-upload index c6a8ef4d..b95f46e2 100644 --- a/docker/nginx/conf.d/include/track-upload +++ b/docker/nginx/conf.d/include/track-upload @@ -15,4 +15,4 @@ log_by_lua_block { ngx.log(ngx.ERR, "accounts endpoint /track/upload/" .. skylink .. " failed with error " .. statusCode) end end -} \ No newline at end of file +} From 439f4309a4c3dbd97235e78972ccbd9985e956a6 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 4 Jun 2021 13:23:07 +0200 Subject: [PATCH 6/6] move registry tracking to separate file --- docker/nginx/conf.d/client.conf | 20 +------------------- docker/nginx/conf.d/include/track-registry | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 19 deletions(-) create mode 100644 docker/nginx/conf.d/include/track-registry diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index c155ca5d..eb54c2eb 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -253,6 +253,7 @@ server { location /skynet/registry { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; + include /etc/nginx/conf.d/include/track-registry; limit_req zone=registry_access_by_ip burst=600 nodelay; limit_req zone=registry_access_by_ip_throttled burst=200 nodelay; @@ -274,25 +275,6 @@ server { end end } - - # register the registry access in accounts service (cookies should contain jwt) - log_by_lua_block { - -- this block runs only when accounts are enabled - if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end - - if ngx.status == ngx.HTTP_OK or ngx.status == ngx.HTTP_NOT_FOUND then - local http = require("socket.http") - local method = ngx.req.get_method() == ngx.HTTP_GET and "read" or "write" - local ok, statusCode, headers, statusText = http.request { - url = "http://accounts:3000/track/registry/" .. method, - method = "POST", - headers = ngx.req.get_headers() - } - if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then - ngx.log(ngx.ERR, "accounts endpoint /track/registry/" .. method .. " failed with error " .. statusCode) - end - end - } } location /skynet/skyfile { diff --git a/docker/nginx/conf.d/include/track-registry b/docker/nginx/conf.d/include/track-registry new file mode 100644 index 00000000..a8324f3b --- /dev/null +++ b/docker/nginx/conf.d/include/track-registry @@ -0,0 +1,18 @@ +# register the registry access in accounts service (cookies should contain jwt) +log_by_lua_block { + -- this block runs only when accounts are enabled + if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end + + if ngx.status == ngx.HTTP_OK or ngx.status == ngx.HTTP_NOT_FOUND then + local http = require("socket.http") + local method = ngx.req.get_method() == ngx.HTTP_GET and "read" or "write" + local ok, statusCode, headers, statusText = http.request { + url = "http://accounts:3000/track/registry/" .. method, + method = "POST", + headers = ngx.req.get_headers() + } + if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then + ngx.log(ngx.ERR, "accounts endpoint /track/registry/" .. method .. " failed with error " .. statusCode) + end + end +}