From ed7c1f71c7b4f839210708a23fadf3bc76102ecb Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 4 Jun 2021 12:33:37 +0200 Subject: [PATCH] add /skynet/pin endpoint and clean up config --- docker/nginx/conf.d/client.conf | 68 ++------------------ docker/nginx/conf.d/include/generate-siapath | 11 ++++ docker/nginx/conf.d/include/track-download | 19 ++++++ docker/nginx/conf.d/include/track-upload | 18 ++++++ 4 files changed, 53 insertions(+), 63 deletions(-) create mode 100644 docker/nginx/conf.d/include/generate-siapath create mode 100644 docker/nginx/conf.d/include/track-download create mode 100644 docker/nginx/conf.d/include/track-upload diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 17c67bd3..3798e43c 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -325,6 +325,8 @@ server { location /skynet/skyfile { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; + include /etc/nginx/conf.d/include/track-upload; + include /etc/nginx/conf.d/include/generate-siapath; limit_req zone=uploads_by_ip burst=100 nodelay; limit_req zone=uploads_by_ip_throttled; @@ -338,18 +340,6 @@ server { proxy_set_header Expect $http_expect; proxy_set_header User-Agent: Sia-Agent; - # Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2 - # respectfully. The rest of the $request_id is going to be assigned to $dir3. - # We use those variables to automatically generate a unique path for the uploaded file. - # This ensures that not all uploaded files end up in the same directory, which is something - # that causes performance issues in the renter. - # Example path result: /af/24/9bc5ec894920ccc45634dc9a8065 - if ($request_id ~* "(\w{2})(\w{2})(\w+)") { - set $dir1 $1; - set $dir2 $2; - set $dir3 $3; - } - # access_by_lua_block { # -- this block runs only when accounts are enabled # if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end @@ -365,25 +355,6 @@ server { # proxy this call to siad endpoint (make sure the ip is correct) proxy_pass http://siad/skynet/skyfile/$dir1/$dir2/$dir3$is_args$args; - - # register the upload in accounts service (cookies should contain jwt) - log_by_lua_block { - -- this block runs only when accounts are enabled - if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end - - local skylink = ngx.header["Skynet-Skylink"] - if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then - local http = require("socket.http") - local ok, statusCode, headers, statusText = http.request { - url = "http://accounts:3000/track/upload/" .. skylink, - method = "POST", - headers = ngx.req.get_headers() - } - if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then - ngx.log(ngx.ERR, "accounts endpoint /track/upload/" .. skylink .. " failed with error " .. statusCode) - end - end - } } # endpoing implementing resumable file uploads open protocol https://tus.io @@ -405,18 +376,8 @@ server { location /skynet/pin { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; - - # Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2 - # respectfully. The rest of the $request_id is going to be assigned to $dir3. - # We use those variables to automatically generate a unique path for the uploaded file. - # This ensures that not all uploaded files end up in the same directory, which is something - # that causes performance issues in the renter. - # Example path result: /af/24/9bc5ec894920ccc45634dc9a8065 - if ($request_id ~* "(\w{2})(\w{2})(\w+)") { - set $dir1 $1; - set $dir2 $2; - set $dir3 $3; - } + include /etc/nginx/conf.d/include/track-upload; + include /etc/nginx/conf.d/include/generate-siapath; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad$uri?siapath=$dir1/$dir2/$dir3&$args; @@ -433,6 +394,7 @@ server { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/proxy-buffer; include /etc/nginx/conf.d/include/proxy-cache-downloads; + include /etc/nginx/conf.d/include/track-download; # redirect purge calls to separate location error_page 462 = @purge; @@ -467,26 +429,6 @@ server { end } - # register the download in accounts service (cookies should contain jwt) - log_by_lua_block { - -- this block runs only when accounts are enabled - if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end - - local skylink = ngx.header["Skynet-Skylink"] - if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then - local http = require("socket.http") - local query = table.concat({ "status=" .. ngx.status, "bytes=" .. ngx.var.body_bytes_sent }, "&") - local ok, statusCode, headers, statusText = http.request { - url = "http://accounts:3000/track/download/" .. skylink .. "?" .. query, - method = "POST", - headers = ngx.req.get_headers() - } - if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then - ngx.log(ngx.ERR, "accounts endpoint /track/download/" .. skylink .. " failed with error " .. statusCode) - end - end - } - proxy_read_timeout 600; proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) diff --git a/docker/nginx/conf.d/include/generate-siapath b/docker/nginx/conf.d/include/generate-siapath new file mode 100644 index 00000000..a25c9c15 --- /dev/null +++ b/docker/nginx/conf.d/include/generate-siapath @@ -0,0 +1,11 @@ +# Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2 +# respectfully. The rest of the $request_id is going to be assigned to $dir3. +# We use those variables to automatically generate a unique path for the uploaded file. +# This ensures that not all uploaded files end up in the same directory, which is something +# that causes performance issues in the renter. +# Example path result: /af/24/9bc5ec894920ccc45634dc9a8065 +if ($request_id ~* "(\w{2})(\w{2})(\w+)") { + set $dir1 $1; + set $dir2 $2; + set $dir3 $3; +} \ No newline at end of file diff --git a/docker/nginx/conf.d/include/track-download b/docker/nginx/conf.d/include/track-download new file mode 100644 index 00000000..c767eb8a --- /dev/null +++ b/docker/nginx/conf.d/include/track-download @@ -0,0 +1,19 @@ +# register the download in accounts service (cookies should contain jwt) +log_by_lua_block { + -- this block runs only when accounts are enabled + if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end + + local skylink = ngx.header["Skynet-Skylink"] + if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then + local http = require("socket.http") + local query = table.concat({ "status=" .. ngx.status, "bytes=" .. ngx.var.body_bytes_sent }, "&") + local ok, statusCode, headers, statusText = http.request { + url = "http://accounts:3000/track/download/" .. skylink .. "?" .. query, + method = "POST", + headers = ngx.req.get_headers() + } + if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then + ngx.log(ngx.ERR, "accounts endpoint /track/download/" .. skylink .. " failed with error " .. statusCode) + end + end +} \ No newline at end of file diff --git a/docker/nginx/conf.d/include/track-upload b/docker/nginx/conf.d/include/track-upload new file mode 100644 index 00000000..c6a8ef4d --- /dev/null +++ b/docker/nginx/conf.d/include/track-upload @@ -0,0 +1,18 @@ +# register the upload in accounts service (cookies should contain jwt) +log_by_lua_block { + -- this block runs only when accounts are enabled + if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end + + local skylink = ngx.header["Skynet-Skylink"] + if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then + local http = require("socket.http") + local ok, statusCode, headers, statusText = http.request { + url = "http://accounts:3000/track/upload/" .. skylink, + method = "POST", + headers = ngx.req.get_headers() + } + if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then + ngx.log(ngx.ERR, "accounts endpoint /track/upload/" .. skylink .. " failed with error " .. statusCode) + end + end +} \ No newline at end of file