diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 6b825280..eb54c2eb 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -253,6 +253,7 @@ server { location /skynet/registry { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; + include /etc/nginx/conf.d/include/track-registry; limit_req zone=registry_access_by_ip burst=600 nodelay; limit_req zone=registry_access_by_ip_throttled burst=200 nodelay; @@ -274,30 +275,13 @@ server { end end } - - # register the registry access in accounts service (cookies should contain jwt) - log_by_lua_block { - -- this block runs only when accounts are enabled - if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end - - if ngx.status == ngx.HTTP_OK or ngx.status == ngx.HTTP_NOT_FOUND then - local http = require("socket.http") - local method = ngx.req.get_method() == ngx.HTTP_GET and "read" or "write" - local ok, statusCode, headers, statusText = http.request { - url = "http://accounts:3000/track/registry/" .. method, - method = "POST", - headers = ngx.req.get_headers() - } - if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then - ngx.log(ngx.ERR, "accounts endpoint /track/registry/" .. method .. " failed with error " .. statusCode) - end - end - } } location /skynet/skyfile { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; + include /etc/nginx/conf.d/include/track-upload; + include /etc/nginx/conf.d/include/generate-siapath; limit_req zone=uploads_by_ip burst=100 nodelay; limit_req zone=uploads_by_ip_throttled; @@ -315,18 +299,6 @@ server { proxy_set_header Expect $http_expect; proxy_set_header User-Agent: Sia-Agent; - # Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2 - # respectfully. The rest of the $request_id is going to be assigned to $dir3. - # We use those variables to automatically generate a unique path for the uploaded file. - # This ensures that not all uploaded files end up in the same directory, which is something - # that causes performance issues in the renter. - # Example path result: /af/24/9bc5ec894920ccc45634dc9a8065 - if ($request_id ~* "(\w{2})(\w{2})(\w+)") { - set $dir1 $1; - set $dir2 $2; - set $dir3 $3; - } - # access_by_lua_block { # -- this block runs only when accounts are enabled # if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end @@ -342,25 +314,6 @@ server { # proxy this call to siad endpoint (make sure the ip is correct) proxy_pass http://siad/skynet/skyfile/$dir1/$dir2/$dir3$is_args$args; - - # register the upload in accounts service (cookies should contain jwt) - log_by_lua_block { - -- this block runs only when accounts are enabled - if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end - - local skylink = ngx.header["Skynet-Skylink"] - if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then - local http = require("socket.http") - local ok, statusCode, headers, statusText = http.request { - url = "http://accounts:3000/track/upload/" .. skylink, - method = "POST", - headers = ngx.req.get_headers() - } - if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then - ngx.log(ngx.ERR, "accounts endpoint /track/upload/" .. skylink .. " failed with error " .. statusCode) - end - end - } } # endpoing implementing resumable file uploads open protocol https://tus.io @@ -383,6 +336,16 @@ server { proxy_redirect https://siad/ https://$domain.$tld/; } + location /skynet/pin { + include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/sia-auth; + include /etc/nginx/conf.d/include/track-upload; + include /etc/nginx/conf.d/include/generate-siapath; + + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://siad$uri?siapath=$dir1/$dir2/$dir3&$args; + } + location /skynet/metadata { include /etc/nginx/conf.d/include/cors; @@ -401,6 +364,7 @@ server { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/proxy-buffer; include /etc/nginx/conf.d/include/proxy-cache-downloads; + include /etc/nginx/conf.d/include/track-download; # redirect purge calls to separate location error_page 462 = @purge; @@ -447,37 +411,6 @@ server { end } - # in case siad returns location header and we have skylink v2, we need to replace the skylink v1 with v2 - header_filter_by_lua_block { - if ngx.var.skylink_v1 ~= ngx.var.skylink_v2 then - if ngx.header.location then - local path = string.match(ngx.header.location, "[^/?]+(.*)"); - ngx.header.location = ngx.var.skylink_v2 .. path - end - ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2 - end - } - - # register the download in accounts service (cookies should contain jwt) - log_by_lua_block { - -- this block runs only when accounts are enabled - if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end - - local skylink = ngx.header["Skynet-Skylink"] - if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then - local http = require("socket.http") - local query = table.concat({ "status=" .. ngx.status, "bytes=" .. ngx.var.body_bytes_sent }, "&") - local ok, statusCode, headers, statusText = http.request { - url = "http://accounts:3000/track/download/" .. skylink .. "?" .. query, - method = "POST", - headers = ngx.req.get_headers() - } - if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then - ngx.log(ngx.ERR, "accounts endpoint /track/download/" .. skylink .. " failed with error " .. statusCode) - end - end - } - proxy_read_timeout 600; proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) diff --git a/docker/nginx/conf.d/include/generate-siapath b/docker/nginx/conf.d/include/generate-siapath new file mode 100644 index 00000000..51ef8d84 --- /dev/null +++ b/docker/nginx/conf.d/include/generate-siapath @@ -0,0 +1,11 @@ +# Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2 +# respectfully. The rest of the $request_id is going to be assigned to $dir3. +# We use those variables to automatically generate a unique path for the uploaded file. +# This ensures that not all uploaded files end up in the same directory, which is something +# that causes performance issues in the renter. +# Example path result: /af/24/9bc5ec894920ccc45634dc9a8065 +if ($request_id ~* "(\w{2})(\w{2})(\w+)") { + set $dir1 $1; + set $dir2 $2; + set $dir3 $3; +} diff --git a/docker/nginx/conf.d/include/track-download b/docker/nginx/conf.d/include/track-download new file mode 100644 index 00000000..376568c9 --- /dev/null +++ b/docker/nginx/conf.d/include/track-download @@ -0,0 +1,19 @@ +# register the download in accounts service (cookies should contain jwt) +log_by_lua_block { + -- this block runs only when accounts are enabled + if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end + + local skylink = ngx.header["Skynet-Skylink"] + if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then + local http = require("socket.http") + local query = table.concat({ "status=" .. ngx.status, "bytes=" .. ngx.var.body_bytes_sent }, "&") + local ok, statusCode, headers, statusText = http.request { + url = "http://accounts:3000/track/download/" .. skylink .. "?" .. query, + method = "POST", + headers = ngx.req.get_headers() + } + if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then + ngx.log(ngx.ERR, "accounts endpoint /track/download/" .. skylink .. " failed with error " .. statusCode) + end + end +} diff --git a/docker/nginx/conf.d/include/track-registry b/docker/nginx/conf.d/include/track-registry new file mode 100644 index 00000000..a8324f3b --- /dev/null +++ b/docker/nginx/conf.d/include/track-registry @@ -0,0 +1,18 @@ +# register the registry access in accounts service (cookies should contain jwt) +log_by_lua_block { + -- this block runs only when accounts are enabled + if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end + + if ngx.status == ngx.HTTP_OK or ngx.status == ngx.HTTP_NOT_FOUND then + local http = require("socket.http") + local method = ngx.req.get_method() == ngx.HTTP_GET and "read" or "write" + local ok, statusCode, headers, statusText = http.request { + url = "http://accounts:3000/track/registry/" .. method, + method = "POST", + headers = ngx.req.get_headers() + } + if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then + ngx.log(ngx.ERR, "accounts endpoint /track/registry/" .. method .. " failed with error " .. statusCode) + end + end +} diff --git a/docker/nginx/conf.d/include/track-upload b/docker/nginx/conf.d/include/track-upload new file mode 100644 index 00000000..b95f46e2 --- /dev/null +++ b/docker/nginx/conf.d/include/track-upload @@ -0,0 +1,18 @@ +# register the upload in accounts service (cookies should contain jwt) +log_by_lua_block { + -- this block runs only when accounts are enabled + if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end + + local skylink = ngx.header["Skynet-Skylink"] + if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then + local http = require("socket.http") + local ok, statusCode, headers, statusText = http.request { + url = "http://accounts:3000/track/upload/" .. skylink, + method = "POST", + headers = ngx.req.get_headers() + } + if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then + ngx.log(ngx.ERR, "accounts endpoint /track/upload/" .. skylink .. " failed with error " .. statusCode) + end + end +}