LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #830 from SkynetLabs/pin-endpoint 

/skynet/pin/<skylink> endpoint
This commit is contained in:
Christopher Schinnerl 2021-06-04 13:26:49 +02:00 committed by GitHub
parent 3ac8cf2fe9 439f4309a4
commit 490bcd1d44
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 80 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
docker/nginx/conf.d/client.conf
View File

@ -253,6 +253,7 @@ server {
location /skynet/registry { location /skynet/registry {
include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/cors;
include /etc/nginx/conf.d/include/sia-auth; include /etc/nginx/conf.d/include/sia-auth;
include /etc/nginx/conf.d/include/track-registry;
limit_req zone=registry_access_by_ip burst=600 nodelay; limit_req zone=registry_access_by_ip burst=600 nodelay;
limit_req zone=registry_access_by_ip_throttled burst=200 nodelay; limit_req zone=registry_access_by_ip_throttled burst=200 nodelay;
@ -274,30 +275,13 @@ server {
end end
end end
} }
# register the registry access in accounts service (cookies should contain jwt)
log_by_lua_block {
-- this block runs only when accounts are enabled
if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end
if ngx.status == ngx.HTTP_OK or ngx.status == ngx.HTTP_NOT_FOUND then
local http = require("socket.http")
local method = ngx.req.get_method() == ngx.HTTP_GET and "read" or "write"
local ok, statusCode, headers, statusText = http.request {
url = "http://accounts:3000/track/registry/" .. method,
method = "POST",
headers = ngx.req.get_headers()
}
if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then
ngx.log(ngx.ERR, "accounts endpoint /track/registry/" .. method .. " failed with error " .. statusCode)
end
end
}
} }
location /skynet/skyfile { location /skynet/skyfile {
include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/cors;
include /etc/nginx/conf.d/include/sia-auth; include /etc/nginx/conf.d/include/sia-auth;
include /etc/nginx/conf.d/include/track-upload;
include /etc/nginx/conf.d/include/generate-siapath;
limit_req zone=uploads_by_ip burst=100 nodelay; limit_req zone=uploads_by_ip burst=100 nodelay;
limit_req zone=uploads_by_ip_throttled; limit_req zone=uploads_by_ip_throttled;
@ -315,18 +299,6 @@ server {
proxy_set_header Expect $http_expect; proxy_set_header Expect $http_expect;
proxy_set_header User-Agent: Sia-Agent; proxy_set_header User-Agent: Sia-Agent;
# Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2
# respectfully. The rest of the $request_id is going to be assigned to $dir3.
# We use those variables to automatically generate a unique path for the uploaded file.
# This ensures that not all uploaded files end up in the same directory, which is something
# that causes performance issues in the renter.
# Example path result: /af/24/9bc5ec894920ccc45634dc9a8065
if ($request_id ~* "(\w{2})(\w{2})(\w+)") {
set $dir1 $1;
set $dir2 $2;
set $dir3 $3;
}
# access_by_lua_block { # access_by_lua_block {
# -- this block runs only when accounts are enabled # -- this block runs only when accounts are enabled
# if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end # if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end
@ -342,25 +314,6 @@ server {
# proxy this call to siad endpoint (make sure the ip is correct) # proxy this call to siad endpoint (make sure the ip is correct)
proxy_pass http://siad/skynet/skyfile/$dir1/$dir2/$dir3$is_args$args; proxy_pass http://siad/skynet/skyfile/$dir1/$dir2/$dir3$is_args$args;
# register the upload in accounts service (cookies should contain jwt)
log_by_lua_block {
-- this block runs only when accounts are enabled
if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end
local skylink = ngx.header["Skynet-Skylink"]
if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then
local http = require("socket.http")
local ok, statusCode, headers, statusText = http.request {
url = "http://accounts:3000/track/upload/" .. skylink,
method = "POST",
headers = ngx.req.get_headers()
}
if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then
ngx.log(ngx.ERR, "accounts endpoint /track/upload/" .. skylink .. " failed with error " .. statusCode)
end
end
}
} }
# endpoing implementing resumable file uploads open protocol https://tus.io # endpoing implementing resumable file uploads open protocol https://tus.io
@ -383,6 +336,16 @@ server {
proxy_redirect https://siad/ https://$domain.$tld/; proxy_redirect https://siad/ https://$domain.$tld/;
} }
location /skynet/pin {
include /etc/nginx/conf.d/include/cors;
include /etc/nginx/conf.d/include/sia-auth;
include /etc/nginx/conf.d/include/track-upload;
include /etc/nginx/conf.d/include/generate-siapath;
proxy_set_header User-Agent: Sia-Agent;
proxy_pass http://siad$uri?siapath=$dir1/$dir2/$dir3&$args;
}
location /skynet/metadata { location /skynet/metadata {
include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/cors;
@ -401,6 +364,7 @@ server {
include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/cors;
include /etc/nginx/conf.d/include/proxy-buffer; include /etc/nginx/conf.d/include/proxy-buffer;
include /etc/nginx/conf.d/include/proxy-cache-downloads; include /etc/nginx/conf.d/include/proxy-cache-downloads;
include /etc/nginx/conf.d/include/track-download;
# redirect purge calls to separate location # redirect purge calls to separate location
error_page 462 = @purge; error_page 462 = @purge;
@ -447,37 +411,6 @@ server {
end end
} }
# in case siad returns location header and we have skylink v2, we need to replace the skylink v1 with v2
header_filter_by_lua_block {
if ngx.var.skylink_v1 ~= ngx.var.skylink_v2 then
if ngx.header.location then
local path = string.match(ngx.header.location, "[^/?]+(.*)");
ngx.header.location = ngx.var.skylink_v2 .. path
end
ngx.header["Skynet-Skylink"] = ngx.var.skylink_v2
end
}
# register the download in accounts service (cookies should contain jwt)
log_by_lua_block {
-- this block runs only when accounts are enabled
if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end
local skylink = ngx.header["Skynet-Skylink"]
if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then
local http = require("socket.http")
local query = table.concat({ "status=" .. ngx.status, "bytes=" .. ngx.var.body_bytes_sent }, "&")
local ok, statusCode, headers, statusText = http.request {
url = "http://accounts:3000/track/download/" .. skylink .. "?" .. query,
method = "POST",
headers = ngx.req.get_headers()
}
if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then
ngx.log(ngx.ERR, "accounts endpoint /track/download/" .. skylink .. " failed with error " .. statusCode)
end
end
}
proxy_read_timeout 600; proxy_read_timeout 600;
proxy_set_header User-Agent: Sia-Agent; proxy_set_header User-Agent: Sia-Agent;
# proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct)

11
docker/nginx/conf.d/include/generate-siapath Normal file
View File

@ -0,0 +1,11 @@
# Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2
# respectfully. The rest of the $request_id is going to be assigned to $dir3.
# We use those variables to automatically generate a unique path for the uploaded file.
# This ensures that not all uploaded files end up in the same directory, which is something
# that causes performance issues in the renter.
# Example path result: /af/24/9bc5ec894920ccc45634dc9a8065
if ($request_id ~* "(\w{2})(\w{2})(\w+)") {
set $dir1 $1;
set $dir2 $2;
set $dir3 $3;
}

19
docker/nginx/conf.d/include/track-download Normal file
View File

@ -0,0 +1,19 @@
# register the download in accounts service (cookies should contain jwt)
log_by_lua_block {
-- this block runs only when accounts are enabled
if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end
local skylink = ngx.header["Skynet-Skylink"]
if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then
local http = require("socket.http")
local query = table.concat({ "status=" .. ngx.status, "bytes=" .. ngx.var.body_bytes_sent }, "&")
local ok, statusCode, headers, statusText = http.request {
url = "http://accounts:3000/track/download/" .. skylink .. "?" .. query,
method = "POST",
headers = ngx.req.get_headers()
}
if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then
ngx.log(ngx.ERR, "accounts endpoint /track/download/" .. skylink .. " failed with error " .. statusCode)
end
end
}

18
docker/nginx/conf.d/include/track-registry Normal file
View File

@ -0,0 +1,18 @@
# register the registry access in accounts service (cookies should contain jwt)
log_by_lua_block {
-- this block runs only when accounts are enabled
if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end
if ngx.status == ngx.HTTP_OK or ngx.status == ngx.HTTP_NOT_FOUND then
local http = require("socket.http")
local method = ngx.req.get_method() == ngx.HTTP_GET and "read" or "write"
local ok, statusCode, headers, statusText = http.request {
url = "http://accounts:3000/track/registry/" .. method,
method = "POST",
headers = ngx.req.get_headers()
}
if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then
ngx.log(ngx.ERR, "accounts endpoint /track/registry/" .. method .. " failed with error " .. statusCode)
end
end
}

18
docker/nginx/conf.d/include/track-upload Normal file
View File

@ -0,0 +1,18 @@
# register the upload in accounts service (cookies should contain jwt)
log_by_lua_block {
-- this block runs only when accounts are enabled
if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end
local skylink = ngx.header["Skynet-Skylink"]
if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then
local http = require("socket.http")
local ok, statusCode, headers, statusText = http.request {
url = "http://accounts:3000/track/upload/" .. skylink,
method = "POST",
headers = ngx.req.get_headers()
}
if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then
ngx.log(ngx.ERR, "accounts endpoint /track/upload/" .. skylink .. " failed with error " .. statusCode)
end
end
}