proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=skynet:10m max_size=10g inactive=24h use_temp_path=off; # this runs before forking out nginx worker processes init_by_lua_block { require "cjson" require "socket.http" } # ratelimit specified IPs geo $limit { default 0; include /etc/nginx/conf.d/include/ratelimited; } map $limit $limit_key { 0 ""; 1 $binary_remote_addr; } limit_req_zone $binary_remote_addr zone=uploads_by_ip:10m rate=10r/s; limit_req_zone $limit_key zone=uploads_by_ip_throttled:10m rate=10r/m; limit_req_zone $binary_remote_addr zone=registry_access_by_ip:10m rate=60r/m; limit_req_zone $limit_key zone=registry_access_by_ip_throttled:10m rate=20r/m; limit_conn_zone $binary_remote_addr zone=upload_conn:10m; limit_conn_zone $limit_key zone=upload_conn_rl:10m; limit_conn_zone $binary_remote_addr zone=downloads_by_ip:10m; limit_req_status 429; limit_conn_status 429; # since we are proxying request to nginx from caddy, access logs will contain caddy's ip address # as the request address so we need to use real_ip_header module to use ip address from # X-Forwarded-For header as a real ip address of the request set_real_ip_from 10.0.0.0/8; set_real_ip_from 127.0.0.1/32; set_real_ip_from 172.16.0.0/12; set_real_ip_from 192.168.0.0/16; real_ip_header X-Forwarded-For; # skynet-jwt contains dash so we cannot use $cookie_skynet-jwt # https://richardhart.me/2012/03/18/logging-nginx-cookies-with-dashes/ map $http_cookie $skynet_jwt { default ''; ~skynet-jwt=(?[^\;]+) $match; } upstream siad { server sia:9980; } server { listen 80 default_server; listen [::]:80 default_server; # understand the regex https://regex101.com/r/BGQvi6/6 server_name "~^(((?([a-z0-9]{55}))|(?[^\.]+)\.hns)\.)?((?[^.]+)\.)?(?[^.]+)\.(?[^.]+)$"; # ddos protection: closing slow connections client_body_timeout 5s; client_header_timeout 5s; # Increase the body buffer size, to ensure the internal POSTs can always # parse the full POST contents into memory. client_body_buffer_size 128k; client_max_body_size 128k; # legacy endpoint rewrite rewrite ^/portals /skynet/portals permanent; rewrite ^/stats /skynet/stats permanent; rewrite ^/skynet/blacklist /skynet/blocklist permanent; rewrite ^/account/(.*) https://account.$domain.$tld/$1 permanent; # This is only safe workaround to reroute based on some conditions # See https://www.nginx.com/resources/wiki/start/topics/depth/ifisevil/ recursive_error_pages on; # redirect links with base32 encoded skylink in subdomain error_page 460 = @base32_subdomain; if ($base32_subdomain != "") { return 460; } # redirect links with handshake domain on hns subdomain error_page 461 = @hns_domain; if ($hns_domain != "") { return 461; } location / { include /etc/nginx/conf.d/include/cors; proxy_pass http://website:9000; } location /docs { proxy_pass https://skynetlabs.github.io/skynet-docs; } location /skynet/blocklist { include /etc/nginx/conf.d/include/cors; proxy_cache skynet; proxy_cache_valid any 1m; # cache blocklist for 1 minute proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad/skynet/blocklist; } location /skynet/portals { include /etc/nginx/conf.d/include/cors; proxy_cache skynet; proxy_cache_valid any 1m; # cache portals for 1 minute proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad/skynet/portals; } location /skynet/stats { include /etc/nginx/conf.d/include/cors; set $response_body ''; # we need a variable for full response body (not chuncked) # modify the response to add numfiles and totalsize to account for node rotation # example prevstats.lua: 'return { numfiles = 12345, totalsize = 123456789 }' body_filter_by_lua_block { local file_exists = io.open("/data/nginx/skynet/prevstats.lua") if file_exists then file_exists.close() -- because response data is chunked, we need to concat ngx.arg[1] until -- last chunk is received (when ngx.arg[2] is set to true) ngx.var.response_body = ngx.var.response_body .. ngx.arg[1] if ngx.arg[2] then local json = require('cjson') local prevstats = require('/data/nginx/skynet/prevstats') local stats = json.decode(ngx.var.response_body) stats.uploadstats.numfiles = stats.uploadstats.numfiles + prevstats.numfiles stats.uploadstats.totalsize = stats.uploadstats.totalsize + prevstats.totalsize ngx.arg[1] = json.encode(stats) else -- do not send any data in this chunk (wait for last chunk) ngx.arg[1] = nil end end } proxy_cache skynet; proxy_cache_valid any 1m; # cache stats for 1 minute proxy_set_header User-Agent: Sia-Agent; proxy_read_timeout 5m; # extend the read timeout proxy_pass http://siad/skynet/stats; } location /health-check { include /etc/nginx/conf.d/include/cors; access_log off; # do not log traffic to health-check endpoint proxy_pass http://10.10.10.60:3100; # hardcoded ip because health-check waits for nginx } location /hns { include /etc/nginx/conf.d/include/proxy-buffer; # variable definititions - we need to define a variable to be able to access it in lua by ngx.var.something set $skylink ''; # placeholder for the raw 46 bit skylink set $rest ''; # placeholder for the rest of the url that gets appended to skylink (path and args) # resolve handshake domain by requesting to /hnsres endpoint and assign correct values to $skylink and $rest access_by_lua_block { local json = require('cjson') -- match the request_uri and extract the hns domain and anything that is passed in the uri after it -- example: /hns/something/foo/bar?baz=1 matches: -- > hns_domain_name: something -- > request_uri_rest: /foo/bar/?baz=1 local hns_domain_name, request_uri_rest = string.match(ngx.var.request_uri, "/hns/([^/?]+)(.*)") -- make a get request to /hnsres endpoint with the domain name from request_uri local hnsres_res = ngx.location.capture("/hnsres/" .. hns_domain_name) -- we want to fail with a generic 404 when /hnsres returns anything but 200 OK with a skylink if hnsres_res.status ~= ngx.HTTP_OK then ngx.exit(ngx.HTTP_NOT_FOUND) end -- since /hnsres endpoint response is a json, we need to decode it before we access it -- example response: '{"skylink":"sia://XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg"}' local hnsres_json = json.decode(hnsres_res.body) -- define local variable containing rest of the skylink if provided local skylink_rest if hnsres_json.skylink then -- try to match the skylink with sia:// prefix skylink, skylink_rest = string.match(hnsres_json.skylink, "sia://([^/?]+)(.*)") -- in case the skylink did not match, assume that there is no sia:// prefix and try to match again if skylink == nil then skylink, skylink_rest = string.match(hnsres_json.skylink, "/?([^/?]+)(.*)") end elseif hnsres_json.registry then local publickey = hnsres_json.registry.publickey local datakey = hnsres_json.registry.datakey -- make a get request to /skynet/registry endpoint with the credentials from text record local registry_res = ngx.location.capture("/skynet/registry/cached?publickey=" .. publickey .. "&datakey=" .. datakey) -- we want to fail with a generic 404 when /skynet/registry returns anything but 200 OK if registry_res.status ~= ngx.HTTP_OK then ngx.exit(ngx.HTTP_NOT_FOUND) end -- since /skynet/registry endpoint response is a json, we need to decode it before we access it local registry_json = json.decode(registry_res.body) -- response will contain a hex encoded skylink, we need to decode it local data = (registry_json.data:gsub('..', function (cc) return string.char(tonumber(cc, 16)) end)) skylink = data end -- fail with a generic 404 if skylink has not been extracted from a valid /hnsres response for some reason if not skylink then ngx.exit(ngx.HTTP_NOT_FOUND) end ngx.var.skylink = skylink if request_uri_rest == "/" and skylink_rest ~= nil and skylink_rest ~= "" and skylink_rest ~= "/" then ngx.var.rest = skylink_rest else ngx.var.rest = request_uri_rest end } # we proxy to another nginx location rather than directly to siad because we don't want to deal with caching here proxy_pass http://127.0.0.1/$skylink$rest; # in case siad returns location header, we need to replace the skylink with the domain name header_filter_by_lua_block { if ngx.header.location then -- match hns domain from the request_uri local hns_domain_name = string.match(ngx.var.request_uri, "/hns/([^/?]+)") -- match location redirect part after the skylink local location_rest = string.match(ngx.header.location, "[^/?]+(.*)"); -- because siad will set the location header to ie. XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg/index.html -- we need to replace the skylink with the domain_name so we are not redirected to skylink ngx.header.location = hns_domain_name .. location_rest end } } location /hnsres { include /etc/nginx/conf.d/include/cors; proxy_pass http://handshake-api:3100; } # internal registry endpoint that caches calls for a certain period of time # it is not suitable for every registry call but some requests might be cached # and we are using it currently for caching registry resolutions from /hns calls location /skynet/registry/cached { internal; # internal endpoint only access_log off; # do not log traffic proxy_cache skynet; proxy_cache_key publickey=$arg_publickey&datakey=$arg_datakey; # cache based on publickey and datakey proxy_cache_valid 200 30s; # cache only 200 responses and only for 30 seconds proxy_cache_lock on; # queue cache requests for the same resource until it is fully cached proxy_cache_bypass $cookie_nocache $arg_nocache; # add cache bypass option proxy_pass http://127.0.0.1/skynet/registry$is_args$args; } location /skynet/registry { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; limit_req zone=registry_access_by_ip burst=600 nodelay; limit_req zone=registry_access_by_ip_throttled burst=200 nodelay; proxy_set_header User-Agent: Sia-Agent; proxy_read_timeout 600; # siad should timeout with 404 after 5 minutes proxy_pass http://siad/skynet/registry; access_by_lua_block { -- this block runs only when accounts are enabled if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end local res = ngx.location.capture("/accounts/user/limits", { copy_all_vars = true }) if res.status == ngx.HTTP_OK then local json = require('cjson') local limits = json.decode(res.body) if limits.registry > 0 then ngx.sleep(limits.registry / 1000) end end } # register the registry access in accounts service (cookies should contain jwt) log_by_lua_block { -- this block runs only when accounts are enabled if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end if ngx.status == ngx.HTTP_OK or ngx.status == ngx.HTTP_NOT_FOUND then local http = require("socket.http") local method = ngx.req.get_method() == ngx.HTTP_GET and "read" or "write" local ok, statusCode, headers, statusText = http.request { url = "http://accounts:3000/track/registry/" .. method, method = "POST", headers = ngx.req.get_headers() } if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then ngx.log(ngx.ERR, "accounts endpoint /track/registry/" .. method .. " failed with error " .. statusCode) end end } } location /skynet/skyfile { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/sia-auth; limit_req zone=uploads_by_ip burst=100 nodelay; limit_req zone=uploads_by_ip_throttled; limit_conn upload_conn 10; limit_conn upload_conn_rl 1; client_max_body_size 1000M; # make sure to limit the size of upload to a sane value proxy_read_timeout 600; proxy_request_buffering off; # stream uploaded files through the proxy as it comes in proxy_set_header Expect $http_expect; proxy_set_header User-Agent: Sia-Agent; # Extract 2 sets of 2 characters from $request_id and assign to $dir1, $dir2 # respectfully. The rest of the $request_id is going to be assigned to $dir3. # We use those variables to automatically generate a unique path for the uploaded file. # This ensures that not all uploaded files end up in the same directory, which is something # that causes performance issues in the renter. # Example path result: /af/24/9bc5ec894920ccc45634dc9a8065 if ($request_id ~* "(\w{2})(\w{2})(\w+)") { set $dir1 $1; set $dir2 $2; set $dir3 $3; } # access_by_lua_block { # -- this block runs only when accounts are enabled # if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end # ngx.var.upload_limit_rate = 5 * 1024 * 1024 # local res = ngx.location.capture("/accounts/user", { copy_all_vars = true }) # if res.status == ngx.HTTP_OK then # local json = require('cjson') # local user = json.decode(res.body) # ngx.var.upload_limit_rate = ngx.var.upload_limit_rate * (user.tier + 1) # end # } # proxy this call to siad endpoint (make sure the ip is correct) proxy_pass http://siad/skynet/skyfile/$dir1/$dir2/$dir3$is_args$args; # register the upload in accounts service (cookies should contain jwt) log_by_lua_block { -- this block runs only when accounts are enabled if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end local skylink = ngx.header["Skynet-Skylink"] if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then local http = require("socket.http") local ok, statusCode, headers, statusText = http.request { url = "http://accounts:3000/track/upload/" .. skylink, method = "POST", headers = ngx.req.get_headers() } if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then ngx.log(ngx.ERR, "accounts endpoint /track/upload/" .. skylink .. " failed with error " .. statusCode) end end } } # endpoing implementing resumable file uploads open protocol https://tus.io location /skynet/tus { include /etc/nginx/conf.d/include/cors; client_max_body_size 50M; # tus chunks size is 40M + leaving 10M of breathing room proxy_read_timeout 600; proxy_request_buffering off; # stream uploaded files through the proxy as it comes in proxy_set_header Expect $http_expect; # proxy /skynet/tus requests to siad endpoint with all arguments proxy_pass http://siad; # rewrite tus headers to use correct uri proxy_redirect https://siad/ https://$domain.$tld/; } location /skynet/metadata { include /etc/nginx/conf.d/include/cors; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad; } location /skynet/resolve { include /etc/nginx/conf.d/include/cors; proxy_set_header User-Agent: Sia-Agent; proxy_pass http://siad; } location ~ "^/(([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?)$" { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/proxy-buffer; include /etc/nginx/conf.d/include/proxy-cache-downloads; # redirect purge calls to separate location error_page 462 = @purge; if ($request_method = PURGE) { return 462; } limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time # we need to explicitly use set directive here because $2 and $3 will contain values with # decoded whitespaces and set will re-encode it for us before passing it to proxy_pass set $skylink $2; set $path $3; # v2 support set $skylink_v1 $skylink; access_by_lua_block { -- disable cache if this is skylink v2 local isBase32v2 = string.len(ngx.var.skylink) == 55 and string.sub(ngx.var.skylink, 0, 2) == "04" local isBase64v2 = string.len(ngx.var.skylink) == 46 and string.sub(ngx.var.skylink, 0, 2) == "AQ" if isBase32v2 or isBase64v2 then local res = ngx.location.capture("/skynet/resolve/" .. ngx.var.skylink) if res.status == ngx.HTTP_OK then local json = require('cjson') local resolve = json.decode(res.body) ngx.var.skylink_v1 = resolve.skylink else ngx.say(res.body) ngx.exit(res.status) end end -- this block runs only when accounts are enabled if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end local res = ngx.location.capture("/accounts/user/limits", { copy_all_vars = true }) if res.status == ngx.HTTP_OK then local json = require('cjson') local limits = json.decode(res.body) ngx.var.limit_rate = limits.download end } # in case siad returns location header and we have skylink v2, we need to replace the skylink v1 with v2 header_filter_by_lua_block { if ngx.var.skylink != ngx.var.skylink_v1 then if ngx.header.location then ngx.header.location = ngx.header.location:gsub("^" .. ngx.var.skylink_v1, ngx.var.skylink) end ngx.header["Skynet-Skylink"] = ngx.var.skylink end } # register the download in accounts service (cookies should contain jwt) log_by_lua_block { -- this block runs only when accounts are enabled if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then return end local skylink = ngx.header["Skynet-Skylink"] if skylink and ngx.status >= ngx.HTTP_OK and ngx.status < ngx.HTTP_SPECIAL_RESPONSE then local http = require("socket.http") local query = table.concat({ "status=" .. ngx.status, "bytes=" .. ngx.var.body_bytes_sent }, "&") local ok, statusCode, headers, statusText = http.request { url = "http://accounts:3000/track/download/" .. skylink .. "?" .. query, method = "POST", headers = ngx.req.get_headers() } if statusCode ~= ngx.HTTP_NO_CONTENT and statusCode ~= ngx.HTTP_UNAUTHORIZED then ngx.log(ngx.ERR, "accounts endpoint /track/download/" .. skylink .. " failed with error " .. statusCode) end end } proxy_read_timeout 600; proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) proxy_pass http://siad/skynet/skylink/$skylink$path$is_args$args; } location @base32_subdomain { include /etc/nginx/conf.d/include/proxy-buffer; proxy_pass http://127.0.0.1/$base32_subdomain/$request_uri; } location @hns_domain { include /etc/nginx/conf.d/include/proxy-buffer; proxy_pass http://127.0.0.1/hns/$hns_domain/$request_uri; } location @purge { allow 10.0.0.0/8; allow 127.0.0.1/32; allow 172.16.0.0/12; allow 192.168.0.0/16; deny all; set $lua_purge_path "/data/nginx/cache/"; content_by_lua_file /etc/nginx/conf.d/scripts/purge-multi.lua; } location ~ "^/file/([a-zA-Z0-9-_]{46}(/.*)?)$" { include /etc/nginx/conf.d/include/proxy-buffer; rewrite /file/(.*) $1 break; # drop the /file/ prefix from uri proxy_pass http://127.0.0.1/$uri?attachment=true&$args; } location /__internal/do/not/use/authenticated { include /etc/nginx/conf.d/include/cors; charset utf-8; charset_types application/json; default_type application/json; content_by_lua_block { local json = require('cjson') -- this block runs only when accounts are enabled if os.getenv("ACCOUNTS_ENABLED", "0") == "0" then ngx.say(json.encode{authenticated = false}) return ngx.exit(ngx.HTTP_OK) end local res = ngx.location.capture("/accounts/user", { copy_all_vars = true }) if res.status == ngx.HTTP_OK then local limits = json.decode(res.body) ngx.say(json.encode{authenticated = limits.tier > 0}) return ngx.exit(ngx.HTTP_OK) end ngx.say(json.encode{authenticated = false}) return ngx.exit(ngx.HTTP_OK) } } location /accounts { internal; # internal endpoint only access_log off; # do not log traffic proxy_cache skynet; # use general nginx cache proxy_cache_key $uri+$skynet_jwt; # include skynet-jwt cookie (mapped to skynet_jwt) proxy_cache_valid 200 401 1m; # cache success and unauthorized responses for 1 minute rewrite /accounts(.*) $1 break; # drop the /accounts prefix from uri proxy_pass http://10.10.10.70:3000; # hardcoded ip because accounts might not be available } # include custom locations, specific to the server include /etc/nginx/conf.d/server-override/*; }