From c155fcfa5548d4e770304bac6d871822427ea939 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 15 Sep 2020 17:09:22 +0200 Subject: [PATCH 1/6] move handshake proxy to nginx --- docker/nginx/conf.d/client.conf | 29 +++++++++++++++++++++- packages/handshake-api/package.json | 1 - packages/handshake-api/src/index.js | 38 ----------------------------- yarn.lock | 33 +++++++++---------------- 4 files changed, 40 insertions(+), 61 deletions(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index c1a78ee8..9485c13b 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -130,7 +130,34 @@ server { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/proxy-buffer; - proxy_pass http://handshake-api:3100; + set $skylink ''; + set $skylink_path ''; + access_by_lua_block { + local json = require('cjson') + local hns_domain_name, request_uri_path = string.match(ngx.var.request_uri, "/hns/([^/]+)(.*)") + local hnsres_res = ngx.location.capture("/hnsres/" .. hns_domain_name) + local hnsres_json = json.decode(hnsres_res.body) + local skylink_prefix, skylink, skylink_path = string.match(hnsres_json.skylink, "(sia://)([^/]+)(.*)") + + ngx.var.skylink = skylink + if request_uri_path == "" or (request_uri_path == "/" and skylink_path ~= "") then + ngx.var.skylink_path = skylink_path + else + ngx.var.skylink_path = request_uri_path + end + } + + proxy_pass http://siad/skynet/skylink/$skylink$skylink_path; + + header_filter_by_lua_block { + if ngx.header.Location then + local json = require('cjson') + local hns_domain_name = string.match(ngx.var.request_uri, "/hns/(.+)/?") + local location = string.gsub(ngx.header.Location, ngx.var.skylink, hns_domain_name) + + ngx.header.Location = location + end + } } location /hnsres { diff --git a/packages/handshake-api/package.json b/packages/handshake-api/package.json index fbbf3a32..d8db7d3f 100644 --- a/packages/handshake-api/package.json +++ b/packages/handshake-api/package.json @@ -5,7 +5,6 @@ "license": "MIT", "dependencies": { "express": "^4.17.1", - "express-http-proxy": "^1.6.2", "hs-client": "^0.0.9", "node-cache": "^5.1.2" }, diff --git a/packages/handshake-api/src/index.js b/packages/handshake-api/src/index.js index fa522387..72aa2234 100644 --- a/packages/handshake-api/src/index.js +++ b/packages/handshake-api/src/index.js @@ -1,6 +1,4 @@ -const url = require("url"); const express = require("express"); -const proxy = require("express-http-proxy"); const NodeCache = require("node-cache"); const { NodeClient } = require("hs-client"); @@ -76,42 +74,6 @@ function isValidSkylink(link) { const server = express(); -server.use( - "/hns/:name", - proxy("nginx", { - // eslint-disable-next-line no-unused-vars - userResHeaderDecorator(headers, userReq, userRes, proxyReq, proxyRes) { - if (headers.location && headers.location.match(startsWithSkylinkRegExp)) { - headers.location = headers.location.replace( - startsWithSkylinkRegExp, - `/hns/${userReq.params.name.replace("sia://", "")}` - ); - } - - return headers; - }, - proxyReqPathResolver: async (req) => { - const records = await getDomainRecords(req.params.name); - if (!records) throw new Error(`No records found for ${req.params.name}`); - - const record = findSkylinkRecord(records); - if (!record) throw new Error(`No skylink found in dns records of ${req.params.name}`); - - const skylink = getSkylinkFromRecord(record).replace("sia://", ""); // get skylink and strip sia:// prefix - const basepath = url.resolve("/", skylink); // make the url absolute - const subpath = req.url.slice(1); // drop the leading slash - - // if the record is just a raw skylink, replace baseUrl with /skylink - if (skylink.length === 46) { - return req.originalUrl.replace(req.baseUrl, basepath); - } - - // if the record contains more than a skylink then it needs to be resolved - return url.resolve(basepath, subpath); - }, - }) -); - server.get("/hnsres/:name", resolveDomainHandler); server.listen(port, host, (error) => { diff --git a/yarn.lock b/yarn.lock index c0c8570c..91a4c8e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4836,7 +4836,7 @@ debug@=3.1.0, debug@~3.1.0: dependencies: ms "2.0.0" -debug@^3.0.0, debug@^3.0.1, debug@^3.1.0, debug@^3.1.1, debug@^3.2.5, debug@^3.2.6: +debug@^3.0.0, debug@^3.1.0, debug@^3.1.1, debug@^3.2.5, debug@^3.2.6: version "3.2.6" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ== @@ -5568,11 +5568,6 @@ es-to-primitive@^1.2.1: is-date-object "^1.0.1" is-symbol "^1.0.2" -es6-promise@^4.1.1: - version "4.2.8" - resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.8.tgz#4eb21594c972bc40553d276e510539143db53e0a" - integrity sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w== - escalade@^3.0.1: version "3.0.2" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.0.2.tgz#6a580d70edb87880f22b4c91d0d56078df6962c4" @@ -5759,10 +5754,10 @@ eslint-visitor-keys@^1.0.0, eslint-visitor-keys@^1.1.0, eslint-visitor-keys@^1.3 resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz#30ebd1ef7c2fdff01c3a4f151044af25fab0523e" integrity sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ== -eslint@7.8.1: - version "7.8.1" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.8.1.tgz#e59de3573fb6a5be8ff526c791571646d124a8fa" - integrity sha512-/2rX2pfhyUG0y+A123d0ccXtMm7DV7sH1m3lk9nk2DZ2LReq39FXHueR9xZwshE5MdfSf0xunSaMWRqyIA6M1w== +eslint@7.9.0: + version "7.9.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.9.0.tgz#522aeccc5c3a19017cf0cb46ebfd660a79acf337" + integrity sha512-V6QyhX21+uXp4T+3nrNfI3hQNBDa/P8ga7LoQOenwrlEFXrEnUEE+ok1dMtaS3b6rmLXhT1TkTIsG75HMLbknA== dependencies: "@babel/code-frame" "^7.0.0" "@eslint/eslintrc" "^0.1.3" @@ -6058,15 +6053,6 @@ express-graphql@^0.9.0: http-errors "^1.7.3" raw-body "^2.4.1" -express-http-proxy@^1.6.2: - version "1.6.2" - resolved "https://registry.yarnpkg.com/express-http-proxy/-/express-http-proxy-1.6.2.tgz#e87152e45958cee4b91da2fdaa20a1ffd581204a" - integrity sha512-soP7UXySFdLbeeMYL1foBkEoZj6HELq9BDAOCr1sLRpqjPaFruN5o6+bZeC+7U4USWIl4JMKEiIvTeKJ2WQdlQ== - dependencies: - debug "^3.0.1" - es6-promise "^4.1.1" - raw-body "^2.3.0" - express@^4.17.1: version "4.17.1" resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134" @@ -8097,7 +8083,12 @@ http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.7.0" -http-status-codes@2.1.2, http-status-codes@^2.1.2: +http-status-codes@2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/http-status-codes/-/http-status-codes-2.1.3.tgz#d0ab99a7f79afb43dd735bc862ff350171f3cc2f" + integrity sha512-/kDMtEEhAw747LvzDupRRsUOrmw/oEPmwf61guegI1ycj0vyywitq4BhGPknLzqAEBQvsW6rSv0dd2de1MU+yg== + +http-status-codes@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/http-status-codes/-/http-status-codes-2.1.2.tgz#dc5b7ccd72fb383d1d7c21adbbddc3b125f6a5b7" integrity sha512-zpZ1nBcoR0j1FLQ7xbXXBy1z/yUfAi+0a5IZBoZnmOseYkaljdzQ17ZeVXFlK23IbLxMJn6aWI0uU92DQQrG0g== @@ -12169,7 +12160,7 @@ raw-body@2.4.0: iconv-lite "0.4.24" unpipe "1.0.0" -raw-body@^2.3.0, raw-body@^2.4.1: +raw-body@^2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.4.1.tgz#30ac82f98bb5ae8c152e67149dac8d55153b168c" integrity sha512-9WmIKF6mkvA0SLmA2Knm9+qj89e+j1zqgyn8aXGd7+nAduPoqgI9lO57SAZNn/Byzo5P7JhXTyg9PzaJbH73bA== From c050948d23c44dfa8d63659a85e1d55fcc383082 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 15 Sep 2020 17:22:49 +0200 Subject: [PATCH 2/6] improve caching --- docker/nginx/conf.d/client.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 9485c13b..06ae4ffa 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -129,6 +129,10 @@ server { location /hns { include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/proxy-buffer; + include /etc/nginx/conf.d/include/proxy-cache-downloads; + + limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time + add_header Cache-Control "public, max-age=86400"; # allow consumer to cache response set $skylink ''; set $skylink_path ''; @@ -147,6 +151,9 @@ server { end } + proxy_read_timeout 600; + proxy_set_header User-Agent: Sia-Agent; + proxy_pass http://siad/skynet/skylink/$skylink$skylink_path; header_filter_by_lua_block { From 06d0466a1c7faeb5795c7897fdbe7bab2c9f6f14 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 15 Sep 2020 17:27:44 +0200 Subject: [PATCH 3/6] decrease cache time --- docker/nginx/conf.d/client.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 06ae4ffa..195abb7d 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -132,7 +132,7 @@ server { include /etc/nginx/conf.d/include/proxy-cache-downloads; limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time - add_header Cache-Control "public, max-age=86400"; # allow consumer to cache response + add_header Cache-Control "public, max-age=300"; # allow consumer to cache response just for 5 minutes set $skylink ''; set $skylink_path ''; From 54cda20b3210bffd536f8bee72d990d978227ca0 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 16 Sep 2020 14:21:22 +0200 Subject: [PATCH 4/6] adjust hns endpoint --- docker/nginx/conf.d/client.conf | 35 ++++++++++++++------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 195abb7d..6c9ce836 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -9,6 +9,7 @@ limit_conn_status 429; # as the request address so we need to use real_ip_header module to use ip address from # X-Forwarded-For header as a real ip address of the request set_real_ip_from 10.0.0.0/8; +set_real_ip_from 127.0.0.1/32; set_real_ip_from 172.16.0.0/12; set_real_ip_from 192.168.0.0/16; real_ip_header X-Forwarded-For; @@ -127,42 +128,36 @@ server { } location /hns { - include /etc/nginx/conf.d/include/cors; include /etc/nginx/conf.d/include/proxy-buffer; - include /etc/nginx/conf.d/include/proxy-cache-downloads; - - limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time - add_header Cache-Control "public, max-age=300"; # allow consumer to cache response just for 5 minutes set $skylink ''; - set $skylink_path ''; + set $rest ''; access_by_lua_block { local json = require('cjson') - local hns_domain_name, request_uri_path = string.match(ngx.var.request_uri, "/hns/([^/]+)(.*)") + local hns_domain_name, request_uri_rest = string.match(ngx.var.request_uri, "/hns/([^/?]+)(.*)") local hnsres_res = ngx.location.capture("/hnsres/" .. hns_domain_name) local hnsres_json = json.decode(hnsres_res.body) - local skylink_prefix, skylink, skylink_path = string.match(hnsres_json.skylink, "(sia://)([^/]+)(.*)") + local skylink_prefix, skylink, skylink_rest = string.match(hnsres_json.skylink, "(sia://)([^/?]+)(.*)") ngx.var.skylink = skylink - if request_uri_path == "" or (request_uri_path == "/" and skylink_path ~= "") then - ngx.var.skylink_path = skylink_path + if request_uri_rest == "" or (request_uri_rest == "/" and skylink_rest ~= "") then + ngx.var.rest = skylink_rest else - ngx.var.skylink_path = request_uri_path + ngx.var.rest = request_uri_rest end } - proxy_read_timeout 600; - proxy_set_header User-Agent: Sia-Agent; - - proxy_pass http://siad/skynet/skylink/$skylink$skylink_path; + # overwrite the Cache-Control header to only cache for 60s in case the domain gets updated + more_set_headers 'Cache-Control: public, max-age=60'; + + proxy_pass http://127.0.0.1/$skylink$rest; header_filter_by_lua_block { - if ngx.header.Location then - local json = require('cjson') - local hns_domain_name = string.match(ngx.var.request_uri, "/hns/(.+)/?") - local location = string.gsub(ngx.header.Location, ngx.var.skylink, hns_domain_name) + if ngx.header.location then + local hns_domain_name = string.match(ngx.var.request_uri, "/hns/([^/?]+)") + local location = string.gsub(ngx.header.location, ngx.var.skylink, hns_domain_name) - ngx.header.Location = location + ngx.header.location = location end } } From 936d4ecd0c5fc110d63032af1a65627a2d9419a6 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 16 Sep 2020 14:37:14 +0200 Subject: [PATCH 5/6] add more comments --- docker/nginx/conf.d/client.conf | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 6c9ce836..fa047b70 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -130,8 +130,10 @@ server { location /hns { include /etc/nginx/conf.d/include/proxy-buffer; - set $skylink ''; - set $rest ''; + set $skylink ''; # placeholder for the raw 46 bit skylink + set $rest ''; # placeholder for the rest of the url that gets appended to skylink (path and args) + + # resolve handshake domain by requesting to /hnsres endpoint and assign correct values to $skylink and $rest access_by_lua_block { local json = require('cjson') local hns_domain_name, request_uri_rest = string.match(ngx.var.request_uri, "/hns/([^/?]+)(.*)") @@ -150,8 +152,10 @@ server { # overwrite the Cache-Control header to only cache for 60s in case the domain gets updated more_set_headers 'Cache-Control: public, max-age=60'; + # we proxy to another nginx location rather than directly to siad because we don't want to deal with caching here proxy_pass http://127.0.0.1/$skylink$rest; + # in case siad returns location header, we need to replace the skylink with the domain name header_filter_by_lua_block { if ngx.header.location then local hns_domain_name = string.match(ngx.var.request_uri, "/hns/([^/?]+)") From 8b883310654061a68bff3bc29632c0c963652b5b Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 16 Sep 2020 17:06:09 +0200 Subject: [PATCH 6/6] drop 404 not found on hnsres error --- docker/nginx/conf.d/client.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index fa047b70..34b9dafb 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -138,6 +138,11 @@ server { local json = require('cjson') local hns_domain_name, request_uri_rest = string.match(ngx.var.request_uri, "/hns/([^/?]+)(.*)") local hnsres_res = ngx.location.capture("/hnsres/" .. hns_domain_name) + + if hnsres_res.status ~= ngx.HTTP_OK then + ngx.exit(ngx.HTTP_NOT_FOUND) + end + local hnsres_json = json.decode(hnsres_res.body) local skylink_prefix, skylink, skylink_rest = string.match(hnsres_json.skylink, "(sia://)([^/?]+)(.*)")