From 06f0946317d78b81d4a4d984a866a7913334d9e0 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 31 Jul 2020 15:25:57 +0200 Subject: [PATCH 01/18] fix hns header overflow and redirects --- docker/handshake-api/Dockerfile | 2 +- docker/nginx/conf.d/client.conf | 11 +++-------- docker/nginx/conf.d/include/proxy-buffer | 4 ++++ handshake-api/index.js | 10 ++++------ 4 files changed, 12 insertions(+), 15 deletions(-) create mode 100644 docker/nginx/conf.d/include/proxy-buffer diff --git a/docker/handshake-api/Dockerfile b/docker/handshake-api/Dockerfile index 22826b4e..c7a393b3 100644 --- a/docker/handshake-api/Dockerfile +++ b/docker/handshake-api/Dockerfile @@ -17,4 +17,4 @@ ENV HSD_API_KEY="foo" EXPOSE $PORT -ENTRYPOINT ["node", "index.js"] +ENTRYPOINT ["node", "--max-http-header-size=64000", "index.js"] diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 6dca374b..bfa29563 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -114,6 +114,7 @@ server { location /hns { include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/proxy-buffer; proxy_pass http://handshake-api:3100; } @@ -173,6 +174,7 @@ server { location ~ "^/([a-zA-Z0-9-_]{46}(/.*)?)$" { include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/proxy-buffer; limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time @@ -184,10 +186,6 @@ server { proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) proxy_pass http://siad/skynet/skylink/$skylink$is_args$args; - - # if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs - proxy_buffer_size 128k; - proxy_buffers 4 128k; # cache frequent (> 10) downloads for 24 hours proxy_cache skynet; @@ -199,6 +197,7 @@ server { location ~ "^/file/([a-zA-Z0-9-_]{46}(/.*)?)$" { include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/proxy-buffer; limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time @@ -211,10 +210,6 @@ server { # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) # this alias also adds attachment=true url param to force download the file proxy_pass http://siad/skynet/skylink/$skylink?attachment=true&$args; - - # if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs - proxy_buffer_size 128k; - proxy_buffers 4 128k; # cache frequent (> 10) downloads for 24 hours proxy_cache skynet; diff --git a/docker/nginx/conf.d/include/proxy-buffer b/docker/nginx/conf.d/include/proxy-buffer new file mode 100644 index 00000000..aea687fb --- /dev/null +++ b/docker/nginx/conf.d/include/proxy-buffer @@ -0,0 +1,4 @@ +# if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs +proxy_buffer_size 128k; +proxy_buffers 4 256k; +proxy_busy_buffers_size 256k; \ No newline at end of file diff --git a/handshake-api/index.js b/handshake-api/index.js index 07282e35..6c00d2f4 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -85,14 +85,12 @@ server.use( const basepath = url.resolve("/", skylink); // make the url absolute const subpath = req.url.slice(1); // drop the leading slash - // if the skylink from handshake does not contain a subpath but subpath - // is defined in request, join the skylink and subpath together (do not - // use url.resolve because it will replace skylink with subapth thinking - // it is relative) - if (skylink.length === 46 && subpath) { - return `${basepath}/${subpath}`; + // if the record is just a raw skylink, replace baseUrl with /skylink + if (skylink.length === 46) { + return req.originalUrl.replace(req.baseUrl, basepath); } + // if the record contains more than a skylink then it needs to be resolved return url.resolve(basepath, subpath); }, }) From 966b2e82a0af0f0238729056500addfa166d727b Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Sat, 1 Aug 2020 17:04:23 +0200 Subject: [PATCH 02/18] fix up docker-compose --- docker-compose.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0e617ef6..ff60306c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,6 +55,8 @@ services: - 80 depends_on: - docker-host + - health-check + - handshake-api handshake: build: @@ -83,6 +85,7 @@ services: container_name: handshake-api restart: unless-stopped environment: + - HOSTNAME=0.0.0.0 - HSD_HOST=handshake - HSD_NETWORK=main - HSD_PORT=12037 @@ -94,7 +97,6 @@ services: - 3100 depends_on: - handshake - - nginx health-check: build: @@ -108,9 +110,9 @@ services: networks: - shared environment: + - HOSTNAME=0.0.0.0 - PORTAL_URL=nginx expose: - 3100 depends_on: - docker-host - - nginx From 4c46465e4608779426f9161a39a9b3ca2ce2e35b Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Tue, 4 Aug 2020 19:04:06 +0300 Subject: [PATCH 03/18] Add a hard 307 redirect. --- handshake-api/index.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/handshake-api/index.js b/handshake-api/index.js index 6c00d2f4..d64f353c 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -68,6 +68,9 @@ server.use( proxy("nginx", { // eslint-disable-next-line no-unused-vars userResHeaderDecorator(headers, userReq, userRes, proxyReq, proxyRes) { + if (!userReq.path.endsWith("/")) { + userRes.redirect(307, `${userReq.path}/`); + } if (headers.location && headers.location.match(startsWithSkylinkRegExp)) { headers.location = headers.location.replace(startsWithSkylinkRegExp, `/hns/${userReq.params.name}`); } @@ -96,6 +99,20 @@ server.use( }) ); +// server.get("/hns/:name", async (req, res) => { +// if (!req.path.endsWith('/')) { +// res.redirect(307, `${req.path}/`); +// } +// console.log(" > ", req.path) +// if (req.headers.location && req.headers.location.match(startsWithSkylinkRegExp)) { +// req.headers.location = req.headers.location.replace(startsWithSkylinkRegExp, `/hns/${userReq.params.name}`); +// console.log(" > to ", req.headers.location) +// } +// +// // res.redirect(307, '/to'); +// +// return req.headers; +// }); server.get("/hnsres/:name", resolveDomainHandler); server.listen(port, host, (error) => { From b12c1674e4dca22749123f2ac59828d43e22444c Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Tue, 4 Aug 2020 19:39:25 +0300 Subject: [PATCH 04/18] Revert "Add a hard 307 redirect." This reverts commit 4c46465e4608779426f9161a39a9b3ca2ce2e35b. --- handshake-api/index.js | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/handshake-api/index.js b/handshake-api/index.js index d64f353c..6c00d2f4 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -68,9 +68,6 @@ server.use( proxy("nginx", { // eslint-disable-next-line no-unused-vars userResHeaderDecorator(headers, userReq, userRes, proxyReq, proxyRes) { - if (!userReq.path.endsWith("/")) { - userRes.redirect(307, `${userReq.path}/`); - } if (headers.location && headers.location.match(startsWithSkylinkRegExp)) { headers.location = headers.location.replace(startsWithSkylinkRegExp, `/hns/${userReq.params.name}`); } @@ -99,20 +96,6 @@ server.use( }) ); -// server.get("/hns/:name", async (req, res) => { -// if (!req.path.endsWith('/')) { -// res.redirect(307, `${req.path}/`); -// } -// console.log(" > ", req.path) -// if (req.headers.location && req.headers.location.match(startsWithSkylinkRegExp)) { -// req.headers.location = req.headers.location.replace(startsWithSkylinkRegExp, `/hns/${userReq.params.name}`); -// console.log(" > to ", req.headers.location) -// } -// -// // res.redirect(307, '/to'); -// -// return req.headers; -// }); server.get("/hnsres/:name", resolveDomainHandler); server.listen(port, host, (error) => { From f896afada1ee529ac0e6c948714b469b01ae5fc1 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 5 Aug 2020 11:27:25 +0300 Subject: [PATCH 05/18] Make health-check depenent on handshake, handshake-api, and caddy, so it doesn't start before them and report healthy status. --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index ff60306c..71d0671f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -116,3 +116,6 @@ services: - 3100 depends_on: - docker-host + - handshake + - handshake-api + - caddy From 98ec6f5c1caf585dcf03e9d88da8b2dee42a2506 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 5 Aug 2020 13:35:31 +0300 Subject: [PATCH 06/18] Support both bare (no leading `sia://`) and full skylinks. --- handshake-api/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/handshake-api/index.js b/handshake-api/index.js index 6c00d2f4..f92ccf05 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -19,7 +19,8 @@ const clientOptions = { }; const client = new NodeClient(clientOptions); -const startsWithSkylinkRegExp = /^[a-zA-Z0-9_-]{46}/; +// Match both `sia://HASH` and `HASH` links. +const startsWithSkylinkRegExp = /^(sia:\/\/){0,1}[a-zA-Z0-9_-]{46}/; const getDomainRecords = async (name) => { const response = await client.execute("getnameresource", [name]); From 4e37707a07120a126a8b057d43bc8fa54f2df7a9 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 5 Aug 2020 14:42:10 +0300 Subject: [PATCH 07/18] Add a hard 307 redirect. --- handshake-api/index.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/handshake-api/index.js b/handshake-api/index.js index f92ccf05..896514fb 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -69,6 +69,9 @@ server.use( proxy("nginx", { // eslint-disable-next-line no-unused-vars userResHeaderDecorator(headers, userReq, userRes, proxyReq, proxyRes) { + if (!userReq.path.endsWith("/")) { + userRes.redirect(307, `${userReq.path}/`); + } if (headers.location && headers.location.match(startsWithSkylinkRegExp)) { headers.location = headers.location.replace(startsWithSkylinkRegExp, `/hns/${userReq.params.name}`); } From 2b31f6c34a0d9f7702df2acd7ead270421af4c38 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 5 Aug 2020 14:47:15 +0300 Subject: [PATCH 08/18] Support only `sia://HASH` skylinks. --- handshake-api/index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/handshake-api/index.js b/handshake-api/index.js index 896514fb..24f7347a 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -19,8 +19,7 @@ const clientOptions = { }; const client = new NodeClient(clientOptions); -// Match both `sia://HASH` and `HASH` links. -const startsWithSkylinkRegExp = /^(sia:\/\/){0,1}[a-zA-Z0-9_-]{46}/; +const startsWithSkylinkRegExp = /^sia:\/\/[a-zA-Z0-9_-]{46}/; const getDomainRecords = async (name) => { const response = await client.execute("getnameresource", [name]); From 90fa86b0f8e538557f204cb81cd8f0b5429bd874 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 5 Aug 2020 14:55:19 +0300 Subject: [PATCH 09/18] Bump GitHub Actions' Node version to 14.6 to match the version we're using for the services. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 66f0cfcf..909abc86 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ jobs: - name: Use Node.js uses: actions/setup-node@v1 with: - node-version: 12.x + node-version: 14.6 - name: Install dependencies run: yarn From 9dd73cd2aac1aca3753700c80d85f02846dc45ab Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 5 Aug 2020 17:02:42 +0300 Subject: [PATCH 10/18] Resolve a circular dependency between docker-compose services. --- docker-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 71d0671f..e1748e85 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,7 +55,6 @@ services: - 80 depends_on: - docker-host - - health-check - handshake-api handshake: @@ -115,7 +114,6 @@ services: expose: - 3100 depends_on: - - docker-host + - caddy - handshake - handshake-api - - caddy From 651935ea3aa545e71dd508a58a7287d0564905b0 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 5 Aug 2020 17:20:29 +0300 Subject: [PATCH 11/18] Bring back the support for bare skylinks. --- handshake-api/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/handshake-api/index.js b/handshake-api/index.js index 24f7347a..896514fb 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -19,7 +19,8 @@ const clientOptions = { }; const client = new NodeClient(clientOptions); -const startsWithSkylinkRegExp = /^sia:\/\/[a-zA-Z0-9_-]{46}/; +// Match both `sia://HASH` and `HASH` links. +const startsWithSkylinkRegExp = /^(sia:\/\/){0,1}[a-zA-Z0-9_-]{46}/; const getDomainRecords = async (name) => { const response = await client.execute("getnameresource", [name]); From 6f408e49c93de7c4e9c6b68328997f23bee307ff Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 5 Aug 2020 17:32:51 +0300 Subject: [PATCH 12/18] Get the last available skylink. --- handshake-api/index.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/handshake-api/index.js b/handshake-api/index.js index 896514fb..63cd75d7 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -32,7 +32,13 @@ const getDomainRecords = async (name) => { }; const findSkylinkRecord = (records) => { - return records?.find(({ txt }) => txt?.some((entry) => isValidSkylink(entry))); + // Find the last one, so people can update their domains in a non-destructive + // way by simply adding a new link. This will also allow keeping links to + // older versions for backwards compatibility. + return records + ?.slice() + .reverse() + .find(({ txt }) => txt?.some((entry) => isValidSkylink(entry))); }; const getSkylinkFromRecord = (record) => { From 87a3cb8aa74162131036de1ecdb7801388d1acc9 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 5 Aug 2020 17:40:16 +0300 Subject: [PATCH 13/18] Strip the `sia://` prefix. --- handshake-api/index.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/handshake-api/index.js b/handshake-api/index.js index 63cd75d7..f1cf23db 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -79,7 +79,10 @@ server.use( userRes.redirect(307, `${userReq.path}/`); } if (headers.location && headers.location.match(startsWithSkylinkRegExp)) { - headers.location = headers.location.replace(startsWithSkylinkRegExp, `/hns/${userReq.params.name}`); + headers.location = headers.location.replace( + startsWithSkylinkRegExp, + `/hns/${userReq.params.name.replace("sia://", "")}` + ); } return headers; From 71d2b3dc9140526665ce9be34a062653add43e34 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 5 Aug 2020 17:55:49 +0300 Subject: [PATCH 14/18] Fix the redirect. --- handshake-api/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handshake-api/index.js b/handshake-api/index.js index f1cf23db..41dad687 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -75,8 +75,8 @@ server.use( proxy("nginx", { // eslint-disable-next-line no-unused-vars userResHeaderDecorator(headers, userReq, userRes, proxyReq, proxyRes) { - if (!userReq.path.endsWith("/")) { - userRes.redirect(307, `${userReq.path}/`); + if (!userReq.url.endsWith("/")) { + userRes.redirect(307, `${userReq.url}/`); } if (headers.location && headers.location.match(startsWithSkylinkRegExp)) { headers.location = headers.location.replace( From 8ee13bb96c7172879099aadf3c4cf586e1b82119 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 5 Aug 2020 18:18:36 +0300 Subject: [PATCH 15/18] Remove the problematic redirect. --- handshake-api/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/handshake-api/index.js b/handshake-api/index.js index 41dad687..452d50c7 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -75,9 +75,9 @@ server.use( proxy("nginx", { // eslint-disable-next-line no-unused-vars userResHeaderDecorator(headers, userReq, userRes, proxyReq, proxyRes) { - if (!userReq.url.endsWith("/")) { - userRes.redirect(307, `${userReq.url}/`); - } + // if (!userReq.url.endsWith('/')) { + // userRes.redirect(307, `${userReq.url}/`); + // } if (headers.location && headers.location.match(startsWithSkylinkRegExp)) { headers.location = headers.location.replace( startsWithSkylinkRegExp, From f3d918d7e299fa2e4e0608d649eaa34ae55f5f8f Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Fri, 7 Aug 2020 13:03:32 +0300 Subject: [PATCH 16/18] Simplify regex. Remove dead code. --- .gitignore | 1 + handshake-api/index.js | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 43f25aa7..6a4f2eb0 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,4 @@ docker/data # Cache files __pycache__ /.idea/ +/venv/ diff --git a/handshake-api/index.js b/handshake-api/index.js index 452d50c7..87b5a09a 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -20,7 +20,7 @@ const clientOptions = { const client = new NodeClient(clientOptions); // Match both `sia://HASH` and `HASH` links. -const startsWithSkylinkRegExp = /^(sia:\/\/){0,1}[a-zA-Z0-9_-]{46}/; +const startsWithSkylinkRegExp = /^(sia:\/\/)?[a-zA-Z0-9_-]{46}/; const getDomainRecords = async (name) => { const response = await client.execute("getnameresource", [name]); @@ -75,9 +75,6 @@ server.use( proxy("nginx", { // eslint-disable-next-line no-unused-vars userResHeaderDecorator(headers, userReq, userRes, proxyReq, proxyRes) { - // if (!userReq.url.endsWith('/')) { - // userRes.redirect(307, `${userReq.url}/`); - // } if (headers.location && headers.location.match(startsWithSkylinkRegExp)) { headers.location = headers.location.replace( startsWithSkylinkRegExp, From 69abeee7074efeb9a940f00c3f19b7f5e533dcc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Wypch=C5=82o?= Date: Mon, 17 Aug 2020 09:50:37 +0200 Subject: [PATCH 17/18] Update proxy-buffer --- docker/nginx/conf.d/include/proxy-buffer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/include/proxy-buffer b/docker/nginx/conf.d/include/proxy-buffer index aea687fb..f7a2b12e 100644 --- a/docker/nginx/conf.d/include/proxy-buffer +++ b/docker/nginx/conf.d/include/proxy-buffer @@ -1,4 +1,4 @@ # if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs proxy_buffer_size 128k; proxy_buffers 4 256k; -proxy_busy_buffers_size 256k; \ No newline at end of file +proxy_busy_buffers_size 256k; From 2ba0159289adac8639a3ef6146be5f56aa64850d Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 17 Aug 2020 12:16:34 +0200 Subject: [PATCH 18/18] adjust for sia:// prefixed skylinks --- handshake-api/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handshake-api/index.js b/handshake-api/index.js index 87b5a09a..6c057ca1 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -91,7 +91,7 @@ server.use( const record = findSkylinkRecord(records); if (!record) throw new Error(`No skylink found in dns records of ${req.params.name}`); - const skylink = getSkylinkFromRecord(record); + const skylink = getSkylinkFromRecord(record).replace("sia://", ""); // get skylink and strip sia:// prefix const basepath = url.resolve("/", skylink); // make the url absolute const subpath = req.url.slice(1); // drop the leading slash