From 13853540740d7ee46923b2d13950f832d5aaffdd Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 25 Jun 2021 14:05:36 +0200 Subject: [PATCH 1/5] dnslink beta support --- .github/dependabot.yml | 8 +++ .github/workflows/static-code-analysis.yml | 3 + docker-compose.yml | 14 ++++ docker/caddy/Caddyfile | 6 +- docker/nginx/conf.d/client.conf | 31 +++++++++ packages/dnslink-api/.prettierignore | 1 + packages/dnslink-api/.prettierrc | 3 + packages/dnslink-api/Dockerfile | 10 +++ packages/dnslink-api/package.json | 14 ++++ packages/dnslink-api/src/index.js | 78 ++++++++++++++++++++++ 10 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 packages/dnslink-api/.prettierignore create mode 100644 packages/dnslink-api/.prettierrc create mode 100644 packages/dnslink-api/Dockerfile create mode 100644 packages/dnslink-api/package.json create mode 100644 packages/dnslink-api/src/index.js diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e7b5a6d5..f5ca2228 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,6 +8,14 @@ updates: open-pull-requests-limit: 10 assignees: - kwypchlo + - package-ecosystem: npm + directory: "/packages/dnslink-api" + schedule: + interval: weekly + time: "10:00" + open-pull-requests-limit: 10 + assignees: + - kwypchlo - package-ecosystem: npm directory: "/packages/handshake-api" schedule: diff --git a/.github/workflows/static-code-analysis.yml b/.github/workflows/static-code-analysis.yml index 08d0cb5d..b216b446 100644 --- a/.github/workflows/static-code-analysis.yml +++ b/.github/workflows/static-code-analysis.yml @@ -26,3 +26,6 @@ jobs: - name: "Static code analysis: dashboard" run: yarn workspace dashboard prettier --check . + + - name: "Static code analysis: dnslink-api" + run: yarn workspace dnslink-api prettier --check . diff --git a/docker-compose.yml b/docker-compose.yml index 416f4e92..c72eb356 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,6 +81,7 @@ services: depends_on: - sia - handshake-api + - dnslink-api - website website: @@ -142,6 +143,19 @@ services: depends_on: - handshake + dnslink-api: + build: + context: ./packages/dnslink-api + dockerfile: Dockerfile + container_name: dnslink-api + restart: unless-stopped + logging: *default-logging + networks: + shared: + ipv4_address: 10.10.10.55 + expose: + - 3100 + health-check: build: context: ./packages/health-check diff --git a/docker/caddy/Caddyfile b/docker/caddy/Caddyfile index 28c9754d..a2f53797 100644 --- a/docker/caddy/Caddyfile +++ b/docker/caddy/Caddyfile @@ -7,7 +7,11 @@ on_demand } - reverse_proxy nginx:80 + reverse_proxy nginx:80 { + # add Dnslink-Lookup header so nginx knows that the request comes from a domain + # outside of our certificate string and should perform a dnslink lookup + header_up Dnslink-Lookup true + } } # Make sure you have SSL_CERTIFICATE_STRING specified in .env file because you need it to fetch correct certificates. diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 7316f03b..8a9374a3 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -88,6 +88,12 @@ server { return 461; } + # redirect to dnslookup endpoint + error_page 462 = @dnslink_lookup; + if ($http_dnslink_lookup) { + return 462; + } + location / { include /etc/nginx/conf.d/include/cors; @@ -444,6 +450,31 @@ server { proxy_pass http://siad/skynet/skylink/$skylink_v1$path$is_args$args; } + location @dnslink_lookup { + include /etc/nginx/conf.d/include/proxy-buffer; + + set $dnslink ''; + + rewrite_by_lua_block { + local http = require("socket.http") + local ok, statusCode, headers, statusText = http.request { + url = "http://dnslink-api:3100/dnslink/" .. ngx.var.host + } + + if statusCode == ngx.HTTP_OK then + ngx.var.dnslink = headers["skynet-skylink"] + else + ngx.status = statusCode + ngx.header["content-type"] = "text/plain" + ngx.say(headers["dnslink-error"]) + ngx.exit(statusCode) + end + } + + proxy_set_header Dnslink-Lookup ""; + proxy_pass http://127.0.0.1/$dnslink/$request_uri; + } + location @base32_subdomain { include /etc/nginx/conf.d/include/proxy-buffer; diff --git a/packages/dnslink-api/.prettierignore b/packages/dnslink-api/.prettierignore new file mode 100644 index 00000000..00c008c9 --- /dev/null +++ b/packages/dnslink-api/.prettierignore @@ -0,0 +1 @@ +/package.json diff --git a/packages/dnslink-api/.prettierrc b/packages/dnslink-api/.prettierrc new file mode 100644 index 00000000..963354f2 --- /dev/null +++ b/packages/dnslink-api/.prettierrc @@ -0,0 +1,3 @@ +{ + "printWidth": 120 +} diff --git a/packages/dnslink-api/Dockerfile b/packages/dnslink-api/Dockerfile new file mode 100644 index 00000000..39b3980b --- /dev/null +++ b/packages/dnslink-api/Dockerfile @@ -0,0 +1,10 @@ +FROM node:16.3.0-alpine + +WORKDIR /usr/app + +COPY package.json . +RUN yarn --no-lockfile +COPY src/* src/ + +EXPOSE 3100 +CMD node src diff --git a/packages/dnslink-api/package.json b/packages/dnslink-api/package.json new file mode 100644 index 00000000..804fc988 --- /dev/null +++ b/packages/dnslink-api/package.json @@ -0,0 +1,14 @@ +{ + "name": "dnslink-api", + "version": "1.0.0", + "main": "index.js", + "license": "SEE LICENSE IN LICENSE.md", + "dependencies": { + "express": "^4.17.1", + "is-valid-domain": "^0.0.20", + "node-cache": "^5.1.2" + }, + "devDependencies": { + "prettier": "^2.2.1" + } +} diff --git a/packages/dnslink-api/src/index.js b/packages/dnslink-api/src/index.js new file mode 100644 index 00000000..7f81b04c --- /dev/null +++ b/packages/dnslink-api/src/index.js @@ -0,0 +1,78 @@ +const dns = require("dns"); +const express = require("express"); +const NodeCache = require("node-cache"); +const isValidDomain = require("is-valid-domain"); + +const host = process.env.HOSTNAME || "0.0.0.0"; +const port = Number(process.env.PORT) || 3100; + +const server = express(); +const cache = new NodeCache({ stdTTL: 300 }); // cache for 5 minutes + +const dnslinkRegExp = /^dnslink=.+$/; +const skylinkDnslinkRegExp = /^dnslink=\/skynet-ns\/([a-zA-Z0-9_-]{46}|[a-z0-9]{55})/; +const hint = "valid example: dnslink=/skynet-ns/3ACpC9Umme41zlWUgMQh1fw0sNwgWwyfDDhRQ9Sppz9hjQ"; + +server.get("/dnslink/:name", async (req, res) => { + const success = (skylink) => res.set("Skynet-Skylink", skylink).send(skylink); + const failure = (message) => res.status(400).set("Dnslink-Error", message).send(message); + + if (!isValidDomain(req.params.name)) { + return failure(`"${req.params.name}" is not a valid domain`); + } + + if (cache.has(req.params.name)) { + return success(cache.get(req.params.name)); + } + + const lookup = `_dnslink.${req.params.name}`; + + dns.resolveTxt(lookup, (error, records) => { + if (error) { + if (error.code === "ENOTFOUND") { + return failure(`ENOTFOUND: ${lookup} TXT record doesn't exist`); + } + + if (error.code === "ENODATA") { + return failure(`ENODATA: ${lookup} dns lookup returned no data`); + } + + return failure(`Failed to fetch ${lookup} TXT record: ${error.message}`); + } + + if (records.length === 0) { + return failure(`No TXT record found for ${lookup}`); + } + + const dnslinks = records.flat().filter((record) => dnslinkRegExp.test(record)); + + if (dnslinks.length === 0) { + return failure(`TXT records for ${lookup} found but none of them contained valid dnslink - ${hint}`); + } + + if (dnslinks.length > 1) { + return failure(`Multiple TXT records with valid dnslink found for ${lookup}, only one allowed`); + } + + const [dnslink] = dnslinks; + const matchSkylink = dnslink.match(skylinkDnslinkRegExp); + + if (!matchSkylink) { + return failure(`TXT record with dnslink for ${lookup} contains invalid skylink - ${hint}`); + } + + const skylink = matchSkylink[1]; + + cache.set(req.params.name, skylink); + + console.log(`${req.params.name} => ${skylink}`); + + return success(skylink); + }); +}); + +server.listen(port, host, (error) => { + if (error) throw error; + + console.info(`Server listening at http://${host}:${port}`); +}); From e86abc3438a292fea844d9a7f061e7bf5d41d91c Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 7 Jul 2021 16:02:58 +0200 Subject: [PATCH 2/5] serve dnslink over http --- docker/caddy/Caddyfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/caddy/Caddyfile b/docker/caddy/Caddyfile index a2f53797..8483182e 100644 --- a/docker/caddy/Caddyfile +++ b/docker/caddy/Caddyfile @@ -7,6 +7,10 @@ on_demand } + reverse_proxy nginx:80 +} + +:80 { reverse_proxy nginx:80 { # add Dnslink-Lookup header so nginx knows that the request comes from a domain # outside of our certificate string and should perform a dnslink lookup From ee475de13b1d230c1e7b5ae90906e4100aef8e11 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 7 Jul 2021 23:29:10 +0200 Subject: [PATCH 3/5] disable :443 --- docker/caddy/Caddyfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/caddy/Caddyfile b/docker/caddy/Caddyfile index 8483182e..e78040dc 100644 --- a/docker/caddy/Caddyfile +++ b/docker/caddy/Caddyfile @@ -2,13 +2,13 @@ # It is useful in case you have services trying to reach the server through ip and not domain like health checks. # It will generate an internal certificate so browsers will warn you when connecting but that not a problem. -:443 { - tls internal { - on_demand - } - - reverse_proxy nginx:80 -} +# :443 { +# tls internal { +# on_demand +# } +# +# reverse_proxy nginx:80 +# } :80 { reverse_proxy nginx:80 { From c9a13cab8ec18f03b90a4e04d1c1b01670f5a021 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Wed, 7 Jul 2021 23:31:14 +0200 Subject: [PATCH 4/5] do not disable :443 --- docker/caddy/Caddyfile | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docker/caddy/Caddyfile b/docker/caddy/Caddyfile index e78040dc..db49d4a5 100644 --- a/docker/caddy/Caddyfile +++ b/docker/caddy/Caddyfile @@ -2,13 +2,17 @@ # It is useful in case you have services trying to reach the server through ip and not domain like health checks. # It will generate an internal certificate so browsers will warn you when connecting but that not a problem. -# :443 { -# tls internal { -# on_demand -# } -# -# reverse_proxy nginx:80 -# } +:443 { + tls internal { + on_demand + } + + reverse_proxy nginx:80 { + # add Dnslink-Lookup header so nginx knows that the request comes from a domain + # outside of our certificate string and should perform a dnslink lookup + header_up Dnslink-Lookup true + } +} :80 { reverse_proxy nginx:80 { From 7a135ed5ed101c9bc5d5b4b3e962f041054997a4 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Thu, 8 Jul 2021 14:52:39 +0200 Subject: [PATCH 5/5] allow multiple dnslinks with different namespaces --- packages/dnslink-api/src/index.js | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/dnslink-api/src/index.js b/packages/dnslink-api/src/index.js index 7f81b04c..93e5cdeb 100644 --- a/packages/dnslink-api/src/index.js +++ b/packages/dnslink-api/src/index.js @@ -3,15 +3,17 @@ const express = require("express"); const NodeCache = require("node-cache"); const isValidDomain = require("is-valid-domain"); -const host = process.env.HOSTNAME || "0.0.0.0"; -const port = Number(process.env.PORT) || 3100; +const host = process.env.DNSLINK_API_HOSTNAME || "0.0.0.0"; +const port = Number(process.env.DNSLINK_API_PORT) || 3100; +const cacheTTL = Number(process.env.DNSLINK_API_CACHE_TTL) || 300; // default to 5 minutes const server = express(); -const cache = new NodeCache({ stdTTL: 300 }); // cache for 5 minutes +const cache = new NodeCache({ stdTTL: cacheTTL }); -const dnslinkRegExp = /^dnslink=.+$/; -const skylinkDnslinkRegExp = /^dnslink=\/skynet-ns\/([a-zA-Z0-9_-]{46}|[a-z0-9]{55})/; -const hint = "valid example: dnslink=/skynet-ns/3ACpC9Umme41zlWUgMQh1fw0sNwgWwyfDDhRQ9Sppz9hjQ"; +const dnslinkNamespace = "skynet-ns"; +const dnslinkRegExp = new RegExp(`^dnslink=/${dnslinkNamespace}/.+$`); +const dnslinkSkylinkRegExp = new RegExp(`^dnslink=/${dnslinkNamespace}/([a-zA-Z0-9_-]{46}|[a-z0-9]{55})`); +const hint = `valid example: dnslink=/${dnslinkNamespace}/3ACpC9Umme41zlWUgMQh1fw0sNwgWwyfDDhRQ9Sppz9hjQ`; server.get("/dnslink/:name", async (req, res) => { const success = (skylink) => res.set("Skynet-Skylink", skylink).send(skylink); @@ -47,18 +49,18 @@ server.get("/dnslink/:name", async (req, res) => { const dnslinks = records.flat().filter((record) => dnslinkRegExp.test(record)); if (dnslinks.length === 0) { - return failure(`TXT records for ${lookup} found but none of them contained valid dnslink - ${hint}`); + return failure(`TXT records for ${lookup} found but none of them contained valid skynet dnslink - ${hint}`); } if (dnslinks.length > 1) { - return failure(`Multiple TXT records with valid dnslink found for ${lookup}, only one allowed`); + return failure(`Multiple TXT records with valid skynet dnslink found for ${lookup}, only one allowed`); } const [dnslink] = dnslinks; - const matchSkylink = dnslink.match(skylinkDnslinkRegExp); + const matchSkylink = dnslink.match(dnslinkSkylinkRegExp); if (!matchSkylink) { - return failure(`TXT record with dnslink for ${lookup} contains invalid skylink - ${hint}`); + return failure(`TXT record with skynet dnslink for ${lookup} contains invalid skylink - ${hint}`); } const skylink = matchSkylink[1];