From f1346f23ada3bce7b414e68cfb2c275112e9bdbc Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 20 Sep 2021 22:20:48 +0200 Subject: [PATCH 1/8] expose skylink endpoint over external domains --- docker/nginx/conf.d/include/location-hns | 2 +- docker/nginx/conf.d/include/location-skylink | 4 ++-- docker/nginx/conf.d/server/server.dnslink | 21 ++++++++++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docker/nginx/conf.d/include/location-hns b/docker/nginx/conf.d/include/location-hns index bd5644fd..8cc662da 100644 --- a/docker/nginx/conf.d/include/location-hns +++ b/docker/nginx/conf.d/include/location-hns @@ -75,7 +75,7 @@ access_by_lua_block { end } -# we proxy to another nginx location rather than directly to siad because we don't want to deal with caching here +# we proxy to another nginx location rather than directly to siad because we do not want to deal with caching here proxy_pass https://127.0.0.1/$skylink$path$is_args$args; # in case siad returns location header, we need to replace the skylink with the domain name diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index 96dfacef..a3692d00 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -12,7 +12,7 @@ if ($request_method = PURGE) { limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time # $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was: -# a) skylink v1 - it wouldn't matter, no additional logic is executed +# a) skylink v1 - it would not matter, no additional logic is executed # b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update # $skylink_v1 variable so then the proxy request to skyd can be cached in nginx (proxy_cache_key # in proxy-cache-downloads includes $skylink_v1 as a part of the cache key) @@ -91,7 +91,7 @@ limit_rate $limit_rate; proxy_read_timeout 600; proxy_set_header User-Agent: Sia-Agent; -# in case the requested skylink was v2 and we already resolved it to skylink v1, we're going to pass resolved +# in case the requested skylink was v2 and we already resolved it to skylink v1, we are going to pass resolved # skylink v1 to skyd to save that extra skylink v2 lookup in skyd but in turn, in case skyd returns a redirect # we need to rewrite the skylink v1 to skylink v2 in the location header with proxy_redirect proxy_redirect $skylink_v1 $skylink_v2; diff --git a/docker/nginx/conf.d/server/server.dnslink b/docker/nginx/conf.d/server/server.dnslink index db8948e3..e04c5a7a 100644 --- a/docker/nginx/conf.d/server/server.dnslink +++ b/docker/nginx/conf.d/server/server.dnslink @@ -11,15 +11,24 @@ location / { local res, err = httpc:request_uri("http://10.10.10.55:3100/dnslink/" .. ngx.var.host) if err or (res and res.status ~= ngx.HTTP_OK) then - ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status - ngx.header["content-type"] = "text/plain" - ngx.say(err or res.body) - ngx.exit(ngx.status) + -- check whether we can fallback to regular skylink request + local match_skylink = ngx.re.match(ngx.var.uri, "^/([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?") + + if match_skylink then + ngx.var.skylink = match_skylink[1] + ngx.var.path = match_skylink[2] + else + ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status + ngx.header["content-type"] = "text/plain" + ngx.say(err or res.body) + ngx.exit(ngx.status) + end else ngx.var.skylink = res.body - ngx.var.skylink_v1 = ngx.var.skylink - ngx.var.skylink_v2 = ngx.var.skylink end + + ngx.var.skylink_v1 = ngx.var.skylink + ngx.var.skylink_v2 = ngx.var.skylink } include /etc/nginx/conf.d/include/location-skylink; From a96158d5a432e9bbee7096a706be097323ae2410 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 20 Sep 2021 22:28:38 +0200 Subject: [PATCH 2/8] add / --- docker/nginx/conf.d/server/server.dnslink | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/server/server.dnslink b/docker/nginx/conf.d/server/server.dnslink index e04c5a7a..06e3e3ca 100644 --- a/docker/nginx/conf.d/server/server.dnslink +++ b/docker/nginx/conf.d/server/server.dnslink @@ -16,7 +16,7 @@ location / { if match_skylink then ngx.var.skylink = match_skylink[1] - ngx.var.path = match_skylink[2] + ngx.var.path = match_skylink[2] or "/" else ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status ngx.header["content-type"] = "text/plain" From 9737f22fed079b80f6083d9e1f0196defc012c56 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 26 Oct 2021 12:50:35 +0200 Subject: [PATCH 3/8] changelog --- changelog/items/key-updates/1223-wildcard-api.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/items/key-updates/1223-wildcard-api.md diff --git a/changelog/items/key-updates/1223-wildcard-api.md b/changelog/items/key-updates/1223-wildcard-api.md new file mode 100644 index 00000000..88ca91d9 --- /dev/null +++ b/changelog/items/key-updates/1223-wildcard-api.md @@ -0,0 +1 @@ +- expose generic skylink serving endpoint on domain aliases From 81a1338f00827108210eaa0a5af6a750842dabd9 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 26 Oct 2021 13:01:15 +0200 Subject: [PATCH 4/8] move cache to lua to minimise number of requests --- docker/nginx/conf.d/server.dnslink.conf | 2 ++ docker/nginx/conf.d/server/server.dnslink | 40 ++++++++++++++--------- packages/dnslink-api/package.json | 3 +- packages/dnslink-api/src/index.js | 9 ----- packages/dnslink-api/yarn.lock | 12 ------- 5 files changed, 28 insertions(+), 38 deletions(-) diff --git a/docker/nginx/conf.d/server.dnslink.conf b/docker/nginx/conf.d/server.dnslink.conf index 8a051d3f..491bc389 100644 --- a/docker/nginx/conf.d/server.dnslink.conf +++ b/docker/nginx/conf.d/server.dnslink.conf @@ -1,3 +1,5 @@ +lua_shared_dict dnslink 10m; + server { listen 80 default_server; listen [::]:80 default_server; diff --git a/docker/nginx/conf.d/server/server.dnslink b/docker/nginx/conf.d/server/server.dnslink index 06e3e3ca..f8c7b49d 100644 --- a/docker/nginx/conf.d/server/server.dnslink +++ b/docker/nginx/conf.d/server/server.dnslink @@ -5,28 +5,38 @@ location / { set $path $uri; rewrite_by_lua_block { - local httpc = require("resty.http").new() + local cache = ngx.shared.dnslink + local cache_value = cache:get(ngx.var.host) - -- 10.10.10.55 points to dnslink-api service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.55:3100/dnslink/" .. ngx.var.host) + if cache_value == nil then + local httpc = require("resty.http").new() - if err or (res and res.status ~= ngx.HTTP_OK) then - -- check whether we can fallback to regular skylink request - local match_skylink = ngx.re.match(ngx.var.uri, "^/([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?") + -- 10.10.10.55 points to dnslink-api service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.55:3100/dnslink/" .. ngx.var.host) - if match_skylink then - ngx.var.skylink = match_skylink[1] - ngx.var.path = match_skylink[2] or "/" + if err or (res and res.status ~= ngx.HTTP_OK) then + -- check whether we can fallback to regular skylink request + local match_skylink = ngx.re.match(ngx.var.uri, "^/([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?") + + if match_skylink then + ngx.var.skylink = match_skylink[1] + ngx.var.path = match_skylink[2] or "/" + else + ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status + ngx.header["content-type"] = "text/plain" + ngx.say(err or res.body) + ngx.exit(ngx.status) + end else - ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status - ngx.header["content-type"] = "text/plain" - ngx.say(err or res.body) - ngx.exit(ngx.status) + ngx.var.skylink = res.body end + + local cache_ttl = 300 -- 5 minutes cache expire time + cache:set(ngx.var.host, ngx.var.skylink, cache_ttl) else - ngx.var.skylink = res.body + ngx.var.skylink = cache_value end - + ngx.var.skylink_v1 = ngx.var.skylink ngx.var.skylink_v2 = ngx.var.skylink } diff --git a/packages/dnslink-api/package.json b/packages/dnslink-api/package.json index a29295e1..7f833ed2 100644 --- a/packages/dnslink-api/package.json +++ b/packages/dnslink-api/package.json @@ -5,8 +5,7 @@ "license": "SEE LICENSE IN LICENSE.md", "dependencies": { "express": "^4.17.1", - "is-valid-domain": "^0.1.2", - "node-cache": "^5.1.2" + "is-valid-domain": "^0.1.2" }, "devDependencies": { "prettier": "^2.4.1" diff --git a/packages/dnslink-api/src/index.js b/packages/dnslink-api/src/index.js index fdddf9aa..300277c2 100644 --- a/packages/dnslink-api/src/index.js +++ b/packages/dnslink-api/src/index.js @@ -1,14 +1,11 @@ const dns = require("dns"); const express = require("express"); -const NodeCache = require("node-cache"); const isValidDomain = require("is-valid-domain"); const host = process.env.DNSLINK_API_HOSTNAME || "0.0.0.0"; const port = Number(process.env.DNSLINK_API_PORT) || 3100; -const cacheTTL = Number(process.env.DNSLINK_API_CACHE_TTL) || 300; // default to 5 minutes const server = express(); -const cache = new NodeCache({ stdTTL: cacheTTL }); const dnslinkNamespace = "skynet-ns"; const dnslinkRegExp = new RegExp(`^dnslink=/${dnslinkNamespace}/.+$`); @@ -23,10 +20,6 @@ server.get("/dnslink/:name", async (req, res) => { return failure(`"${req.params.name}" is not a valid domain`); } - if (cache.has(req.params.name)) { - return success(cache.get(req.params.name)); - } - const lookup = `_dnslink.${req.params.name}`; dns.resolveTxt(lookup, (error, records) => { @@ -65,8 +58,6 @@ server.get("/dnslink/:name", async (req, res) => { const skylink = matchSkylink[1]; - cache.set(req.params.name, skylink); - console.log(`${req.params.name} => ${skylink}`); return success(skylink); diff --git a/packages/dnslink-api/yarn.lock b/packages/dnslink-api/yarn.lock index 4a359f7a..5cff1dda 100644 --- a/packages/dnslink-api/yarn.lock +++ b/packages/dnslink-api/yarn.lock @@ -36,11 +36,6 @@ bytes@3.1.0: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg== -clone@2.x: - version "2.1.2" - resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" - integrity sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18= - content-disposition@0.5.3: version "0.5.3" resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd" @@ -257,13 +252,6 @@ negotiator@0.6.2: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== -node-cache@^5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/node-cache/-/node-cache-5.1.2.tgz#f264dc2ccad0a780e76253a694e9fd0ed19c398d" - integrity sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg== - dependencies: - clone "2.x" - on-finished@~2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" From d13a68d8254e33fb3c355e5e073ba39897b7aebe Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 26 Oct 2021 16:44:06 +0200 Subject: [PATCH 5/8] fix caching --- docker/nginx/conf.d/server/server.dnslink | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/nginx/conf.d/server/server.dnslink b/docker/nginx/conf.d/server/server.dnslink index f8c7b49d..efaf314d 100644 --- a/docker/nginx/conf.d/server/server.dnslink +++ b/docker/nginx/conf.d/server/server.dnslink @@ -27,12 +27,12 @@ location / { ngx.say(err or res.body) ngx.exit(ngx.status) end + + local cache_ttl = 300 -- 5 minutes cache expire time + cache:set(ngx.var.host, ngx.var.skylink, cache_ttl) else ngx.var.skylink = res.body end - - local cache_ttl = 300 -- 5 minutes cache expire time - cache:set(ngx.var.host, ngx.var.skylink, cache_ttl) else ngx.var.skylink = cache_value end From 40d0bb0e4d7c1bb185ace227bdbe9a2d5a95439d Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 26 Oct 2021 16:47:06 +0200 Subject: [PATCH 6/8] fix caching --- docker/nginx/conf.d/server/server.dnslink | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/nginx/conf.d/server/server.dnslink b/docker/nginx/conf.d/server/server.dnslink index efaf314d..1dd3a489 100644 --- a/docker/nginx/conf.d/server/server.dnslink +++ b/docker/nginx/conf.d/server/server.dnslink @@ -27,11 +27,11 @@ location / { ngx.say(err or res.body) ngx.exit(ngx.status) end + else + ngx.var.skylink = res.body local cache_ttl = 300 -- 5 minutes cache expire time cache:set(ngx.var.host, ngx.var.skylink, cache_ttl) - else - ngx.var.skylink = res.body end else ngx.var.skylink = cache_value From b52f055175c96334b6a1377a183458adef05a46a Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 26 Oct 2021 16:53:42 +0200 Subject: [PATCH 7/8] fix website skylink check --- packages/health-check/src/checks/critical.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/health-check/src/checks/critical.js b/packages/health-check/src/checks/critical.js index daa261f3..471e51e3 100644 --- a/packages/health-check/src/checks/critical.js +++ b/packages/health-check/src/checks/critical.js @@ -61,8 +61,9 @@ async function handshakeSubdomainCheck(done) { // websiteSkylinkCheck returns the result of accessing siasky.net website through skylink async function websiteSkylinkCheck(done) { const websiteSkylink = "0404dsjvti046fsua4ktor9grrpe76erq9jot9cvopbhsvsu76r4r30"; + const url = await skynetClient.getSkylinkUrl(websiteSkylink, { subdomain: true }); - return done(await genericAccessCheck("website_skylink", websiteSkylink)); + return done(await genericAccessCheck("website_skylink", url)); } // accountWebsiteCheck returns the result of accessing account dashboard website From 313855a690092ea907cbd3f9bf9dad4de117be73 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 26 Oct 2021 16:56:07 +0200 Subject: [PATCH 8/8] Revert "fix website skylink check" This reverts commit b52f055175c96334b6a1377a183458adef05a46a. --- packages/health-check/src/checks/critical.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/health-check/src/checks/critical.js b/packages/health-check/src/checks/critical.js index 471e51e3..daa261f3 100644 --- a/packages/health-check/src/checks/critical.js +++ b/packages/health-check/src/checks/critical.js @@ -61,9 +61,8 @@ async function handshakeSubdomainCheck(done) { // websiteSkylinkCheck returns the result of accessing siasky.net website through skylink async function websiteSkylinkCheck(done) { const websiteSkylink = "0404dsjvti046fsua4ktor9grrpe76erq9jot9cvopbhsvsu76r4r30"; - const url = await skynetClient.getSkylinkUrl(websiteSkylink, { subdomain: true }); - return done(await genericAccessCheck("website_skylink", url)); + return done(await genericAccessCheck("website_skylink", websiteSkylink)); } // accountWebsiteCheck returns the result of accessing account dashboard website