Merge pull request #1223 from SkynetLabs/wildcard-api

expose skylink api over external domains

changelog/items/key-updates/1223-wildcard-api.md

@@ -0,0 +1 @@
+- expose generic skylink serving endpoint on domain aliases

docker/nginx/conf.d/include/location-hns

@@ -75,7 +75,7 @@ access_by_lua_block {
     end
 }
 
-# we proxy to another nginx location rather than directly to siad because we don't want to deal with caching here
+# we proxy to another nginx location rather than directly to siad because we do not want to deal with caching here
 proxy_pass https://127.0.0.1/$skylink$path$is_args$args;
 
 # in case siad returns location header, we need to replace the skylink with the domain name

docker/nginx/conf.d/include/location-skylink

@@ -12,7 +12,7 @@ if ($request_method = PURGE) {
 limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
 
 # $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was:
-# a) skylink v1 - it wouldn't matter, no additional logic is executed
+# a) skylink v1 - it would not matter, no additional logic is executed
 # b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update 
 #      $skylink_v1 variable so then the proxy request to skyd can be cached in nginx (proxy_cache_key 
 #      in proxy-cache-downloads includes $skylink_v1 as a part of the cache key)
@@ -91,7 +91,7 @@ limit_rate $limit_rate;
 proxy_read_timeout 600;
 proxy_set_header User-Agent: Sia-Agent;
 
-# in case the requested skylink was v2 and we already resolved it to skylink v1, we're going to pass resolved 
+# in case the requested skylink was v2 and we already resolved it to skylink v1, we are going to pass resolved 
 # skylink v1 to skyd to save that extra skylink v2 lookup in skyd but in turn, in case skyd returns a redirect 
 # we need to rewrite the skylink v1 to skylink v2 in the location header with proxy_redirect
 proxy_redirect $skylink_v1 $skylink_v2;

docker/nginx/conf.d/server.dnslink.conf

@@ -1,3 +1,5 @@
+lua_shared_dict dnslink 10m;
+
 server {
     listen 80 default_server;
     listen [::]:80 default_server;

docker/nginx/conf.d/server/server.dnslink

@@ -5,21 +5,40 @@ location / {
     set $path $uri;
 
     rewrite_by_lua_block {
-        local httpc = require("resty.http").new()
+        local cache = ngx.shared.dnslink
+        local cache_value = cache:get(ngx.var.host)
 
-        -- 10.10.10.55 points to dnslink-api service (alias not available when using resty-http)
+        if cache_value == nil then
-        local res, err = httpc:request_uri("http://10.10.10.55:3100/dnslink/" .. ngx.var.host)
+            local httpc = require("resty.http").new()
 
-        if err or (res and res.status ~= ngx.HTTP_OK) then
+            -- 10.10.10.55 points to dnslink-api service (alias not available when using resty-http)
-            ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
+            local res, err = httpc:request_uri("http://10.10.10.55:3100/dnslink/" .. ngx.var.host)
-            ngx.header["content-type"] = "text/plain"
+
-            ngx.say(err or res.body)
+            if err or (res and res.status ~= ngx.HTTP_OK) then
-            ngx.exit(ngx.status)
+                -- check whether we can fallback to regular skylink request
+                local match_skylink = ngx.re.match(ngx.var.uri, "^/([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?")
+
+                if match_skylink then
+                    ngx.var.skylink = match_skylink[1]
+                    ngx.var.path = match_skylink[2] or "/"
+                else
+                    ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
+                    ngx.header["content-type"] = "text/plain"
+                    ngx.say(err or res.body)
+                    ngx.exit(ngx.status)
+                end
+            else
+                ngx.var.skylink = res.body
+
+                local cache_ttl = 300 -- 5 minutes cache expire time
+                cache:set(ngx.var.host, ngx.var.skylink, cache_ttl)
+            end
         else
-            ngx.var.skylink = res.body
+            ngx.var.skylink = cache_value
-            ngx.var.skylink_v1 = ngx.var.skylink
-            ngx.var.skylink_v2 = ngx.var.skylink
         end
+
+        ngx.var.skylink_v1 = ngx.var.skylink
+        ngx.var.skylink_v2 = ngx.var.skylink
     }
 
     include /etc/nginx/conf.d/include/location-skylink;

packages/dnslink-api/package.json

@@ -5,8 +5,7 @@
   "license": "SEE LICENSE IN LICENSE.md",
   "dependencies": {
     "express": "^4.17.1",
-    "is-valid-domain": "^0.1.2",
+    "is-valid-domain": "^0.1.2"
-    "node-cache": "^5.1.2"
   },
   "devDependencies": {
     "prettier": "^2.4.1"

packages/dnslink-api/src/index.js

@@ -1,14 +1,11 @@
 const dns = require("dns");
 const express = require("express");
-const NodeCache = require("node-cache");
 const isValidDomain = require("is-valid-domain");
 
 const host = process.env.DNSLINK_API_HOSTNAME || "0.0.0.0";
 const port = Number(process.env.DNSLINK_API_PORT) || 3100;
-const cacheTTL = Number(process.env.DNSLINK_API_CACHE_TTL) || 300; // default to 5 minutes
 
 const server = express();
-const cache = new NodeCache({ stdTTL: cacheTTL });
 
 const dnslinkNamespace = "skynet-ns";
 const dnslinkRegExp = new RegExp(`^dnslink=/${dnslinkNamespace}/.+$`);
@@ -23,10 +20,6 @@ server.get("/dnslink/:name", async (req, res) => {
     return failure(`"${req.params.name}" is not a valid domain`);
   }
 
-  if (cache.has(req.params.name)) {
-    return success(cache.get(req.params.name));
-  }
-
   const lookup = `_dnslink.${req.params.name}`;
 
   dns.resolveTxt(lookup, (error, records) => {
@@ -65,8 +58,6 @@ server.get("/dnslink/:name", async (req, res) => {
 
     const skylink = matchSkylink[1];
 
-    cache.set(req.params.name, skylink);
-
     console.log(`${req.params.name} => ${skylink}`);
 
     return success(skylink);

packages/dnslink-api/yarn.lock

@@ -36,11 +36,6 @@ bytes@3.1.0:
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6"
   integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==
 
-clone@2.x:
-  version "2.1.2"
-  resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f"
-  integrity sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=
-
 content-disposition@0.5.3:
   version "0.5.3"
   resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd"
@@ -257,13 +252,6 @@ negotiator@0.6.2:
   resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb"
   integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==
 
-node-cache@^5.1.2:
-  version "5.1.2"
-  resolved "https://registry.yarnpkg.com/node-cache/-/node-cache-5.1.2.tgz#f264dc2ccad0a780e76253a694e9fd0ed19c398d"
-  integrity sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==
-  dependencies:
-    clone "2.x"
-
 on-finished@~2.3.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947"