Merge pull request #310 from NebulousLabs/fix-hns-header-overflow-and-redirects

fix hns header overflow and redirects

.github/workflows/build.yml

@@ -11,7 +11,7 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v1
         with:
-          node-version: 12.x
+          node-version: 14.6
 
       - name: Install dependencies
         run: yarn

.gitignore

@@ -78,3 +78,4 @@ docker/data
 # Cache files
 __pycache__
 /.idea/
+/venv/

docker-compose.yml

@@ -55,6 +55,7 @@ services:
       - 80
     depends_on:
       - docker-host
+      - handshake-api
 
   handshake:
     build:
@@ -83,6 +84,7 @@ services:
     container_name: handshake-api
     restart: unless-stopped
     environment:
+      - HOSTNAME=0.0.0.0
       - HSD_HOST=handshake
       - HSD_NETWORK=main
       - HSD_PORT=12037
@@ -94,7 +96,6 @@ services:
       - 3100
     depends_on:
       - handshake
-      - nginx
 
   health-check:
     build:
@@ -108,9 +109,11 @@ services:
     networks:
       - shared
     environment:
+      - HOSTNAME=0.0.0.0
       - PORTAL_URL=nginx
     expose:
       - 3100
     depends_on:
-      - docker-host
+      - caddy
-      - nginx
+      - handshake
+      - handshake-api

docker/handshake-api/Dockerfile

@@ -17,4 +17,4 @@ ENV HSD_API_KEY="foo"
 
 EXPOSE $PORT
 
-ENTRYPOINT ["node", "index.js"]
+ENTRYPOINT ["node", "--max-http-header-size=64000", "index.js"]

docker/nginx/conf.d/client.conf

@@ -114,6 +114,7 @@ server {
 
 	location /hns {
 		include /etc/nginx/conf.d/include/cors;
+		include /etc/nginx/conf.d/include/proxy-buffer;
 
 		proxy_pass http://handshake-api:3100;
 	}
@@ -173,6 +174,7 @@ server {
 
 	location ~ "^/([a-zA-Z0-9-_]{46}(/.*)?)$" {
 		include /etc/nginx/conf.d/include/cors;
+		include /etc/nginx/conf.d/include/proxy-buffer;
 
 		limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
 
@@ -185,10 +187,6 @@ server {
 		# proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct)
 		proxy_pass http://siad/skynet/skylink/$skylink$is_args$args;
 
-		# if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs
-		proxy_buffer_size 128k;
-		proxy_buffers 4 128k;
-
 		# cache frequent (> 10) downloads for 24 hours
 		proxy_cache skynet;
 		proxy_cache_key $uri;
@@ -199,6 +197,7 @@ server {
 
 	location ~ "^/file/([a-zA-Z0-9-_]{46}(/.*)?)$" {
 		include /etc/nginx/conf.d/include/cors;
+		include /etc/nginx/conf.d/include/proxy-buffer;
 
 		limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
 
@@ -212,10 +211,6 @@ server {
 		# this alias also adds attachment=true url param to force download the file
 		proxy_pass http://siad/skynet/skylink/$skylink?attachment=true&$args;
 		
-		# if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs
-		proxy_buffer_size 128k;
-		proxy_buffers 4 128k;
-		
 		# cache frequent (> 10) downloads for 24 hours
 		proxy_cache skynet;
 		proxy_cache_key $uri;

docker/nginx/conf.d/include/proxy-buffer

@@ -0,0 +1,4 @@
+# if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs
+proxy_buffer_size 128k;
+proxy_buffers 4 256k;
+proxy_busy_buffers_size 256k;

handshake-api/index.js

@@ -19,7 +19,8 @@ const clientOptions = {
 };
 const client = new NodeClient(clientOptions);
 
-const startsWithSkylinkRegExp = /^[a-zA-Z0-9_-]{46}/;
+// Match both `sia://HASH` and `HASH` links.
+const startsWithSkylinkRegExp = /^(sia:\/\/)?[a-zA-Z0-9_-]{46}/;
 
 const getDomainRecords = async (name) => {
   const response = await client.execute("getnameresource", [name]);
@@ -31,7 +32,13 @@ const getDomainRecords = async (name) => {
 };
 
 const findSkylinkRecord = (records) => {
-  return records?.find(({ txt }) => txt?.some((entry) => isValidSkylink(entry)));
+  // Find the last one, so people can update their domains in a non-destructive
+  // way by simply adding a new link. This will also allow keeping links to
+  // older versions for backwards compatibility.
+  return records
+    ?.slice()
+    .reverse()
+    .find(({ txt }) => txt?.some((entry) => isValidSkylink(entry)));
 };
 
 const getSkylinkFromRecord = (record) => {
@@ -69,7 +76,10 @@ server.use(
     // eslint-disable-next-line no-unused-vars
     userResHeaderDecorator(headers, userReq, userRes, proxyReq, proxyRes) {
       if (headers.location && headers.location.match(startsWithSkylinkRegExp)) {
-        headers.location = headers.location.replace(startsWithSkylinkRegExp, `/hns/${userReq.params.name}`);
+        headers.location = headers.location.replace(
+          startsWithSkylinkRegExp,
+          `/hns/${userReq.params.name.replace("sia://", "")}`
+        );
       }
 
       return headers;
@@ -81,18 +91,16 @@ server.use(
       const record = findSkylinkRecord(records);
       if (!record) throw new Error(`No skylink found in dns records of ${req.params.name}`);
 
-      const skylink = getSkylinkFromRecord(record);
+      const skylink = getSkylinkFromRecord(record).replace("sia://", ""); // get skylink and strip sia:// prefix
       const basepath = url.resolve("/", skylink); // make the url absolute
       const subpath = req.url.slice(1); // drop the leading slash
 
-      // if the skylink from handshake does not contain a subpath but subpath
+      // if the record is just a raw skylink, replace baseUrl with /skylink
-      // is defined in request, join the skylink and subpath together (do not
+      if (skylink.length === 46) {
-      // use url.resolve because it will replace skylink with subapth thinking
+        return req.originalUrl.replace(req.baseUrl, basepath);
-      // it is relative)
-      if (skylink.length === 46 && subpath) {
-        return `${basepath}/${subpath}`;
       }
 
+      // if the record contains more than a skylink then it needs to be resolved
       return url.resolve(basepath, subpath);
     },
   })