diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 66f0cfcf..909abc86 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ jobs: - name: Use Node.js uses: actions/setup-node@v1 with: - node-version: 12.x + node-version: 14.6 - name: Install dependencies run: yarn diff --git a/.gitignore b/.gitignore index 43f25aa7..6a4f2eb0 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,4 @@ docker/data # Cache files __pycache__ /.idea/ +/venv/ diff --git a/docker-compose.yml b/docker-compose.yml index 0e617ef6..e1748e85 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,6 +55,7 @@ services: - 80 depends_on: - docker-host + - handshake-api handshake: build: @@ -83,6 +84,7 @@ services: container_name: handshake-api restart: unless-stopped environment: + - HOSTNAME=0.0.0.0 - HSD_HOST=handshake - HSD_NETWORK=main - HSD_PORT=12037 @@ -94,7 +96,6 @@ services: - 3100 depends_on: - handshake - - nginx health-check: build: @@ -108,9 +109,11 @@ services: networks: - shared environment: + - HOSTNAME=0.0.0.0 - PORTAL_URL=nginx expose: - 3100 depends_on: - - docker-host - - nginx + - caddy + - handshake + - handshake-api diff --git a/docker/handshake-api/Dockerfile b/docker/handshake-api/Dockerfile index 22826b4e..c7a393b3 100644 --- a/docker/handshake-api/Dockerfile +++ b/docker/handshake-api/Dockerfile @@ -17,4 +17,4 @@ ENV HSD_API_KEY="foo" EXPOSE $PORT -ENTRYPOINT ["node", "index.js"] +ENTRYPOINT ["node", "--max-http-header-size=64000", "index.js"] diff --git a/docker/nginx/conf.d/client.conf b/docker/nginx/conf.d/client.conf index 6dca374b..bfa29563 100644 --- a/docker/nginx/conf.d/client.conf +++ b/docker/nginx/conf.d/client.conf @@ -114,6 +114,7 @@ server { location /hns { include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/proxy-buffer; proxy_pass http://handshake-api:3100; } @@ -173,6 +174,7 @@ server { location ~ "^/([a-zA-Z0-9-_]{46}(/.*)?)$" { include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/proxy-buffer; limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time @@ -184,10 +186,6 @@ server { proxy_set_header User-Agent: Sia-Agent; # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) proxy_pass http://siad/skynet/skylink/$skylink$is_args$args; - - # if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs - proxy_buffer_size 128k; - proxy_buffers 4 128k; # cache frequent (> 10) downloads for 24 hours proxy_cache skynet; @@ -199,6 +197,7 @@ server { location ~ "^/file/([a-zA-Z0-9-_]{46}(/.*)?)$" { include /etc/nginx/conf.d/include/cors; + include /etc/nginx/conf.d/include/proxy-buffer; limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time @@ -211,10 +210,6 @@ server { # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) # this alias also adds attachment=true url param to force download the file proxy_pass http://siad/skynet/skylink/$skylink?attachment=true&$args; - - # if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs - proxy_buffer_size 128k; - proxy_buffers 4 128k; # cache frequent (> 10) downloads for 24 hours proxy_cache skynet; diff --git a/docker/nginx/conf.d/include/proxy-buffer b/docker/nginx/conf.d/include/proxy-buffer new file mode 100644 index 00000000..f7a2b12e --- /dev/null +++ b/docker/nginx/conf.d/include/proxy-buffer @@ -0,0 +1,4 @@ +# if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs +proxy_buffer_size 128k; +proxy_buffers 4 256k; +proxy_busy_buffers_size 256k; diff --git a/handshake-api/index.js b/handshake-api/index.js index 07282e35..6c057ca1 100644 --- a/handshake-api/index.js +++ b/handshake-api/index.js @@ -19,7 +19,8 @@ const clientOptions = { }; const client = new NodeClient(clientOptions); -const startsWithSkylinkRegExp = /^[a-zA-Z0-9_-]{46}/; +// Match both `sia://HASH` and `HASH` links. +const startsWithSkylinkRegExp = /^(sia:\/\/)?[a-zA-Z0-9_-]{46}/; const getDomainRecords = async (name) => { const response = await client.execute("getnameresource", [name]); @@ -31,7 +32,13 @@ const getDomainRecords = async (name) => { }; const findSkylinkRecord = (records) => { - return records?.find(({ txt }) => txt?.some((entry) => isValidSkylink(entry))); + // Find the last one, so people can update their domains in a non-destructive + // way by simply adding a new link. This will also allow keeping links to + // older versions for backwards compatibility. + return records + ?.slice() + .reverse() + .find(({ txt }) => txt?.some((entry) => isValidSkylink(entry))); }; const getSkylinkFromRecord = (record) => { @@ -69,7 +76,10 @@ server.use( // eslint-disable-next-line no-unused-vars userResHeaderDecorator(headers, userReq, userRes, proxyReq, proxyRes) { if (headers.location && headers.location.match(startsWithSkylinkRegExp)) { - headers.location = headers.location.replace(startsWithSkylinkRegExp, `/hns/${userReq.params.name}`); + headers.location = headers.location.replace( + startsWithSkylinkRegExp, + `/hns/${userReq.params.name.replace("sia://", "")}` + ); } return headers; @@ -81,18 +91,16 @@ server.use( const record = findSkylinkRecord(records); if (!record) throw new Error(`No skylink found in dns records of ${req.params.name}`); - const skylink = getSkylinkFromRecord(record); + const skylink = getSkylinkFromRecord(record).replace("sia://", ""); // get skylink and strip sia:// prefix const basepath = url.resolve("/", skylink); // make the url absolute const subpath = req.url.slice(1); // drop the leading slash - // if the skylink from handshake does not contain a subpath but subpath - // is defined in request, join the skylink and subpath together (do not - // use url.resolve because it will replace skylink with subapth thinking - // it is relative) - if (skylink.length === 46 && subpath) { - return `${basepath}/${subpath}`; + // if the record is just a raw skylink, replace baseUrl with /skylink + if (skylink.length === 46) { + return req.originalUrl.replace(req.baseUrl, basepath); } + // if the record contains more than a skylink then it needs to be resolved return url.resolve(basepath, subpath); }, })