From ec16b161010e98c6b033083516a0bf6af0ab73bb Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Tue, 5 Oct 2021 12:13:48 +0200 Subject: [PATCH 01/16] update skynet-js to new beta release --- packages/website/package.json | 2 +- packages/website/yarn.lock | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/website/package.json b/packages/website/package.json index 69e0be64..21b5dfb9 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -59,7 +59,7 @@ "react-svg-loader": "^3.0.3", "react-syntax-highlighter": "^15.4.4", "react-use": "^17.3.1", - "skynet-js": "^4.0.11-beta", + "skynet-js": "4.0.17-beta", "stream-browserify": "^3.0.0", "swr": "^1.0.1", "tailwindcss": "^2.2.19" diff --git a/packages/website/yarn.lock b/packages/website/yarn.lock index a1611d7b..a8479e61 100644 --- a/packages/website/yarn.lock +++ b/packages/website/yarn.lock @@ -2844,7 +2844,7 @@ axe-core@^4.0.2: resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.3.3.tgz#b55cd8e8ddf659fe89b064680e1c6a4dceab0325" integrity sha512-/lqqLAmuIPi79WYfRpy2i8z+x+vxU3zX2uAm0gs1q52qTuKwolOj1P8XbufpXcsydrpKx2yGn2wzAnxCMV86QA== -axios@^0.21.0, axios@^0.21.1, axios@^0.21.4: +axios@^0.21.1, axios@^0.21.4: version "0.21.4" resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== @@ -12064,12 +12064,12 @@ sjcl@^1.0.8: resolved "https://registry.yarnpkg.com/sjcl/-/sjcl-1.0.8.tgz#f2ec8d7dc1f0f21b069b8914a41a8f236b0e252a" integrity sha512-LzIjEQ0S0DpIgnxMEayM1rq9aGwGRG4OnZhCdjx7glTaJtf4zRfpg87ImfjSJjoW9vKpagd82McDOwbRT5kQKQ== -skynet-js@^4.0.11-beta: - version "4.0.11-beta" - resolved "https://registry.yarnpkg.com/skynet-js/-/skynet-js-4.0.11-beta.tgz#ec313d586f8e026e0b3b4b608f2f3b4a449e8a71" - integrity sha512-/fpxBeegGJeK+VWE8svUhBc2tVX0kJAZod5K172kKPO1g/GECeQLLSuWL2yvCMbSSTyH0dIFr5gemwGNsZHlMQ== +skynet-js@4.0.17-beta: + version "4.0.17-beta" + resolved "https://registry.yarnpkg.com/skynet-js/-/skynet-js-4.0.17-beta.tgz#0e0f07799e3635bef9c8f3387e2393a1255aafd2" + integrity sha512-Yl5qGdasrVf6ZQzz/huAkmNHxyNv1UsgfoX7OE6Yg1q8SxZXmQnTsPK9oupgow6xjN/VINenXuY5ZVjoEmhZyw== dependencies: - axios "^0.21.0" + axios "^0.21.1" base32-decode "^1.0.0" base32-encode "^1.1.1" base64-js "^1.3.1" @@ -12080,16 +12080,16 @@ skynet-js@^4.0.11-beta: post-me "^0.4.5" randombytes "^2.1.0" sjcl "^1.0.8" - skynet-mysky-utils "^0.2.2" + skynet-mysky-utils "^0.3.0" tus-js-client "^2.2.0" tweetnacl "^1.0.3" url-join "^4.0.1" url-parse "^1.5.1" -skynet-mysky-utils@^0.2.2: - version "0.2.3" - resolved "https://registry.yarnpkg.com/skynet-mysky-utils/-/skynet-mysky-utils-0.2.3.tgz#5007cf8f7599b665ccf016003b37a4ed0fb19abf" - integrity sha512-wRrAASn4haux2fu+2pJLv+uV/TGbBecXT1jaqD3/IQgqbEwZUpDNJJrYnYAfp/0cY5Xmuc2ZX90NNr34neAcWg== +skynet-mysky-utils@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/skynet-mysky-utils/-/skynet-mysky-utils-0.3.0.tgz#87fdc0a5f8547cf660280ef86b7a762269919bad" + integrity sha512-X9L6SrVTdwTUFook/E6zUWCOpXHdyspLAu0elQbbPkZCWeFpr/XXTMbiyPV3m1liYsesngAKxzaSqylaTWOGUA== dependencies: post-me "^0.4.5" From e0f75df542273891f0ffc72dfc37ef4bb9a9ee2a Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 29 Oct 2021 09:38:25 +0200 Subject: [PATCH 02/16] update to 4.0.18 --- packages/website/package.json | 2 +- packages/website/yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/website/package.json b/packages/website/package.json index 21b5dfb9..e64ad847 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -59,7 +59,7 @@ "react-svg-loader": "^3.0.3", "react-syntax-highlighter": "^15.4.4", "react-use": "^17.3.1", - "skynet-js": "4.0.17-beta", + "skynet-js": "^4.0.18-beta", "stream-browserify": "^3.0.0", "swr": "^1.0.1", "tailwindcss": "^2.2.19" diff --git a/packages/website/yarn.lock b/packages/website/yarn.lock index a8479e61..b213208b 100644 --- a/packages/website/yarn.lock +++ b/packages/website/yarn.lock @@ -12064,10 +12064,10 @@ sjcl@^1.0.8: resolved "https://registry.yarnpkg.com/sjcl/-/sjcl-1.0.8.tgz#f2ec8d7dc1f0f21b069b8914a41a8f236b0e252a" integrity sha512-LzIjEQ0S0DpIgnxMEayM1rq9aGwGRG4OnZhCdjx7glTaJtf4zRfpg87ImfjSJjoW9vKpagd82McDOwbRT5kQKQ== -skynet-js@4.0.17-beta: - version "4.0.17-beta" - resolved "https://registry.yarnpkg.com/skynet-js/-/skynet-js-4.0.17-beta.tgz#0e0f07799e3635bef9c8f3387e2393a1255aafd2" - integrity sha512-Yl5qGdasrVf6ZQzz/huAkmNHxyNv1UsgfoX7OE6Yg1q8SxZXmQnTsPK9oupgow6xjN/VINenXuY5ZVjoEmhZyw== +skynet-js@^4.0.18-beta: + version "4.0.18-beta" + resolved "https://registry.yarnpkg.com/skynet-js/-/skynet-js-4.0.18-beta.tgz#4683f0837ae552802f39c0e7081a1b978b79ef4a" + integrity sha512-7mE9xrejTpRacZfhhCqx+dm7k1y6ITLZMWZnsPp13D2N9CNroyzB75Yi7033qwPtdo9i6BEzIDolZl66j+uALw== dependencies: axios "^0.21.1" base32-decode "^1.0.0" From de16b33ed89c508d9ee7be253ed6044746c43d14 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 8 Nov 2021 15:20:26 +0100 Subject: [PATCH 03/16] update skynet-js --- packages/health-check/package.json | 2 +- packages/health-check/yarn.lock | 28 ++++++++++++------------ packages/website/package.json | 2 +- packages/website/yarn.lock | 34 ++++++++++++++++-------------- 4 files changed, 34 insertions(+), 32 deletions(-) diff --git a/packages/health-check/package.json b/packages/health-check/package.json index c7020969..77a21543 100644 --- a/packages/health-check/package.json +++ b/packages/health-check/package.json @@ -13,7 +13,7 @@ "http-status-codes": "^2.1.2", "lodash": "^4.17.21", "lowdb": "^1.0.0", - "skynet-js": "^4.0.18-beta", + "skynet-js": "^4.0.19-beta", "write-file-atomic": "^3.0.3", "yargs": "^17.2.1" }, diff --git a/packages/health-check/yarn.lock b/packages/health-check/yarn.lock index ca7aeabf..d19d2be0 100644 --- a/packages/health-check/yarn.lock +++ b/packages/health-check/yarn.lock @@ -78,12 +78,12 @@ asynckit@^0.4.0: resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= -axios@^0.21.1: - version "0.21.4" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" - integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== +axios@^0.24.0: + version "0.24.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" + integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== dependencies: - follow-redirects "^1.14.0" + follow-redirects "^1.14.4" base32-decode@^1.0.0: version "1.0.0" @@ -354,10 +354,10 @@ finalhandler@~1.1.2: statuses "~1.5.0" unpipe "~1.0.0" -follow-redirects@^1.14.0: - version "1.14.4" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.4.tgz#838fdf48a8bbdd79e52ee51fb1c94e3ed98b9379" - integrity sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g== +follow-redirects@^1.14.4: + version "1.14.5" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.5.tgz#f09a5848981d3c772b5392309778523f8d85c381" + integrity sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA== form-data@^4.0.0: version "4.0.0" @@ -863,12 +863,12 @@ sjcl@^1.0.8: resolved "https://registry.yarnpkg.com/sjcl/-/sjcl-1.0.8.tgz#f2ec8d7dc1f0f21b069b8914a41a8f236b0e252a" integrity sha512-LzIjEQ0S0DpIgnxMEayM1rq9aGwGRG4OnZhCdjx7glTaJtf4zRfpg87ImfjSJjoW9vKpagd82McDOwbRT5kQKQ== -skynet-js@^4.0.18-beta: - version "4.0.18-beta" - resolved "https://registry.yarnpkg.com/skynet-js/-/skynet-js-4.0.18-beta.tgz#4683f0837ae552802f39c0e7081a1b978b79ef4a" - integrity sha512-7mE9xrejTpRacZfhhCqx+dm7k1y6ITLZMWZnsPp13D2N9CNroyzB75Yi7033qwPtdo9i6BEzIDolZl66j+uALw== +skynet-js@^4.0.19-beta: + version "4.0.19-beta" + resolved "https://registry.yarnpkg.com/skynet-js/-/skynet-js-4.0.19-beta.tgz#d4c640898c79cf69e45aa1c3c1ed5c80aa1aeced" + integrity sha512-d8/q3E3OjUxgCCAW28gNFvbahj0ks8ym122XTopbRyvAZKk9+/Z4ians9v8Tov36Z4k/un+Ilw/0i6DtM8c8Dw== dependencies: - axios "^0.21.1" + axios "^0.24.0" base32-decode "^1.0.0" base32-encode "^1.1.1" base64-js "^1.3.1" diff --git a/packages/website/package.json b/packages/website/package.json index e64ad847..fe6cd49d 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -59,7 +59,7 @@ "react-svg-loader": "^3.0.3", "react-syntax-highlighter": "^15.4.4", "react-use": "^17.3.1", - "skynet-js": "^4.0.18-beta", + "skynet-js": "^4.0.19-beta", "stream-browserify": "^3.0.0", "swr": "^1.0.1", "tailwindcss": "^2.2.19" diff --git a/packages/website/yarn.lock b/packages/website/yarn.lock index b213208b..ae0038f0 100644 --- a/packages/website/yarn.lock +++ b/packages/website/yarn.lock @@ -2851,6 +2851,13 @@ axios@^0.21.1, axios@^0.21.4: dependencies: follow-redirects "^1.14.0" +axios@^0.24.0: + version "0.24.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" + integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== + dependencies: + follow-redirects "^1.14.4" + axobject-query@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.2.0.tgz#943d47e10c0b704aa42275e20edf3722648989be" @@ -5783,6 +5790,11 @@ follow-redirects@^1.0.0, follow-redirects@^1.14.0: resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.4.tgz#838fdf48a8bbdd79e52ee51fb1c94e3ed98b9379" integrity sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g== +follow-redirects@^1.14.4: + version "1.14.5" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.5.tgz#f09a5848981d3c772b5392309778523f8d85c381" + integrity sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA== + for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" @@ -10318,17 +10330,7 @@ polished@^4.1.3: dependencies: "@babel/runtime" "^7.14.0" -popmotion@11.0.0: - version "11.0.0" - resolved "https://registry.yarnpkg.com/popmotion/-/popmotion-11.0.0.tgz#910e2e7077d9aeba520db8744d40bb5354992212" - integrity sha512-kJDyaG00TtcANP5JZ51od+DCqopxBm2a/Txh3Usu23L9qntjY5wumvcVf578N8qXEHR1a+jx9XCv8zOntdYalQ== - dependencies: - framesync "^6.0.1" - hey-listen "^1.0.8" - style-value-types "5.0.0" - tslib "^2.1.0" - -popmotion@^11.0.0: +popmotion@11.0.0, popmotion@^11.0.0: version "11.0.0" resolved "https://registry.yarnpkg.com/popmotion/-/popmotion-11.0.0.tgz#910e2e7077d9aeba520db8744d40bb5354992212" integrity sha512-kJDyaG00TtcANP5JZ51od+DCqopxBm2a/Txh3Usu23L9qntjY5wumvcVf578N8qXEHR1a+jx9XCv8zOntdYalQ== @@ -12064,12 +12066,12 @@ sjcl@^1.0.8: resolved "https://registry.yarnpkg.com/sjcl/-/sjcl-1.0.8.tgz#f2ec8d7dc1f0f21b069b8914a41a8f236b0e252a" integrity sha512-LzIjEQ0S0DpIgnxMEayM1rq9aGwGRG4OnZhCdjx7glTaJtf4zRfpg87ImfjSJjoW9vKpagd82McDOwbRT5kQKQ== -skynet-js@^4.0.18-beta: - version "4.0.18-beta" - resolved "https://registry.yarnpkg.com/skynet-js/-/skynet-js-4.0.18-beta.tgz#4683f0837ae552802f39c0e7081a1b978b79ef4a" - integrity sha512-7mE9xrejTpRacZfhhCqx+dm7k1y6ITLZMWZnsPp13D2N9CNroyzB75Yi7033qwPtdo9i6BEzIDolZl66j+uALw== +skynet-js@^4.0.19-beta: + version "4.0.19-beta" + resolved "https://registry.yarnpkg.com/skynet-js/-/skynet-js-4.0.19-beta.tgz#d4c640898c79cf69e45aa1c3c1ed5c80aa1aeced" + integrity sha512-d8/q3E3OjUxgCCAW28gNFvbahj0ks8ym122XTopbRyvAZKk9+/Z4ians9v8Tov36Z4k/un+Ilw/0i6DtM8c8Dw== dependencies: - axios "^0.21.1" + axios "^0.24.0" base32-decode "^1.0.0" base32-encode "^1.1.1" base64-js "^1.3.1" From 6a3edd6848e022ccd7acd4002fd77fe5523f0b94 Mon Sep 17 00:00:00 2001 From: Matthew Sevey Date: Mon, 8 Nov 2021 13:13:44 -0500 Subject: [PATCH 04/16] Drop unused requested skylink header --- changelog/items/other/drop-requested-skylink-header.md | 1 + docker/nginx/conf.d/server/server.api | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) create mode 100644 changelog/items/other/drop-requested-skylink-header.md diff --git a/changelog/items/other/drop-requested-skylink-header.md b/changelog/items/other/drop-requested-skylink-header.md new file mode 100644 index 00000000..bcf702dc --- /dev/null +++ b/changelog/items/other/drop-requested-skylink-header.md @@ -0,0 +1 @@ +- Drop `Skynet-Requested-Skylink` header diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 5fc41cb1..389a2d7f 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -249,9 +249,6 @@ location /skynet/metadata { header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") - - -- do not expose internal header - ngx.header["Skynet-Requested-Skylink"] = "" } proxy_set_header User-Agent: Sia-Agent; @@ -264,9 +261,6 @@ location /skynet/resolve { header_filter_by_lua_block { ngx.header["Skynet-Portal-Api"] = os.getenv("SKYNET_PORTAL_API") ngx.header["Skynet-Server-Api"] = os.getenv("SKYNET_SERVER_API") - - -- do not expose internal header - ngx.header["Skynet-Requested-Skylink"] = "" } proxy_set_header User-Agent: Sia-Agent; From c25618cd7607b18002306b74aa6a54daf25464fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Nov 2021 11:37:06 +0000 Subject: [PATCH 05/16] Bump caddy from 2.4.5-builder to 2.4.6-builder in /docker/caddy Bumps caddy from 2.4.5-builder to 2.4.6-builder. --- updated-dependencies: - dependency-name: caddy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docker/caddy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/caddy/Dockerfile b/docker/caddy/Dockerfile index b9c09acd..e5dca531 100644 --- a/docker/caddy/Dockerfile +++ b/docker/caddy/Dockerfile @@ -1,9 +1,9 @@ -FROM caddy:2.4.5-builder AS caddy-builder +FROM caddy:2.4.6-builder AS caddy-builder # available dns resolvers: https://github.com/caddy-dns RUN xcaddy build --with github.com/caddy-dns/route53 -FROM caddy:2.4.5-alpine +FROM caddy:2.4.6-builder COPY --from=caddy-builder /usr/bin/caddy /usr/bin/caddy From 81acfe1e48d160214ac40abbf7fdf0b61ecc9cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Wypch=C5=82o?= Date: Tue, 9 Nov 2021 12:45:11 +0100 Subject: [PATCH 06/16] Update Dockerfile --- docker/caddy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/caddy/Dockerfile b/docker/caddy/Dockerfile index e5dca531..31075803 100644 --- a/docker/caddy/Dockerfile +++ b/docker/caddy/Dockerfile @@ -3,7 +3,7 @@ FROM caddy:2.4.6-builder AS caddy-builder # available dns resolvers: https://github.com/caddy-dns RUN xcaddy build --with github.com/caddy-dns/route53 -FROM caddy:2.4.6-builder +FROM caddy:2.4.6-alpine COPY --from=caddy-builder /usr/bin/caddy /usr/bin/caddy From 2f0f7d64e067ac72e32d61e98a9476c2da5ceb5b Mon Sep 17 00:00:00 2001 From: Matthew Sevey Date: Tue, 9 Nov 2021 18:19:49 -0500 Subject: [PATCH 07/16] Update handshake to use v3.0.1 --- changelog/items/other/update-handshake.md | 1 + docker-compose.yml | 2 +- docker/handshake/Dockerfile | 7 +------ 3 files changed, 3 insertions(+), 7 deletions(-) create mode 100644 changelog/items/other/update-handshake.md diff --git a/changelog/items/other/update-handshake.md b/changelog/items/other/update-handshake.md new file mode 100644 index 00000000..f9296fa4 --- /dev/null +++ b/changelog/items/other/update-handshake.md @@ -0,0 +1 @@ +- Update handshake to use v3.0.1 diff --git a/docker-compose.yml b/docker-compose.yml index 3f94c3ff..ca4dc51a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -103,7 +103,7 @@ services: build: context: ./docker/handshake dockerfile: Dockerfile - command: --chain-migrate=1 --wallet-migrate=1 + command: --chain-migrate=2 --wallet-migrate=1 container_name: handshake restart: unless-stopped logging: *default-logging diff --git a/docker/handshake/Dockerfile b/docker/handshake/Dockerfile index 3aa6e50b..d7a0a6a1 100644 --- a/docker/handshake/Dockerfile +++ b/docker/handshake/Dockerfile @@ -3,13 +3,8 @@ FROM node:16.13.0-alpine WORKDIR /opt/hsd RUN apk update && apk add bash unbound-dev gmp-dev g++ gcc make python2 git -# Checkout a specific commit until Handshake releases the next release after -# 2.4.0 then we should switch to that tag. -# -# The commit we are targetting right now contains a fix for handling the chain -# migration code for new portals. RUN git clone https://github.com/handshake-org/hsd.git /opt/hsd && \ - cd /opt/hsd && git checkout 6f0927db32723d6320c8bff255a6ccf70b2ccd32 && cd - + cd /opt/hsd && git checkout v3.0.1 && cd - RUN npm install --production ENV PATH="${PATH}:/opt/hsd/bin:/opt/hsd/node_modules/.bin" From 2e15826f58d570790c6f535140cba09b623397ef Mon Sep 17 00:00:00 2001 From: Filip Rysavy <29089732+firyx@users.noreply.github.com> Date: Wed, 10 Nov 2021 11:49:16 +0100 Subject: [PATCH 08/16] Remove Airtable hardcoded default values --- changelog/items/other/airtable-env-vars.md | 2 ++ setup-scripts/blocklist-airtable.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 changelog/items/other/airtable-env-vars.md diff --git a/changelog/items/other/airtable-env-vars.md b/changelog/items/other/airtable-env-vars.md new file mode 100644 index 00000000..dc287984 --- /dev/null +++ b/changelog/items/other/airtable-env-vars.md @@ -0,0 +1,2 @@ +- Remove hardcoded Airtable default values from blocklist script. Portal + operators need to define their own values in portal common config (LastPass). \ No newline at end of file diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 3dbce3ed..e6563901 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -14,9 +14,9 @@ import json setup() AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY") -AIRTABLE_BASE = os.getenv("AIRTABLE_BASE", "app89plJvA9EqTJEc") -AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE", "Table%201") -AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD", "Link") +AIRTABLE_BASE = os.getenv("AIRTABLE_BASE") +AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE") +AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD") async def run_checks(): From d68f8969886e5929242e2c31aa2fabbd42c3f2cd Mon Sep 17 00:00:00 2001 From: Filip Rysavy <29089732+firyx@users.noreply.github.com> Date: Wed, 10 Nov 2021 13:46:43 +0100 Subject: [PATCH 09/16] Add check if Airtable vars are defined --- setup-scripts/blocklist-airtable.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index e6563901..1173a90f 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -6,6 +6,7 @@ from time import sleep import traceback import os +import sys import re import asyncio import requests @@ -13,10 +14,17 @@ import json setup() -AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY") -AIRTABLE_BASE = os.getenv("AIRTABLE_BASE") -AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE") -AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD") + +# Check and load Airtable environment variables +airtable_env_vars = ["AIRTABLE_API_KEY", "AIRTABLE_BASE", "AIRTABLE_TABLE", "AIRTABLE_FIELD"] + +for e in airtable_env_vars: + # Check environment variable is defined + gete = os.getenv(e) + if not gete: + sys.exit("Configuration error: Environment variable " + e + " is not defined") + # Set variable + exec(e + " = \"" +gete + "\"") async def run_checks(): From 07b99b8192f9afee949d28b82a23890b8411b0e4 Mon Sep 17 00:00:00 2001 From: Filip Rysavy <29089732+firyx@users.noreply.github.com> Date: Wed, 10 Nov 2021 13:52:32 +0100 Subject: [PATCH 10/16] Add check if Airtable vars are defined --- setup-scripts/blocklist-airtable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 1173a90f..d5bd360e 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -24,7 +24,7 @@ for e in airtable_env_vars: if not gete: sys.exit("Configuration error: Environment variable " + e + " is not defined") # Set variable - exec(e + " = \"" +gete + "\"") + exec(e + " = \"" + gete + "\"") async def run_checks(): From efeddd6c70e9de9d0b4204fd2d70d70f778829f2 Mon Sep 17 00:00:00 2001 From: Filip Rysavy <29089732+firyx@users.noreply.github.com> Date: Wed, 10 Nov 2021 14:22:00 +0100 Subject: [PATCH 11/16] Add check if Airtable vars are defined --- setup-scripts/blocklist-airtable.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index d5bd360e..d0279e5f 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -15,16 +15,15 @@ import json setup() -# Check and load Airtable environment variables -airtable_env_vars = ["AIRTABLE_API_KEY", "AIRTABLE_BASE", "AIRTABLE_TABLE", "AIRTABLE_FIELD"] +AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY") +AIRTABLE_BASE = os.getenv("AIRTABLE_BASE") +AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE") +AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD") -for e in airtable_env_vars: - # Check environment variable is defined - gete = os.getenv(e) - if not gete: - sys.exit("Configuration error: Environment variable " + e + " is not defined") - # Set variable - exec(e + " = \"" + gete + "\"") +# Check environment variables are defined +for e in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD] + if not e: + sys.exit("Configuration error: Missing AirTable environment variable.") async def run_checks(): From c3d3f2b0e53d9fa7a103ec6a68cc00d643420352 Mon Sep 17 00:00:00 2001 From: firyx <29089732+firyx@users.noreply.github.com> Date: Wed, 10 Nov 2021 14:32:08 +0100 Subject: [PATCH 12/16] Update setup-scripts/blocklist-airtable.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Karol Wypchło --- setup-scripts/blocklist-airtable.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index d0279e5f..43858483 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -21,8 +21,8 @@ AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE") AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD") # Check environment variables are defined -for e in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD] - if not e: +for value in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD] + if not value: sys.exit("Configuration error: Missing AirTable environment variable.") From e99fdfe4601c16aae18e5cf39d8531dbbc5ce658 Mon Sep 17 00:00:00 2001 From: Filip Rysavy <29089732+firyx@users.noreply.github.com> Date: Wed, 10 Nov 2021 15:10:28 +0100 Subject: [PATCH 13/16] Fix missing colon --- setup-scripts/blocklist-airtable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index d0279e5f..028b18ee 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -21,7 +21,7 @@ AIRTABLE_TABLE = os.getenv("AIRTABLE_TABLE") AIRTABLE_FIELD = os.getenv("AIRTABLE_FIELD") # Check environment variables are defined -for e in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD] +for e in [AIRTABLE_API_KEY, AIRTABLE_BASE, AIRTABLE_TABLE, AIRTABLE_FIELD]: if not e: sys.exit("Configuration error: Missing AirTable environment variable.") From e0520de05e3edcb2a1abc6d613f87f2cb7a0da68 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Thu, 11 Nov 2021 16:59:51 +0100 Subject: [PATCH 14/16] introduce hard upload limits --- docker/nginx/conf.d/server/server.api | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 389a2d7f..34dca82e 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -139,7 +139,7 @@ location /skynet/skyfile { limit_req zone=uploads_by_ip burst=100 nodelay; limit_req zone=uploads_by_ip_throttled; - limit_conn upload_conn 10; + limit_conn upload_conn 2; limit_conn upload_conn_rl 1; client_max_body_size 1000M; # make sure to limit the size of upload to a sane value @@ -174,6 +174,12 @@ location /skynet/tus { include /etc/nginx/conf.d/include/cors-headers; # include cors headers but do not overwrite OPTIONS response include /etc/nginx/conf.d/include/track-upload; + limit_req zone=uploads_by_ip burst=100 nodelay; + limit_req zone=uploads_by_ip_throttled; + + limit_conn upload_conn 2; + limit_conn upload_conn_rl 1; + # TUS chunks size is 40M + leaving 10M of breathing room client_max_body_size 50M; @@ -239,6 +245,12 @@ location /skynet/pin { include /etc/nginx/conf.d/include/track-upload; include /etc/nginx/conf.d/include/generate-siapath; + limit_req zone=uploads_by_ip burst=100 nodelay; + limit_req zone=uploads_by_ip_throttled; + + limit_conn upload_conn 2; + limit_conn upload_conn_rl 1; + proxy_set_header User-Agent: Sia-Agent; proxy_pass http://sia:9980$uri?siapath=$dir1/$dir2/$dir3&$args; } From c0eb41d10d77e8a151f5c36a1b009e4c2a085661 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Thu, 11 Nov 2021 17:17:53 +0100 Subject: [PATCH 15/16] Set burst to 2. --- docker/nginx/conf.d/server/server.api | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 34dca82e..01de2ad5 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -136,7 +136,7 @@ location /skynet/skyfile { include /etc/nginx/conf.d/include/track-upload; include /etc/nginx/conf.d/include/generate-siapath; - limit_req zone=uploads_by_ip burst=100 nodelay; + limit_req zone=uploads_by_ip burst=2 nodelay; limit_req zone=uploads_by_ip_throttled; limit_conn upload_conn 2; @@ -174,7 +174,7 @@ location /skynet/tus { include /etc/nginx/conf.d/include/cors-headers; # include cors headers but do not overwrite OPTIONS response include /etc/nginx/conf.d/include/track-upload; - limit_req zone=uploads_by_ip burst=100 nodelay; + limit_req zone=uploads_by_ip burst=2 nodelay; limit_req zone=uploads_by_ip_throttled; limit_conn upload_conn 2; @@ -245,7 +245,7 @@ location /skynet/pin { include /etc/nginx/conf.d/include/track-upload; include /etc/nginx/conf.d/include/generate-siapath; - limit_req zone=uploads_by_ip burst=100 nodelay; + limit_req zone=uploads_by_ip burst=2 nodelay; limit_req zone=uploads_by_ip_throttled; limit_conn upload_conn 2; From 030b8b94198728537a5ad2cf39d5a8e45e937e13 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Fri, 12 Nov 2021 18:10:38 +0100 Subject: [PATCH 16/16] Set burst to 10, parallel to 5. --- docker/nginx/conf.d/server/server.api | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/nginx/conf.d/server/server.api b/docker/nginx/conf.d/server/server.api index 01de2ad5..07be6fcb 100644 --- a/docker/nginx/conf.d/server/server.api +++ b/docker/nginx/conf.d/server/server.api @@ -136,10 +136,10 @@ location /skynet/skyfile { include /etc/nginx/conf.d/include/track-upload; include /etc/nginx/conf.d/include/generate-siapath; - limit_req zone=uploads_by_ip burst=2 nodelay; + limit_req zone=uploads_by_ip burst=10 nodelay; limit_req zone=uploads_by_ip_throttled; - limit_conn upload_conn 2; + limit_conn upload_conn 5; limit_conn upload_conn_rl 1; client_max_body_size 1000M; # make sure to limit the size of upload to a sane value @@ -174,10 +174,10 @@ location /skynet/tus { include /etc/nginx/conf.d/include/cors-headers; # include cors headers but do not overwrite OPTIONS response include /etc/nginx/conf.d/include/track-upload; - limit_req zone=uploads_by_ip burst=2 nodelay; + limit_req zone=uploads_by_ip burst=10 nodelay; limit_req zone=uploads_by_ip_throttled; - limit_conn upload_conn 2; + limit_conn upload_conn 5; limit_conn upload_conn_rl 1; # TUS chunks size is 40M + leaving 10M of breathing room @@ -245,10 +245,10 @@ location /skynet/pin { include /etc/nginx/conf.d/include/track-upload; include /etc/nginx/conf.d/include/generate-siapath; - limit_req zone=uploads_by_ip burst=2 nodelay; + limit_req zone=uploads_by_ip burst=10 nodelay; limit_req zone=uploads_by_ip_throttled; - limit_conn upload_conn 2; + limit_conn upload_conn 5; limit_conn upload_conn_rl 1; proxy_set_header User-Agent: Sia-Agent;