diff --git a/changelog/items/bugs-fixed/add-missing-blocklist-cmd.md b/changelog/items/bugs-fixed/add-missing-blocklist-cmd.md new file mode 100644 index 00000000..3d72b402 --- /dev/null +++ b/changelog/items/bugs-fixed/add-missing-blocklist-cmd.md @@ -0,0 +1 @@ +- Add missing servers and blocklist command to the manual blocklist script. diff --git a/changelog/items/bugs-fixed/fix-blocklist-skylink.md b/changelog/items/bugs-fixed/fix-blocklist-skylink.md new file mode 100644 index 00000000..b7b6d38d --- /dev/null +++ b/changelog/items/bugs-fixed/fix-blocklist-skylink.md @@ -0,0 +1,2 @@ +- Fix `blocklist-skylink.sh` script that didn't removed blocked skylink from + nginx cache. \ No newline at end of file diff --git a/changelog/items/key-updates/1223-wildcard-api.md b/changelog/items/key-updates/1223-wildcard-api.md new file mode 100644 index 00000000..88ca91d9 --- /dev/null +++ b/changelog/items/key-updates/1223-wildcard-api.md @@ -0,0 +1 @@ +- expose generic skylink serving endpoint on domain aliases diff --git a/changelog/items/other/enable-accounting.md b/changelog/items/other/enable-accounting.md new file mode 100644 index 00000000..4771bbe0 --- /dev/null +++ b/changelog/items/other/enable-accounting.md @@ -0,0 +1 @@ +- Enable the accounting module for skyd diff --git a/changelog/items/other/refactor-blocklist.md b/changelog/items/other/refactor-blocklist.md new file mode 100644 index 00000000..28629dab --- /dev/null +++ b/changelog/items/other/refactor-blocklist.md @@ -0,0 +1,2 @@ +- Remove hardcoded server list from `blocklist-skylink.sh` so it removes server + list duplication and can also be called from Ansible. \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 024300ba..3f94c3ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: restart: unless-stopped logging: *default-logging environment: - - SIA_MODULES=gctwr + - SIA_MODULES=gctwra - MONGODB_URI=mongodb://${SKYNET_DB_HOST}:${SKYNET_DB_PORT} - MONGODB_USER=${SKYNET_DB_USER} - MONGODB_PASSWORD=${SKYNET_DB_PASS} diff --git a/docker/handshake/Dockerfile b/docker/handshake/Dockerfile index 4a3d8bd7..ae640f1b 100644 --- a/docker/handshake/Dockerfile +++ b/docker/handshake/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16.11.1-alpine +FROM node:16.12.0-alpine WORKDIR /opt/hsd diff --git a/docker/nginx/conf.d/include/location-hns b/docker/nginx/conf.d/include/location-hns index bd5644fd..8cc662da 100644 --- a/docker/nginx/conf.d/include/location-hns +++ b/docker/nginx/conf.d/include/location-hns @@ -75,7 +75,7 @@ access_by_lua_block { end } -# we proxy to another nginx location rather than directly to siad because we don't want to deal with caching here +# we proxy to another nginx location rather than directly to siad because we do not want to deal with caching here proxy_pass https://127.0.0.1/$skylink$path$is_args$args; # in case siad returns location header, we need to replace the skylink with the domain name diff --git a/docker/nginx/conf.d/include/location-skylink b/docker/nginx/conf.d/include/location-skylink index 96dfacef..a3692d00 100644 --- a/docker/nginx/conf.d/include/location-skylink +++ b/docker/nginx/conf.d/include/location-skylink @@ -12,7 +12,7 @@ if ($request_method = PURGE) { limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time # $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was: -# a) skylink v1 - it wouldn't matter, no additional logic is executed +# a) skylink v1 - it would not matter, no additional logic is executed # b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update # $skylink_v1 variable so then the proxy request to skyd can be cached in nginx (proxy_cache_key # in proxy-cache-downloads includes $skylink_v1 as a part of the cache key) @@ -91,7 +91,7 @@ limit_rate $limit_rate; proxy_read_timeout 600; proxy_set_header User-Agent: Sia-Agent; -# in case the requested skylink was v2 and we already resolved it to skylink v1, we're going to pass resolved +# in case the requested skylink was v2 and we already resolved it to skylink v1, we are going to pass resolved # skylink v1 to skyd to save that extra skylink v2 lookup in skyd but in turn, in case skyd returns a redirect # we need to rewrite the skylink v1 to skylink v2 in the location header with proxy_redirect proxy_redirect $skylink_v1 $skylink_v2; diff --git a/docker/nginx/conf.d/server.dnslink.conf b/docker/nginx/conf.d/server.dnslink.conf index 8a051d3f..491bc389 100644 --- a/docker/nginx/conf.d/server.dnslink.conf +++ b/docker/nginx/conf.d/server.dnslink.conf @@ -1,3 +1,5 @@ +lua_shared_dict dnslink 10m; + server { listen 80 default_server; listen [::]:80 default_server; diff --git a/docker/nginx/conf.d/server/server.dnslink b/docker/nginx/conf.d/server/server.dnslink index db8948e3..1dd3a489 100644 --- a/docker/nginx/conf.d/server/server.dnslink +++ b/docker/nginx/conf.d/server/server.dnslink @@ -5,21 +5,40 @@ location / { set $path $uri; rewrite_by_lua_block { - local httpc = require("resty.http").new() + local cache = ngx.shared.dnslink + local cache_value = cache:get(ngx.var.host) - -- 10.10.10.55 points to dnslink-api service (alias not available when using resty-http) - local res, err = httpc:request_uri("http://10.10.10.55:3100/dnslink/" .. ngx.var.host) + if cache_value == nil then + local httpc = require("resty.http").new() - if err or (res and res.status ~= ngx.HTTP_OK) then - ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status - ngx.header["content-type"] = "text/plain" - ngx.say(err or res.body) - ngx.exit(ngx.status) + -- 10.10.10.55 points to dnslink-api service (alias not available when using resty-http) + local res, err = httpc:request_uri("http://10.10.10.55:3100/dnslink/" .. ngx.var.host) + + if err or (res and res.status ~= ngx.HTTP_OK) then + -- check whether we can fallback to regular skylink request + local match_skylink = ngx.re.match(ngx.var.uri, "^/([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?") + + if match_skylink then + ngx.var.skylink = match_skylink[1] + ngx.var.path = match_skylink[2] or "/" + else + ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status + ngx.header["content-type"] = "text/plain" + ngx.say(err or res.body) + ngx.exit(ngx.status) + end + else + ngx.var.skylink = res.body + + local cache_ttl = 300 -- 5 minutes cache expire time + cache:set(ngx.var.host, ngx.var.skylink, cache_ttl) + end else - ngx.var.skylink = res.body - ngx.var.skylink_v1 = ngx.var.skylink - ngx.var.skylink_v2 = ngx.var.skylink + ngx.var.skylink = cache_value end + + ngx.var.skylink_v1 = ngx.var.skylink + ngx.var.skylink_v2 = ngx.var.skylink } include /etc/nginx/conf.d/include/location-skylink; diff --git a/packages/dashboard/Dockerfile b/packages/dashboard/Dockerfile index 8b2cfc32..03cbb74f 100644 --- a/packages/dashboard/Dockerfile +++ b/packages/dashboard/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16.11.1-alpine +FROM node:16.12.0-alpine WORKDIR /usr/app diff --git a/packages/dashboard/package.json b/packages/dashboard/package.json index ed4adefd..48fb2c50 100644 --- a/packages/dashboard/package.json +++ b/packages/dashboard/package.json @@ -11,7 +11,7 @@ "@fontsource/metropolis": "4.5.0", "@ory/kratos-client": "0.5.4-alpha.1", "@stripe/react-stripe-js": "1.6.0", - "@stripe/stripe-js": "1.20.2", + "@stripe/stripe-js": "1.20.3", "@tailwindcss/forms": "0.3.4", "autoprefixer": "10.3.7", "classnames": "2.3.1", @@ -24,17 +24,17 @@ "ky": "0.25.1", "next": "11.1.2", "normalize.css": "8.0.1", - "postcss": "8.3.9", + "postcss": "8.3.11", "prettier": "2.4.1", "pretty-bytes": "5.6.0", "react": "17.0.2", "react-dom": "17.0.2", "react-toastify": "8.0.3", "skynet-js": "3.0.2", - "stripe": "8.183.0", + "stripe": "8.184.0", "superagent": "6.1.0", "swr": "1.0.1", - "tailwindcss": "2.2.16", + "tailwindcss": "2.2.17", "yup": "0.32.11" }, "resolutions": { diff --git a/packages/dashboard/yarn.lock b/packages/dashboard/yarn.lock index 329476e4..2cc26a35 100644 --- a/packages/dashboard/yarn.lock +++ b/packages/dashboard/yarn.lock @@ -188,10 +188,10 @@ dependencies: prop-types "^15.7.2" -"@stripe/stripe-js@1.20.2": - version "1.20.2" - resolved "https://registry.yarnpkg.com/@stripe/stripe-js/-/stripe-js-1.20.2.tgz#5bb8deff1ba6cdc83d0bfac58e4d61fe2e4a53e7" - integrity sha512-gTPpXQgAgKrAAvV2R5WO4NCuozALD4n3xaDLhPv+nAomIdPLtgEwI2xIGY7WMSKp9KfkKmlcjoPO4pkz8f/bpw== +"@stripe/stripe-js@1.20.3": + version "1.20.3" + resolved "https://registry.yarnpkg.com/@stripe/stripe-js/-/stripe-js-1.20.3.tgz#6f479d53ad933b8b17b6c708bbc2840b7512d80e" + integrity sha512-pFhPvajrZJUzoOZDC3/2YsOeL5VD63CVlbOXs+AriIchmy6Kb2CQqrcHPCuK72FM726lwo4neF0wPRBtsLYXuw== "@tailwindcss/forms@0.3.4": version "0.3.4" @@ -1947,10 +1947,10 @@ nanoclone@^0.2.1: resolved "https://registry.yarnpkg.com/nanoclone/-/nanoclone-0.2.1.tgz#dd4090f8f1a110d26bb32c49ed2f5b9235209ed4" integrity sha512-wynEP02LmIbLpcYw8uBKpcfF6dmg2vcpKqxeH5UcoKEYdExslsdUA4ugFauuaeYdTB76ez6gJW8XAZ6CgkXYxA== -nanoid@^3.1.23, nanoid@^3.1.28: - version "3.1.29" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.29.tgz#214fb2d7a33e1a5bef4757b779dfaeb6a4e5aeb4" - integrity sha512-dW2pUSGZ8ZnCFIlBIA31SV8huOGCHb6OwzVCc7A69rb/a+SgPBwfmLvK5TKQ3INPbRkcI8a/Owo0XbiTNH19wg== +nanoid@^3.1.23, nanoid@^3.1.30: + version "3.1.30" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.30.tgz#63f93cc548d2a113dc5dfbc63bfa09e2b9b64362" + integrity sha512-zJpuPDwOv8D2zq2WRoMe1HsfZthVewpel9CAvTfc/2mBD1uUT/agc5f7GHGWXlYkFvi1mVxe4IjvP2HNrop7nQ== native-url@0.3.4: version "0.3.4" @@ -2270,6 +2270,11 @@ picocolors@^0.2.1: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f" integrity sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA== +picocolors@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" + integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== + picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3: version "2.3.0" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" @@ -2345,13 +2350,13 @@ postcss@8.2.15: nanoid "^3.1.23" source-map "^0.6.1" -postcss@8.3.9, postcss@^8.1.6, postcss@^8.2.1: - version "8.3.9" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.9.tgz#98754caa06c4ee9eb59cc48bd073bb6bd3437c31" - integrity sha512-f/ZFyAKh9Dnqytx5X62jgjhhzttjZS7hMsohcI7HEI5tjELX/HxCy3EFhsRxyzGvrzFF+82XPvCS8T9TFleVJw== +postcss@8.3.11, postcss@^8.1.6, postcss@^8.2.1: + version "8.3.11" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.11.tgz#c3beca7ea811cd5e1c4a3ec6d2e7599ef1f8f858" + integrity sha512-hCmlUAIlUiav8Xdqw3Io4LcpA1DOt7h3LSTAC4G6JGHFFaWzI6qvFt9oilvl8BmkbBRX1IhM90ZAmpk68zccQA== dependencies: - nanoid "^3.1.28" - picocolors "^0.2.1" + nanoid "^3.1.30" + picocolors "^1.0.0" source-map-js "^0.6.2" prettier@2.4.1: @@ -2905,10 +2910,10 @@ strip-eof@^1.0.0: resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf" integrity sha1-u0P/VZim6wXYm1n80SnJgzE2Br8= -stripe@8.183.0: - version "8.183.0" - resolved "https://registry.yarnpkg.com/stripe/-/stripe-8.183.0.tgz#3dc183fafff618526a576a0d9972483b08075e06" - integrity sha512-QcM3nimH1CuP49VPPRt36Wgfu4QoS+Wm0eyGMis7Ej+seWFKqMffMdx7TE2gn8dVsJIOA1kDuIbAQGqpZHozGA== +stripe@8.184.0: + version "8.184.0" + resolved "https://registry.yarnpkg.com/stripe/-/stripe-8.184.0.tgz#ea68470ca6045bb47516c4fea3b775fd6ce15a36" + integrity sha512-ZUdvyX+sizTxXLEbUjgTShrulSWSkMIt7hIKdAkhnajYrHdzVtdmhBJl8sQbR9chMVox3Ig5ohilyeIrvcCE2g== dependencies: "@types/node" ">=8.1.0" qs "^6.6.0" @@ -2982,10 +2987,10 @@ swr@1.0.1: dependencies: dequal "2.0.2" -tailwindcss@2.2.16: - version "2.2.16" - resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-2.2.16.tgz#32f81bdf1758b639cb83b9d30bf7cbecdda49e5e" - integrity sha512-EireCtpQyyJ4Xz8NYzHafBoy4baCOO96flM0+HgtsFcIQ9KFy/YBK3GEtlnD+rXen0e4xm8t3WiUcKBJmN6yjg== +tailwindcss@2.2.17: + version "2.2.17" + resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-2.2.17.tgz#c6332731f9ff1b6628ff589c95c38685347775e3" + integrity sha512-WgRpn+Pxn7eWqlruxnxEbL9ByVRWi3iC10z4b6dW0zSdnkPVC4hPMSWLQkkW8GCyBIv/vbJ0bxIi9dVrl4CfoA== dependencies: arg "^5.0.1" bytes "^3.0.0" diff --git a/packages/dnslink-api/Dockerfile b/packages/dnslink-api/Dockerfile index 903c6757..85c00b9f 100644 --- a/packages/dnslink-api/Dockerfile +++ b/packages/dnslink-api/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16.11.1-alpine +FROM node:16.12.0-alpine WORKDIR /usr/app diff --git a/packages/dnslink-api/package.json b/packages/dnslink-api/package.json index a29295e1..7f833ed2 100644 --- a/packages/dnslink-api/package.json +++ b/packages/dnslink-api/package.json @@ -5,8 +5,7 @@ "license": "SEE LICENSE IN LICENSE.md", "dependencies": { "express": "^4.17.1", - "is-valid-domain": "^0.1.2", - "node-cache": "^5.1.2" + "is-valid-domain": "^0.1.2" }, "devDependencies": { "prettier": "^2.4.1" diff --git a/packages/dnslink-api/src/index.js b/packages/dnslink-api/src/index.js index fdddf9aa..300277c2 100644 --- a/packages/dnslink-api/src/index.js +++ b/packages/dnslink-api/src/index.js @@ -1,14 +1,11 @@ const dns = require("dns"); const express = require("express"); -const NodeCache = require("node-cache"); const isValidDomain = require("is-valid-domain"); const host = process.env.DNSLINK_API_HOSTNAME || "0.0.0.0"; const port = Number(process.env.DNSLINK_API_PORT) || 3100; -const cacheTTL = Number(process.env.DNSLINK_API_CACHE_TTL) || 300; // default to 5 minutes const server = express(); -const cache = new NodeCache({ stdTTL: cacheTTL }); const dnslinkNamespace = "skynet-ns"; const dnslinkRegExp = new RegExp(`^dnslink=/${dnslinkNamespace}/.+$`); @@ -23,10 +20,6 @@ server.get("/dnslink/:name", async (req, res) => { return failure(`"${req.params.name}" is not a valid domain`); } - if (cache.has(req.params.name)) { - return success(cache.get(req.params.name)); - } - const lookup = `_dnslink.${req.params.name}`; dns.resolveTxt(lookup, (error, records) => { @@ -65,8 +58,6 @@ server.get("/dnslink/:name", async (req, res) => { const skylink = matchSkylink[1]; - cache.set(req.params.name, skylink); - console.log(`${req.params.name} => ${skylink}`); return success(skylink); diff --git a/packages/dnslink-api/yarn.lock b/packages/dnslink-api/yarn.lock index 4a359f7a..5cff1dda 100644 --- a/packages/dnslink-api/yarn.lock +++ b/packages/dnslink-api/yarn.lock @@ -36,11 +36,6 @@ bytes@3.1.0: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg== -clone@2.x: - version "2.1.2" - resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" - integrity sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18= - content-disposition@0.5.3: version "0.5.3" resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd" @@ -257,13 +252,6 @@ negotiator@0.6.2: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== -node-cache@^5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/node-cache/-/node-cache-5.1.2.tgz#f264dc2ccad0a780e76253a694e9fd0ed19c398d" - integrity sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg== - dependencies: - clone "2.x" - on-finished@~2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" diff --git a/packages/handshake-api/Dockerfile b/packages/handshake-api/Dockerfile index 2ed56e43..bceb1804 100644 --- a/packages/handshake-api/Dockerfile +++ b/packages/handshake-api/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16.11.1-alpine +FROM node:16.12.0-alpine WORKDIR /usr/app diff --git a/packages/health-check/Dockerfile b/packages/health-check/Dockerfile index dbf0a774..db117f1b 100644 --- a/packages/health-check/Dockerfile +++ b/packages/health-check/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16.11.1-alpine +FROM node:16.12.0-alpine RUN apk update && apk add dnsmasq diff --git a/packages/website/Dockerfile b/packages/website/Dockerfile index d67627e1..ae025cb2 100644 --- a/packages/website/Dockerfile +++ b/packages/website/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16.11.1-alpine +FROM node:16.12.0-alpine RUN apk update && apk add autoconf automake build-base libtool nasm pkgconfig diff --git a/packages/website/package.json b/packages/website/package.json index d147dadd..b980a8c3 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -15,7 +15,7 @@ "copy-text-to-clipboard": "^3.0.1", "crypto-browserify": "^3.12.0", "framer-motion": "^4.1.17", - "gatsby": "^3.14.3", + "gatsby": "^3.14.4", "gatsby-background-image": "^1.5.3", "gatsby-image": "^3.11.0", "gatsby-plugin-image": "^1.14.1", @@ -25,7 +25,7 @@ "gatsby-plugin-postcss": "^4.14.0", "gatsby-plugin-purgecss": "^6.0.2", "gatsby-plugin-react-helmet": "^4.14.0", - "gatsby-plugin-robots-txt": "^1.6.10", + "gatsby-plugin-robots-txt": "^1.6.13", "gatsby-plugin-sharp": "^3.14.1", "gatsby-plugin-svgr": "^3.0.0-beta.0", "gatsby-remark-classes": "^1.0.2", @@ -47,8 +47,8 @@ "normalize.css": "^8.0.1", "path-browserify": "^1.0.1", "polished": "^4.1.3", - "popmotion": "^10.0.1", - "postcss": "^8.3.9", + "popmotion": "^10.0.2", + "postcss": "^8.3.11", "preact-svg-loader": "^0.2.1", "prop-types": "^15.7.2", "react": "^17.0.2", diff --git a/packages/website/yarn.lock b/packages/website/yarn.lock index f5f36f91..54a0a289 100644 --- a/packages/website/yarn.lock +++ b/packages/website/yarn.lock @@ -5875,10 +5875,10 @@ framesync@5.3.0: dependencies: tslib "^2.1.0" -framesync@6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/framesync/-/framesync-6.0.0.tgz#e6ad8ec128d33291a03cc34e58365c41265a1dae" - integrity sha512-9iBw/uZ/5fDURdpLrgc/eoFXiX9HC3DeOSLtTL5lZSyX/vQb+kjEz9CPNTiTObfK5PKyGgIUTstU8PK9W6EvoA== +framesync@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/framesync/-/framesync-6.0.1.tgz#5e32fc01f1c42b39c654c35b16440e07a25d6f20" + integrity sha512-fUY88kXvGiIItgNC7wcTOl0SNRCVXMKSWW2Yzfmn7EKNc+MpCzcz9DhdHcdjbrtN3c6R4H5dTY2jiCpPdysEjA== dependencies: tslib "^2.1.0" @@ -6159,10 +6159,10 @@ gatsby-plugin-react-helmet@^4.14.0: dependencies: "@babel/runtime" "^7.15.4" -gatsby-plugin-robots-txt@^1.6.10: - version "1.6.10" - resolved "https://registry.yarnpkg.com/gatsby-plugin-robots-txt/-/gatsby-plugin-robots-txt-1.6.10.tgz#b178efe3da65718a39d1acd2e5768f687b205cf0" - integrity sha512-soQT765LF0J8/dfrZGFuA/ZB/JUyvt2nVyHEFLUMd/qxgiem9x0EOZquJPId78xDHYePMkxNCPk9UsLcTZdFZw== +gatsby-plugin-robots-txt@^1.6.13: + version "1.6.13" + resolved "https://registry.yarnpkg.com/gatsby-plugin-robots-txt/-/gatsby-plugin-robots-txt-1.6.13.tgz#01e1ce68e3f4e07f957ac9a20cd2a9e12fdd3f79" + integrity sha512-MUPJsvkALwSmfRb1L3IVNmzIqwV35fol1RVot425ZYXvtD+s0OtZli5VnhLNxsX0beodyavzFl0SRJCCQZz0/g== dependencies: "@babel/runtime" "^7.14.0" generate-robotstxt "^8.0.3" @@ -6464,10 +6464,10 @@ gatsby-worker@^0.5.0: "@babel/core" "^7.15.5" "@babel/runtime" "^7.15.4" -gatsby@^3.14.3: - version "3.14.3" - resolved "https://registry.yarnpkg.com/gatsby/-/gatsby-3.14.3.tgz#62bc020d0997f4b30e419abe644f60483f99e8b5" - integrity sha512-B6NmmIvGGVNktky0YGugxMJlZAbKEzGMMBA8ojbaDhfEzfNXxsyHYd0aHKL0fNwPEqMKUsllaACdXrw6CWRk2g== +gatsby@^3.14.4: + version "3.14.4" + resolved "https://registry.yarnpkg.com/gatsby/-/gatsby-3.14.4.tgz#e7ef5c3c1eda45ea3ad0bc1273f014fbc46b316c" + integrity sha512-cpsMhl2qkE0+TL0znwflVfycy143fFucGtFDkNxu8dbY0GRjA9MiDwlS/aSqg5U2HhhF39IcBvwuntiaP6vQ0g== dependencies: "@babel/code-frame" "^7.14.0" "@babel/core" "^7.15.5" @@ -9405,7 +9405,7 @@ nanocolors@^0.2.2: resolved "https://registry.yarnpkg.com/nanocolors/-/nanocolors-0.2.10.tgz#a712df4d3c1bf12d9b4fb8b5aa61b5ba31337503" integrity sha512-i+EDWGsJClQwR/bhLIG/CObZZwaYaS5qt+yjxZbfV+77QiNHNzE9nj4d9Ut1TGZ0R0eSwPcQWzReASzXuw/7oA== -nanoid@^3.1.28, nanoid@^3.1.30: +nanoid@^3.1.30: version "3.1.30" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.30.tgz#63f93cc548d2a113dc5dfbc63bfa09e2b9b64362" integrity sha512-zJpuPDwOv8D2zq2WRoMe1HsfZthVewpel9CAvTfc/2mBD1uUT/agc5f7GHGWXlYkFvi1mVxe4IjvP2HNrop7nQ== @@ -10203,6 +10203,11 @@ picocolors@^0.2.1: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f" integrity sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA== +picocolors@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" + integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== + picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3: version "2.3.0" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" @@ -10285,12 +10290,12 @@ popmotion@9.3.6: style-value-types "4.1.4" tslib "^2.1.0" -popmotion@^10.0.1: - version "10.0.1" - resolved "https://registry.yarnpkg.com/popmotion/-/popmotion-10.0.1.tgz#5be5b80112a1386b7f2a11d081a1ea3ca7508d8e" - integrity sha512-jmVmbXMAl/qrrogQXCMGFR8+F290J6gl9kKHSYzAKBRTHm7B1unadmEeLSXX7rGtzPxOfKVnWHA3t3m3CVnxqw== +popmotion@^10.0.2: + version "10.0.2" + resolved "https://registry.yarnpkg.com/popmotion/-/popmotion-10.0.2.tgz#edb87efb2ba39505d07f79d3ca8cc4b1cbe101bf" + integrity sha512-8vgx02LL7Odk/kwSGigAnlPRLJK+Dfked571ZAHGzkxPNWThkU5mQX0xPmibIMAULZehWBV7hGN7LSCpaV3cfw== dependencies: - framesync "6.0.0" + framesync "^6.0.1" hey-listen "^1.0.8" style-value-types "5.0.0" tslib "^2.1.0" @@ -10616,13 +10621,13 @@ postcss@^7.0.27: nanocolors "^0.2.2" source-map "^0.6.1" -postcss@^8.1.6, postcss@^8.2.1, postcss@^8.2.15, postcss@^8.2.9, postcss@^8.3.5, postcss@^8.3.9: - version "8.3.9" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.9.tgz#98754caa06c4ee9eb59cc48bd073bb6bd3437c31" - integrity sha512-f/ZFyAKh9Dnqytx5X62jgjhhzttjZS7hMsohcI7HEI5tjELX/HxCy3EFhsRxyzGvrzFF+82XPvCS8T9TFleVJw== +postcss@^8.1.6, postcss@^8.2.1, postcss@^8.2.15, postcss@^8.2.9, postcss@^8.3.11, postcss@^8.3.5: + version "8.3.11" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.11.tgz#c3beca7ea811cd5e1c4a3ec6d2e7599ef1f8f858" + integrity sha512-hCmlUAIlUiav8Xdqw3Io4LcpA1DOt7h3LSTAC4G6JGHFFaWzI6qvFt9oilvl8BmkbBRX1IhM90ZAmpk68zccQA== dependencies: - nanoid "^3.1.28" - picocolors "^0.2.1" + nanoid "^3.1.30" + picocolors "^1.0.0" source-map-js "^0.6.2" potrace@^2.1.8: diff --git a/scripts/blocklist-skylink.sh b/scripts/blocklist-skylink.sh index 80885b14..8b81e09f 100755 --- a/scripts/blocklist-skylink.sh +++ b/scripts/blocklist-skylink.sh @@ -1,9 +1,10 @@ #! /usr/bin/env bash -# This script is meant to be used when manually adding a skylink to the -# blocklist on all the skynet web portals. The automatic script that is used to -# continuously sync a google sheets list with the blocklist on the web portals -# is /setup-scripts/blocklist-airtable.py +# This script adds a skylink to the sia blocklist and removes the skylink from +# nginx cache. The script should be run locally on each skynet webportal +# server. The automatic script that is used to continuously sync an Airtable +# sheet list with the blocklist on the web portals is +# /setup-scripts/blocklist-airtable.py set -e # exit on first error @@ -17,44 +18,39 @@ fi ######################################################### skylinks=() if test -f "$1"; then - OLDIFS=$IFS - IFS=',' line_number=1 - while read line + + # Read file including the last line even when it doesn't end with newline + while IFS="" read -r line || [ -n "$line" ]; do - if [[ $line =~ ([a-zA-Z0-9_-]{46}) ]]; then - skylinks+=("$BASH_REMATCH") + if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then + skylinks+=("$line") else echo "Incorrect skylink at line ${line_number}: $line" && exit 1 fi let line_number+=1 done < $1; - IFS=$OLDIFS else skylinks=("$1") # just single skylink passed as input argument fi -######################################################################### -# iterate through all servers, block the skylinks and purge it from cache -######################################################################### -declare -a servers=( "eu-ger-1.siasky.net" "eu-ger-2.siasky.net" "eu-ger-3.siasky.net" "eu-ger-4.siasky.net" "eu-ger-5.siasky.net" "eu-ger-6.siasky.net" "eu-ger-7.siasky.net" "eu-ger-8.siasky.net" - "eu-fin-1.siasky.net" "eu-fin-2.siasky.net" "eu-fin-3.siasky.net" "eu-fin-4.siasky.net" - "eu-pol-1.siasky.net" "eu-pol-2.siasky.net" "eu-pol-3.siasky.net" - "us-or-1.siasky.net" "us-or-2.siasky.net" - "us-pa-1.siasky.net" "us-pa-2.siasky.net" - "us-va-1.siasky.net" "us-va-2.siasky.net" "us-va-3.siasky.net" - "as-hk-1.siasky.net" - "siasky.xyz" "dev1.siasky.dev" "dev2.siasky.dev" "dev3.siasky.dev") -for server in "${servers[@]}"; +for skylink in "${skylinks[@]}"; do - for skylink in "${skylinks[@]}"; - do - echo ".. ⌁ Blocking skylink ${skylink} on ${server}" - cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Elsq '^Skynet-Skylink: ${skylink}'" - ssh -q -t user@${server} "docker exec -it nginx bash -c ${cached_files_command} | xargs -r rm" - echo ".. ⌁ Skylink ${skylink} Blocked on ${server}" - echo "--------------------------------------------" - done + echo ".. ⌁ Blocking skylink ${skylink}" + + # Add to Sia blocklist + docker exec sia siac skynet blocklist add "${skylink}" + + # Remove from NGINX cache + # NOTE: + # If there are changes to how the NGINX cache is being cleared, the same + # changes need to be applied to the /setup-scripts/blocklist-airtable.py + # script. + cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'" + docker exec -it nginx bash -c "${cached_files_command}" | xargs -r rm + + echo ".. ⌁ Skylink ${skylink} Blocked" + echo "--------------------------------------------" done echo "✓ All done !" diff --git a/scripts/db_backup.sh b/scripts/db_backup.sh index 14f7abb1..32e042ce 100755 --- a/scripts/db_backup.sh +++ b/scripts/db_backup.sh @@ -27,26 +27,6 @@ fi # Take the current datetime: DT=$(date +%Y-%m-%d) -### COCKROACH DB ### -echo "Creating a backup of CockroachDB:" -# Check if a backup already exists: -totalFoundObjects=$(aws s3 ls $S3_BACKUP_PATH/$DT --recursive --summarize | grep "cockroach" | wc -l) -if [ "$totalFoundObjects" -ge "1" ]; then - echo "Backup already exists for today. Skipping." -else - # Create a cockroachdb backup: - docker exec cockroach \ - cockroach sql \ - --host cockroach:26257 \ - --certs-dir=/certs \ - --execute="BACKUP TO '$S3_BACKUP_PATH/$DT/cockroach/?AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID&AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY';" - if [[ $? > 0 ]]; then - echo "Creating a CockroachDB backup failed. Skipping." - else - echo "Successfully backed up CockroachDB." - fi -fi - ### MONGO DB ### echo "Creating a backup of MongoDB:" # Check if a backup already exists: @@ -73,3 +53,23 @@ else fi docker exec mongo rm -rf /data/db/backups/$DT fi + +### COCKROACH DB ### +echo "Creating a backup of CockroachDB:" +# Check if a backup already exists: +totalFoundObjects=$(aws s3 ls $S3_BACKUP_PATH/$DT --recursive --summarize | grep "cockroach" | wc -l) +if [ "$totalFoundObjects" -ge "1" ]; then + echo "Backup already exists for today. Skipping." +else + # Create a cockroachdb backup: + docker exec cockroach \ + cockroach sql \ + --host cockroach:26257 \ + --certs-dir=/certs \ + --execute="BACKUP TO '$S3_BACKUP_PATH/$DT/cockroach/?AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID&AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY';" + if [[ $? > 0 ]]; then + echo "Creating a CockroachDB backup failed. Skipping." + else + echo "Successfully backed up CockroachDB." + fi +fi diff --git a/scripts/db_restore.sh b/scripts/db_restore.sh index 79e6e0f7..f19def85 100755 --- a/scripts/db_restore.sh +++ b/scripts/db_restore.sh @@ -1,5 +1,8 @@ #!/bin/bash +# First of all, let's pamper awscli because Python is so special: +pip3 install --upgrade awscli + BACKUP=$1 if [[ $BACKUP == "" ]]; then echo "No backup name given. It should look like '2020-01-29'." @@ -97,7 +100,7 @@ rm mongo.tgz # The name of the backup is not `mongo` due to the way we're creating it, # it's $BACKUP. docker exec mongo \ - mongorestore \ + mongorestore --drop \ mongodb://$SKYNET_DB_USER:$SKYNET_DB_PASS@$SKYNET_DB_HOST:$SKYNET_DB_PORT \ /data/db/backups/to_restore/$BACKUP # Clean up: diff --git a/setup-scripts/blocklist-airtable.py b/setup-scripts/blocklist-airtable.py index 1d4c28ce..3dbce3ed 100755 --- a/setup-scripts/blocklist-airtable.py +++ b/setup-scripts/blocklist-airtable.py @@ -141,6 +141,10 @@ async def block_skylinks_from_airtable(): ) return await send_msg(message, force_notify=False) + # Remove from NGINX cache + # NOTE: + # If there are changes to how the NGINX cache is being cleared, the same + # changes need to be applied to the /scripts/blocklist-skylink.sh script. print("Searching nginx cache for blocked files") cached_files_count = 0 batch_size = 1000