LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into portal-latest

This commit is contained in:
Matthew Sevey 2021-10-26 12:37:13 -04:00
parent fc51d6e2fc 5f6d984566
commit 60b11ede16
No known key found for this signature in database
GPG Key ID: 9ADDD344F13057F6
27 changed files with 168 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog/items/bugs-fixed/add-missing-blocklist-cmd.md Normal file
View File

@ -0,0 +1 @@
- Add missing servers and blocklist command to the manual blocklist script.

2
changelog/items/bugs-fixed/fix-blocklist-skylink.md Normal file
View File

@ -0,0 +1,2 @@
- Fix `blocklist-skylink.sh` script that didn't removed blocked skylink from
nginx cache.

1
changelog/items/key-updates/1223-wildcard-api.md Normal file
View File

@ -0,0 +1 @@
- expose generic skylink serving endpoint on domain aliases

1
changelog/items/other/enable-accounting.md Normal file
View File

@ -0,0 +1 @@
- Enable the accounting module for skyd

2
changelog/items/other/refactor-blocklist.md Normal file
View File

@ -0,0 +1,2 @@
- Remove hardcoded server list from `blocklist-skylink.sh` so it removes server
list duplication and can also be called from Ansible.

2
docker-compose.yml
View File

@ -24,7 +24,7 @@ services:
restart: unless-stopped
logging: *default-logging
environment:
- SIA_MODULES=gctwr
- SIA_MODULES=gctwra
- MONGODB_URI=mongodb://${SKYNET_DB_HOST}:${SKYNET_DB_PORT}
- MONGODB_USER=${SKYNET_DB_USER}
- MONGODB_PASSWORD=${SKYNET_DB_PASS}

2
docker/handshake/Dockerfile
View File

@ -1,4 +1,4 @@
FROM node:16.11.1-alpine
FROM node:16.12.0-alpine
WORKDIR /opt/hsd

2
docker/nginx/conf.d/include/location-hns
View File

@ -75,7 +75,7 @@ access_by_lua_block {
end
}
# we proxy to another nginx location rather than directly to siad because we don't want to deal with caching here
# we proxy to another nginx location rather than directly to siad because we do not want to deal with caching here
proxy_pass https://127.0.0.1/$skylink$path$is_args$args;
# in case siad returns location header, we need to replace the skylink with the domain name

4
docker/nginx/conf.d/include/location-skylink
View File

@ -12,7 +12,7 @@ if ($request_method = PURGE) {
limit_conn downloads_by_ip 100; # ddos protection: max 100 downloads at a time
# $skylink_v1 and $skylink_v2 variables default to the same value but in case the requested skylink was:
# a) skylink v1 - it wouldn't matter, no additional logic is executed
# a) skylink v1 - it would not matter, no additional logic is executed
# b) skylink v2 - in a lua block below we will resolve the skylink v2 into skylink v1 and update
# $skylink_v1 variable so then the proxy request to skyd can be cached in nginx (proxy_cache_key
# in proxy-cache-downloads includes $skylink_v1 as a part of the cache key)
@ -91,7 +91,7 @@ limit_rate $limit_rate;
proxy_read_timeout 600;
proxy_set_header User-Agent: Sia-Agent;
# in case the requested skylink was v2 and we already resolved it to skylink v1, we're going to pass resolved
# in case the requested skylink was v2 and we already resolved it to skylink v1, we are going to pass resolved
# skylink v1 to skyd to save that extra skylink v2 lookup in skyd but in turn, in case skyd returns a redirect
# we need to rewrite the skylink v1 to skylink v2 in the location header with proxy_redirect
proxy_redirect $skylink_v1 $skylink_v2;

2
docker/nginx/conf.d/server.dnslink.conf
View File

@ -1,3 +1,5 @@
lua_shared_dict dnslink 10m;
server {
listen 80 default_server;
listen [::]:80 default_server;

41
docker/nginx/conf.d/server/server.dnslink
View File

@ -5,21 +5,40 @@ location / {
set $path $uri;
rewrite_by_lua_block {
local httpc = require("resty.http").new()
local cache = ngx.shared.dnslink
local cache_value = cache:get(ngx.var.host)
-- 10.10.10.55 points to dnslink-api service (alias not available when using resty-http)
local res, err = httpc:request_uri("http://10.10.10.55:3100/dnslink/" .. ngx.var.host)
if cache_value == nil then
local httpc = require("resty.http").new()
if err or (res and res.status ~= ngx.HTTP_OK) then
ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
ngx.header["content-type"] = "text/plain"
ngx.say(err or res.body)
ngx.exit(ngx.status)
-- 10.10.10.55 points to dnslink-api service (alias not available when using resty-http)
local res, err = httpc:request_uri("http://10.10.10.55:3100/dnslink/" .. ngx.var.host)
if err or (res and res.status ~= ngx.HTTP_OK) then
-- check whether we can fallback to regular skylink request
local match_skylink = ngx.re.match(ngx.var.uri, "^/([a-zA-Z0-9-_]{46}|[a-z0-9]{55})(/.*)?")
if match_skylink then
ngx.var.skylink = match_skylink[1]
ngx.var.path = match_skylink[2] or "/"
else
ngx.status = (err and ngx.HTTP_INTERNAL_SERVER_ERROR) or res.status
ngx.header["content-type"] = "text/plain"
ngx.say(err or res.body)
ngx.exit(ngx.status)
end
else
ngx.var.skylink = res.body
local cache_ttl = 300 -- 5 minutes cache expire time
cache:set(ngx.var.host, ngx.var.skylink, cache_ttl)
end
else
ngx.var.skylink = res.body
ngx.var.skylink_v1 = ngx.var.skylink
ngx.var.skylink_v2 = ngx.var.skylink
ngx.var.skylink = cache_value
end
ngx.var.skylink_v1 = ngx.var.skylink
ngx.var.skylink_v2 = ngx.var.skylink
}
include /etc/nginx/conf.d/include/location-skylink;

2
packages/dashboard/Dockerfile
View File

@ -1,4 +1,4 @@
FROM node:16.11.1-alpine
FROM node:16.12.0-alpine
WORKDIR /usr/app

8
packages/dashboard/package.json
View File

@ -11,7 +11,7 @@
"@fontsource/metropolis": "4.5.0",
"@ory/kratos-client": "0.5.4-alpha.1",
"@stripe/react-stripe-js": "1.6.0",
"@stripe/stripe-js": "1.20.2",
"@stripe/stripe-js": "1.20.3",
"@tailwindcss/forms": "0.3.4",
"autoprefixer": "10.3.7",
"classnames": "2.3.1",
@ -24,17 +24,17 @@
"ky": "0.25.1",
"next": "11.1.2",
"normalize.css": "8.0.1",
"postcss": "8.3.9",
"postcss": "8.3.11",
"prettier": "2.4.1",
"pretty-bytes": "5.6.0",
"react": "17.0.2",
"react-dom": "17.0.2",
"react-toastify": "8.0.3",
"skynet-js": "3.0.2",
"stripe": "8.183.0",
"stripe": "8.184.0",
"superagent": "6.1.0",
"swr": "1.0.1",
"tailwindcss": "2.2.16",
"tailwindcss": "2.2.17",
"yup": "0.32.11"
},
"resolutions": {

49
packages/dashboard/yarn.lock
View File

@ -188,10 +188,10 @@
dependencies:
prop-types "^15.7.2"
"@stripe/stripe-js@1.20.2":
version "1.20.2"
resolved "https://registry.yarnpkg.com/@stripe/stripe-js/-/stripe-js-1.20.2.tgz#5bb8deff1ba6cdc83d0bfac58e4d61fe2e4a53e7"
integrity sha512-gTPpXQgAgKrAAvV2R5WO4NCuozALD4n3xaDLhPv+nAomIdPLtgEwI2xIGY7WMSKp9KfkKmlcjoPO4pkz8f/bpw==
"@stripe/stripe-js@1.20.3":
version "1.20.3"
resolved "https://registry.yarnpkg.com/@stripe/stripe-js/-/stripe-js-1.20.3.tgz#6f479d53ad933b8b17b6c708bbc2840b7512d80e"
integrity sha512-pFhPvajrZJUzoOZDC3/2YsOeL5VD63CVlbOXs+AriIchmy6Kb2CQqrcHPCuK72FM726lwo4neF0wPRBtsLYXuw==
"@tailwindcss/forms@0.3.4":
version "0.3.4"
@ -1947,10 +1947,10 @@ nanoclone@^0.2.1:
resolved "https://registry.yarnpkg.com/nanoclone/-/nanoclone-0.2.1.tgz#dd4090f8f1a110d26bb32c49ed2f5b9235209ed4"
integrity sha512-wynEP02LmIbLpcYw8uBKpcfF6dmg2vcpKqxeH5UcoKEYdExslsdUA4ugFauuaeYdTB76ez6gJW8XAZ6CgkXYxA==
nanoid@^3.1.23, nanoid@^3.1.28:
version "3.1.29"
resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.29.tgz#214fb2d7a33e1a5bef4757b779dfaeb6a4e5aeb4"
integrity sha512-dW2pUSGZ8ZnCFIlBIA31SV8huOGCHb6OwzVCc7A69rb/a+SgPBwfmLvK5TKQ3INPbRkcI8a/Owo0XbiTNH19wg==
nanoid@^3.1.23, nanoid@^3.1.30:
version "3.1.30"
resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.30.tgz#63f93cc548d2a113dc5dfbc63bfa09e2b9b64362"
integrity sha512-zJpuPDwOv8D2zq2WRoMe1HsfZthVewpel9CAvTfc/2mBD1uUT/agc5f7GHGWXlYkFvi1mVxe4IjvP2HNrop7nQ==
native-url@0.3.4:
version "0.3.4"
@ -2270,6 +2270,11 @@ picocolors@^0.2.1:
resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f"
integrity sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA==
picocolors@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c"
integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==
picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3:
version "2.3.0"
resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972"
@ -2345,13 +2350,13 @@ postcss@8.2.15:
nanoid "^3.1.23"
source-map "^0.6.1"
postcss@8.3.9, postcss@^8.1.6, postcss@^8.2.1:
version "8.3.9"
resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.9.tgz#98754caa06c4ee9eb59cc48bd073bb6bd3437c31"
integrity sha512-f/ZFyAKh9Dnqytx5X62jgjhhzttjZS7hMsohcI7HEI5tjELX/HxCy3EFhsRxyzGvrzFF+82XPvCS8T9TFleVJw==
postcss@8.3.11, postcss@^8.1.6, postcss@^8.2.1:
version "8.3.11"
resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.11.tgz#c3beca7ea811cd5e1c4a3ec6d2e7599ef1f8f858"
integrity sha512-hCmlUAIlUiav8Xdqw3Io4LcpA1DOt7h3LSTAC4G6JGHFFaWzI6qvFt9oilvl8BmkbBRX1IhM90ZAmpk68zccQA==
dependencies:
nanoid "^3.1.28"
picocolors "^0.2.1"
nanoid "^3.1.30"
picocolors "^1.0.0"
source-map-js "^0.6.2"
prettier@2.4.1:
@ -2905,10 +2910,10 @@ strip-eof@^1.0.0:
resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf"
integrity sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=
stripe@8.183.0:
version "8.183.0"
resolved "https://registry.yarnpkg.com/stripe/-/stripe-8.183.0.tgz#3dc183fafff618526a576a0d9972483b08075e06"
integrity sha512-QcM3nimH1CuP49VPPRt36Wgfu4QoS+Wm0eyGMis7Ej+seWFKqMffMdx7TE2gn8dVsJIOA1kDuIbAQGqpZHozGA==
stripe@8.184.0:
version "8.184.0"
resolved "https://registry.yarnpkg.com/stripe/-/stripe-8.184.0.tgz#ea68470ca6045bb47516c4fea3b775fd6ce15a36"
integrity sha512-ZUdvyX+sizTxXLEbUjgTShrulSWSkMIt7hIKdAkhnajYrHdzVtdmhBJl8sQbR9chMVox3Ig5ohilyeIrvcCE2g==
dependencies:
"@types/node" ">=8.1.0"
qs "^6.6.0"
@ -2982,10 +2987,10 @@ swr@1.0.1:
dependencies:
dequal "2.0.2"
tailwindcss@2.2.16:
version "2.2.16"
resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-2.2.16.tgz#32f81bdf1758b639cb83b9d30bf7cbecdda49e5e"
integrity sha512-EireCtpQyyJ4Xz8NYzHafBoy4baCOO96flM0+HgtsFcIQ9KFy/YBK3GEtlnD+rXen0e4xm8t3WiUcKBJmN6yjg==
tailwindcss@2.2.17:
version "2.2.17"
resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-2.2.17.tgz#c6332731f9ff1b6628ff589c95c38685347775e3"
integrity sha512-WgRpn+Pxn7eWqlruxnxEbL9ByVRWi3iC10z4b6dW0zSdnkPVC4hPMSWLQkkW8GCyBIv/vbJ0bxIi9dVrl4CfoA==
dependencies:
arg "^5.0.1"
bytes "^3.0.0"

2
packages/dnslink-api/Dockerfile
View File

@ -1,4 +1,4 @@
FROM node:16.11.1-alpine
FROM node:16.12.0-alpine
WORKDIR /usr/app

3
packages/dnslink-api/package.json
View File

@ -5,8 +5,7 @@
"license": "SEE LICENSE IN LICENSE.md",
"dependencies": {
"express": "^4.17.1",
"is-valid-domain": "^0.1.2",
"node-cache": "^5.1.2"
"is-valid-domain": "^0.1.2"
},
"devDependencies": {
"prettier": "^2.4.1"

9
packages/dnslink-api/src/index.js
View File

@ -1,14 +1,11 @@
const dns = require("dns");
const express = require("express");
const NodeCache = require("node-cache");
const isValidDomain = require("is-valid-domain");
const host = process.env.DNSLINK_API_HOSTNAME || "0.0.0.0";
const port = Number(process.env.DNSLINK_API_PORT) || 3100;
const cacheTTL = Number(process.env.DNSLINK_API_CACHE_TTL) || 300; // default to 5 minutes
const server = express();
const cache = new NodeCache({ stdTTL: cacheTTL });
const dnslinkNamespace = "skynet-ns";
const dnslinkRegExp = new RegExp(`^dnslink=/${dnslinkNamespace}/.+$`);
@ -23,10 +20,6 @@ server.get("/dnslink/:name", async (req, res) => {
return failure(`"${req.params.name}" is not a valid domain`);
}
if (cache.has(req.params.name)) {
return success(cache.get(req.params.name));
}
const lookup = `_dnslink.${req.params.name}`;
dns.resolveTxt(lookup, (error, records) => {
@ -65,8 +58,6 @@ server.get("/dnslink/:name", async (req, res) => {
const skylink = matchSkylink[1];
cache.set(req.params.name, skylink);
console.log(`${req.params.name} => ${skylink}`);
return success(skylink);

12
packages/dnslink-api/yarn.lock
View File

@ -36,11 +36,6 @@ bytes@3.1.0:
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6"
integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==
clone@2.x:
version "2.1.2"
resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f"
integrity sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=
content-disposition@0.5.3:
version "0.5.3"
resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd"
@ -257,13 +252,6 @@ negotiator@0.6.2:
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb"
integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==
node-cache@^5.1.2:
version "5.1.2"
resolved "https://registry.yarnpkg.com/node-cache/-/node-cache-5.1.2.tgz#f264dc2ccad0a780e76253a694e9fd0ed19c398d"
integrity sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==
dependencies:
clone "2.x"
on-finished@~2.3.0:
version "2.3.0"
resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947"

2
packages/handshake-api/Dockerfile
View File

@ -1,4 +1,4 @@
FROM node:16.11.1-alpine
FROM node:16.12.0-alpine
WORKDIR /usr/app

2
packages/health-check/Dockerfile
View File

@ -1,4 +1,4 @@
FROM node:16.11.1-alpine
FROM node:16.12.0-alpine
RUN apk update && apk add dnsmasq

2
packages/website/Dockerfile
View File

@ -1,4 +1,4 @@
FROM node:16.11.1-alpine
FROM node:16.12.0-alpine
RUN apk update && apk add autoconf automake build-base libtool nasm pkgconfig

8
packages/website/package.json
View File

@ -15,7 +15,7 @@
"copy-text-to-clipboard": "^3.0.1",
"crypto-browserify": "^3.12.0",
"framer-motion": "^4.1.17",
"gatsby": "^3.14.3",
"gatsby": "^3.14.4",
"gatsby-background-image": "^1.5.3",
"gatsby-image": "^3.11.0",
"gatsby-plugin-image": "^1.14.1",
@ -25,7 +25,7 @@
"gatsby-plugin-postcss": "^4.14.0",
"gatsby-plugin-purgecss": "^6.0.2",
"gatsby-plugin-react-helmet": "^4.14.0",
"gatsby-plugin-robots-txt": "^1.6.10",
"gatsby-plugin-robots-txt": "^1.6.13",
"gatsby-plugin-sharp": "^3.14.1",
"gatsby-plugin-svgr": "^3.0.0-beta.0",
"gatsby-remark-classes": "^1.0.2",
@ -47,8 +47,8 @@
"normalize.css": "^8.0.1",
"path-browserify": "^1.0.1",
"polished": "^4.1.3",
"popmotion": "^10.0.1",
"postcss": "^8.3.9",
"popmotion": "^10.0.2",
"postcss": "^8.3.11",
"preact-svg-loader": "^0.2.1",
"prop-types": "^15.7.2",
"react": "^17.0.2",

53
packages/website/yarn.lock
View File

@ -5875,10 +5875,10 @@ framesync@5.3.0:
dependencies:
tslib "^2.1.0"
framesync@6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/framesync/-/framesync-6.0.0.tgz#e6ad8ec128d33291a03cc34e58365c41265a1dae"
integrity sha512-9iBw/uZ/5fDURdpLrgc/eoFXiX9HC3DeOSLtTL5lZSyX/vQb+kjEz9CPNTiTObfK5PKyGgIUTstU8PK9W6EvoA==
framesync@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/framesync/-/framesync-6.0.1.tgz#5e32fc01f1c42b39c654c35b16440e07a25d6f20"
integrity sha512-fUY88kXvGiIItgNC7wcTOl0SNRCVXMKSWW2Yzfmn7EKNc+MpCzcz9DhdHcdjbrtN3c6R4H5dTY2jiCpPdysEjA==
dependencies:
tslib "^2.1.0"
@ -6159,10 +6159,10 @@ gatsby-plugin-react-helmet@^4.14.0:
dependencies:
"@babel/runtime" "^7.15.4"
gatsby-plugin-robots-txt@^1.6.10:
version "1.6.10"
resolved "https://registry.yarnpkg.com/gatsby-plugin-robots-txt/-/gatsby-plugin-robots-txt-1.6.10.tgz#b178efe3da65718a39d1acd2e5768f687b205cf0"
integrity sha512-soQT765LF0J8/dfrZGFuA/ZB/JUyvt2nVyHEFLUMd/qxgiem9x0EOZquJPId78xDHYePMkxNCPk9UsLcTZdFZw==
gatsby-plugin-robots-txt@^1.6.13:
version "1.6.13"
resolved "https://registry.yarnpkg.com/gatsby-plugin-robots-txt/-/gatsby-plugin-robots-txt-1.6.13.tgz#01e1ce68e3f4e07f957ac9a20cd2a9e12fdd3f79"
integrity sha512-MUPJsvkALwSmfRb1L3IVNmzIqwV35fol1RVot425ZYXvtD+s0OtZli5VnhLNxsX0beodyavzFl0SRJCCQZz0/g==
dependencies:
"@babel/runtime" "^7.14.0"
generate-robotstxt "^8.0.3"
@ -6464,10 +6464,10 @@ gatsby-worker@^0.5.0:
"@babel/core" "^7.15.5"
"@babel/runtime" "^7.15.4"
gatsby@^3.14.3:
version "3.14.3"
resolved "https://registry.yarnpkg.com/gatsby/-/gatsby-3.14.3.tgz#62bc020d0997f4b30e419abe644f60483f99e8b5"
integrity sha512-B6NmmIvGGVNktky0YGugxMJlZAbKEzGMMBA8ojbaDhfEzfNXxsyHYd0aHKL0fNwPEqMKUsllaACdXrw6CWRk2g==
gatsby@^3.14.4:
version "3.14.4"
resolved "https://registry.yarnpkg.com/gatsby/-/gatsby-3.14.4.tgz#e7ef5c3c1eda45ea3ad0bc1273f014fbc46b316c"
integrity sha512-cpsMhl2qkE0+TL0znwflVfycy143fFucGtFDkNxu8dbY0GRjA9MiDwlS/aSqg5U2HhhF39IcBvwuntiaP6vQ0g==
dependencies:
"@babel/code-frame" "^7.14.0"
"@babel/core" "^7.15.5"
@ -9405,7 +9405,7 @@ nanocolors@^0.2.2:
resolved "https://registry.yarnpkg.com/nanocolors/-/nanocolors-0.2.10.tgz#a712df4d3c1bf12d9b4fb8b5aa61b5ba31337503"
integrity sha512-i+EDWGsJClQwR/bhLIG/CObZZwaYaS5qt+yjxZbfV+77QiNHNzE9nj4d9Ut1TGZ0R0eSwPcQWzReASzXuw/7oA==
nanoid@^3.1.28, nanoid@^3.1.30:
nanoid@^3.1.30:
version "3.1.30"
resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.30.tgz#63f93cc548d2a113dc5dfbc63bfa09e2b9b64362"
integrity sha512-zJpuPDwOv8D2zq2WRoMe1HsfZthVewpel9CAvTfc/2mBD1uUT/agc5f7GHGWXlYkFvi1mVxe4IjvP2HNrop7nQ==
@ -10203,6 +10203,11 @@ picocolors@^0.2.1:
resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f"
integrity sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA==
picocolors@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c"
integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==
picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3:
version "2.3.0"
resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972"
@ -10285,12 +10290,12 @@ popmotion@9.3.6:
style-value-types "4.1.4"
tslib "^2.1.0"
popmotion@^10.0.1:
version "10.0.1"
resolved "https://registry.yarnpkg.com/popmotion/-/popmotion-10.0.1.tgz#5be5b80112a1386b7f2a11d081a1ea3ca7508d8e"
integrity sha512-jmVmbXMAl/qrrogQXCMGFR8+F290J6gl9kKHSYzAKBRTHm7B1unadmEeLSXX7rGtzPxOfKVnWHA3t3m3CVnxqw==
popmotion@^10.0.2:
version "10.0.2"
resolved "https://registry.yarnpkg.com/popmotion/-/popmotion-10.0.2.tgz#edb87efb2ba39505d07f79d3ca8cc4b1cbe101bf"
integrity sha512-8vgx02LL7Odk/kwSGigAnlPRLJK+Dfked571ZAHGzkxPNWThkU5mQX0xPmibIMAULZehWBV7hGN7LSCpaV3cfw==
dependencies:
framesync "6.0.0"
framesync "^6.0.1"
hey-listen "^1.0.8"
style-value-types "5.0.0"
tslib "^2.1.0"
@ -10616,13 +10621,13 @@ postcss@^7.0.27:
nanocolors "^0.2.2"
source-map "^0.6.1"
postcss@^8.1.6, postcss@^8.2.1, postcss@^8.2.15, postcss@^8.2.9, postcss@^8.3.5, postcss@^8.3.9:
version "8.3.9"
resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.9.tgz#98754caa06c4ee9eb59cc48bd073bb6bd3437c31"
integrity sha512-f/ZFyAKh9Dnqytx5X62jgjhhzttjZS7hMsohcI7HEI5tjELX/HxCy3EFhsRxyzGvrzFF+82XPvCS8T9TFleVJw==
postcss@^8.1.6, postcss@^8.2.1, postcss@^8.2.15, postcss@^8.2.9, postcss@^8.3.11, postcss@^8.3.5:
version "8.3.11"
resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.11.tgz#c3beca7ea811cd5e1c4a3ec6d2e7599ef1f8f858"
integrity sha512-hCmlUAIlUiav8Xdqw3Io4LcpA1DOt7h3LSTAC4G6JGHFFaWzI6qvFt9oilvl8BmkbBRX1IhM90ZAmpk68zccQA==
dependencies:
nanoid "^3.1.28"
picocolors "^0.2.1"
nanoid "^3.1.30"
picocolors "^1.0.0"
source-map-js "^0.6.2"
potrace@^2.1.8:

56
scripts/blocklist-skylink.sh
View File

@ -1,9 +1,10 @@
#! /usr/bin/env bash
# This script is meant to be used when manually adding a skylink to the
# blocklist on all the skynet web portals. The automatic script that is used to
# continuously sync a google sheets list with the blocklist on the web portals
# is /setup-scripts/blocklist-airtable.py
# This script adds a skylink to the sia blocklist and removes the skylink from
# nginx cache. The script should be run locally on each skynet webportal
# server. The automatic script that is used to continuously sync an Airtable
# sheet list with the blocklist on the web portals is
# /setup-scripts/blocklist-airtable.py
set -e # exit on first error
@ -17,44 +18,39 @@ fi
#########################################################
skylinks=()
if test -f "$1"; then
OLDIFS=$IFS
IFS=','
line_number=1
while read line
# Read file including the last line even when it doesn't end with newline
while IFS="" read -r line || [ -n "$line" ];
do
if [[ $line =~ ([a-zA-Z0-9_-]{46}) ]]; then
skylinks+=("$BASH_REMATCH")
if [[ $line =~ (^[a-zA-Z0-9_-]{46}$) ]]; then
skylinks+=("$line")
else
echo "Incorrect skylink at line ${line_number}: $line" && exit 1
fi
let line_number+=1
done < $1;
IFS=$OLDIFS
else
skylinks=("$1") # just single skylink passed as input argument
fi
#########################################################################
# iterate through all servers, block the skylinks and purge it from cache
#########################################################################
declare -a servers=( "eu-ger-1.siasky.net" "eu-ger-2.siasky.net" "eu-ger-3.siasky.net" "eu-ger-4.siasky.net" "eu-ger-5.siasky.net" "eu-ger-6.siasky.net" "eu-ger-7.siasky.net" "eu-ger-8.siasky.net"
"eu-fin-1.siasky.net" "eu-fin-2.siasky.net" "eu-fin-3.siasky.net" "eu-fin-4.siasky.net"
"eu-pol-1.siasky.net" "eu-pol-2.siasky.net" "eu-pol-3.siasky.net"
"us-or-1.siasky.net" "us-or-2.siasky.net"
"us-pa-1.siasky.net" "us-pa-2.siasky.net"
"us-va-1.siasky.net" "us-va-2.siasky.net" "us-va-3.siasky.net"
"as-hk-1.siasky.net"
"siasky.xyz" "dev1.siasky.dev" "dev2.siasky.dev" "dev3.siasky.dev")
for server in "${servers[@]}";
for skylink in "${skylinks[@]}";
do
for skylink in "${skylinks[@]}";
do
echo ".. ⌁ Blocking skylink ${skylink} on ${server}"
cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Elsq '^Skynet-Skylink: ${skylink}'"
ssh -q -t user@${server} "docker exec -it nginx bash -c ${cached_files_command} | xargs -r rm"
echo ".. ⌁ Skylink ${skylink} Blocked on ${server}"
echo "--------------------------------------------"
done
echo ".. ⌁ Blocking skylink ${skylink}"
# Add to Sia blocklist
docker exec sia siac skynet blocklist add "${skylink}"
# Remove from NGINX cache
# NOTE:
# If there are changes to how the NGINX cache is being cleared, the same
# changes need to be applied to the /setup-scripts/blocklist-airtable.py
# script.
cached_files_command="find /data/nginx/cache/ -type f | xargs -r grep -Els '^Skynet-Skylink: ${skylink}'"
docker exec -it nginx bash -c "${cached_files_command}" | xargs -r rm
echo ".. ⌁ Skylink ${skylink} Blocked"
echo "--------------------------------------------"
done
echo "✓ All done !"

40
scripts/db_backup.sh
View File

@ -27,26 +27,6 @@ fi
# Take the current datetime:
DT=$(date +%Y-%m-%d)
### COCKROACH DB ###
echo "Creating a backup of CockroachDB:"
# Check if a backup already exists:
totalFoundObjects=$(aws s3 ls $S3_BACKUP_PATH/$DT --recursive --summarize | grep "cockroach" | wc -l)
if [ "$totalFoundObjects" -ge "1" ]; then
echo "Backup already exists for today. Skipping."
else
# Create a cockroachdb backup:
docker exec cockroach \
cockroach sql \
--host cockroach:26257 \
--certs-dir=/certs \
--execute="BACKUP TO '$S3_BACKUP_PATH/$DT/cockroach/?AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID&AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY';"
if [[ $? > 0 ]]; then
echo "Creating a CockroachDB backup failed. Skipping."
else
echo "Successfully backed up CockroachDB."
fi
fi
### MONGO DB ###
echo "Creating a backup of MongoDB:"
# Check if a backup already exists:
@ -73,3 +53,23 @@ else
fi
docker exec mongo rm -rf /data/db/backups/$DT
fi
### COCKROACH DB ###
echo "Creating a backup of CockroachDB:"
# Check if a backup already exists:
totalFoundObjects=$(aws s3 ls $S3_BACKUP_PATH/$DT --recursive --summarize | grep "cockroach" | wc -l)
if [ "$totalFoundObjects" -ge "1" ]; then
echo "Backup already exists for today. Skipping."
else
# Create a cockroachdb backup:
docker exec cockroach \
cockroach sql \
--host cockroach:26257 \
--certs-dir=/certs \
--execute="BACKUP TO '$S3_BACKUP_PATH/$DT/cockroach/?AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID&AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY';"
if [[ $? > 0 ]]; then
echo "Creating a CockroachDB backup failed. Skipping."
else
echo "Successfully backed up CockroachDB."
fi
fi

5
scripts/db_restore.sh
View File

@ -1,5 +1,8 @@
#!/bin/bash
# First of all, let's pamper awscli because Python is so special:
pip3 install --upgrade awscli
BACKUP=$1
if [[ $BACKUP == "" ]]; then
echo "No backup name given. It should look like '2020-01-29'."
@ -97,7 +100,7 @@ rm mongo.tgz
# The name of the backup is not `mongo` due to the way we're creating it,
# it's $BACKUP.
docker exec mongo \
mongorestore \
mongorestore --drop \
mongodb://$SKYNET_DB_USER:$SKYNET_DB_PASS@$SKYNET_DB_HOST:$SKYNET_DB_PORT \
/data/db/backups/to_restore/$BACKUP
# Clean up:

4
setup-scripts/blocklist-airtable.py
View File

@ -141,6 +141,10 @@ async def block_skylinks_from_airtable():
)
return await send_msg(message, force_notify=False)
# Remove from NGINX cache
# NOTE:
# If there are changes to how the NGINX cache is being cleared, the same
# changes need to be applied to the /scripts/blocklist-skylink.sh script.
print("Searching nginx cache for blocked files")
cached_files_count = 0
batch_size = 1000