From 2c12b9d00969447aa1a1239c8f5d6374a60a6850 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 4 Sep 2020 14:55:30 +0200 Subject: [PATCH 1/5] add safe stop and restart sia scripts --- setup-scripts/README.md | 23 +++++++++++++++-------- setup-scripts/sia-restart.sh | 14 ++++++++++++++ setup-scripts/sia-stop.sh | 21 +++++++++++++++++++++ 3 files changed, 50 insertions(+), 8 deletions(-) create mode 100755 setup-scripts/sia-restart.sh create mode 100755 setup-scripts/sia-stop.sh diff --git a/setup-scripts/README.md b/setup-scripts/README.md index 9083dae7..ea8aeba9 100644 --- a/setup-scripts/README.md +++ b/setup-scripts/README.md @@ -90,16 +90,23 @@ At this point we have almost everything running, we just need to set up your wal ## Useful Commands +- Starting the whole stack + > `docker-compose up -d` +- Stopping the whole stack + > `docker-compose down` - Accessing siac > `docker exec -it sia siac` -- Checking status of siad service - > `systemctl --user status siad` -- Stopping siad service - > `systemctl --user stop siad` -- Starting siad service - > `systemctl --user start siad` -- Restarting siad service - > `systemctl --user restart siad` +- Stopping sia service + - safe method - stops health-check service and wait for dns propagation + > `setup-scripts/sia-stop.sh` + - unsafe (force stop) + > `docker-compose down sia` +- Restarting sia service + - safe method - stops health-check service and wait for dns propagation + > `setup-scripts/sia-restart.sh` + - unsafe (force restart) + > `docker-compose down sia` + > `docker compose up -d sia` - Restarting caddy gracefully after making changes to Caddyfile > `docker exec caddy caddy reload --config /etc/caddy/Caddyfile` - Restarting nginx gracefully after making changes to nginx configs diff --git a/setup-scripts/sia-restart.sh b/setup-scripts/sia-restart.sh new file mode 100755 index 00000000..7316444b --- /dev/null +++ b/setup-scripts/sia-restart.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e # exit on first error + +cwd + +# get current working directory (pwd doesn't cut it) +cwd=$(cd -P -- "$(dirname -- "$0")" && pwd -P) + +# run the sia-stop script +. ${cwd}/sia-stop.sh + +# start sia +docker-compose up -d sia \ No newline at end of file diff --git a/setup-scripts/sia-stop.sh b/setup-scripts/sia-stop.sh new file mode 100755 index 00000000..64dce4dd --- /dev/null +++ b/setup-scripts/sia-stop.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e # exit on first error + +countdown() { + local secs=$1 + while [ $secs -gt 0 ]; do + echo -ne "Waiting $secs\033[0K\r" + sleep 1 + : $((secs--)) + done +} + +# first stop healh-check so the server is taken our of load balancer +docker-compose stop health-check + +# then wait 5 minutes for the load balancer to propagate the dns records +countdown 300 + +# now stop sia process +docker-compose stop sia \ No newline at end of file From 4f849ddf53c71fa7ab9ac412b769be232936cbcc Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Fri, 4 Sep 2020 14:58:02 +0200 Subject: [PATCH 2/5] change comments --- setup-scripts/sia-restart.sh | 6 ++---- setup-scripts/sia-stop.sh | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/setup-scripts/sia-restart.sh b/setup-scripts/sia-restart.sh index 7316444b..967638ef 100755 --- a/setup-scripts/sia-restart.sh +++ b/setup-scripts/sia-restart.sh @@ -2,13 +2,11 @@ set -e # exit on first error -cwd - -# get current working directory (pwd doesn't cut it) +# get running script directory cwd=$(cd -P -- "$(dirname -- "$0")" && pwd -P) # run the sia-stop script . ${cwd}/sia-stop.sh # start sia -docker-compose up -d sia \ No newline at end of file +docker-compose up -d sia diff --git a/setup-scripts/sia-stop.sh b/setup-scripts/sia-stop.sh index 64dce4dd..1bbe2ce5 100755 --- a/setup-scripts/sia-stop.sh +++ b/setup-scripts/sia-stop.sh @@ -18,4 +18,4 @@ docker-compose stop health-check countdown 300 # now stop sia process -docker-compose stop sia \ No newline at end of file +docker-compose stop sia From abacfc1ed1336689b203a9456557be5164a0ddb0 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 14 Sep 2020 15:57:44 +0200 Subject: [PATCH 3/5] improve docs --- README.md | 8 +++++--- setup-scripts/README.md | 24 ++++++++++++------------ setup-scripts/setup-docker-services.sh | 3 +++ 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 3ebef7ac..f119ce20 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,13 @@ List of available parameters: ## Contributing ### Testing you Code -Before pushing your code you should verify that it will pass our online test -suite. -**Cypress Tests** +Before pushing your code you should verify that it will pass our online test +suite. + +**Cypress Tests** Verify the Cypress test suite by doing the following: + 1. In one terminal screen run `GATSBY_API_URL=https://siasky.net yarn workspace webapp start` 1. In a second terminal screen run `yarn workspace webapp cypress run` diff --git a/setup-scripts/README.md b/setup-scripts/README.md index ea8aeba9..5536517c 100644 --- a/setup-scripts/README.md +++ b/setup-scripts/README.md @@ -15,21 +15,22 @@ You may want to fork this repository and replace ssh keys in - [sia](https://sia.tech) ([docker hub](https://hub.docker.com/r/nebulouslabs/sia)): storage provider, heart of the portal setup - [caddy](https://caddyserver.com) ([docker hub](https://hub.docker.com/r/caddy/caddy)): reverse proxy (similar to nginx) that handles ssl out of a box and acts as a transparent entry point - [openresty](https://openresty.org) ([docker hub](https://hub.docker.com/r/openresty/openresty)): nginx custom build, acts as a cached proxy to siad and exposes all api endpoints - - health-check: this is a simple service that runs periodically and collects health data about the server (status and response times) and exposes `/health-check` api endpoint that is deliberately delayed based on the response times of the server so potential load balancer could prioritize servers based on that (we use it with cloudflare) -- siad setup: we use "double siad" setup that has one node solely for download and one for upload to improve performance - - we use systemd to manage siad service - - siad is not installed as docker service for improved performance + - [health-check](https://github.com/NebulousLabs/skynet-webportal/tree/master/packages/health-check): simple service that runs periodically and collects health data about the server (status and response times) - [read more](https://github.com/NebulousLabs/skynet-webportal/blob/master/packages/health-check/README.md) + - [handshake](https://handshake.org) ([github](https://github.com/handshake-org/hsd)): full handshake node + - [handshake-api](https://github.com/NebulousLabs/skynet-webportal/tree/master/packages/handshake-api): simple API talking to the handshake node - [read more](https://github.com/NebulousLabs/skynet-webportal/blob/master/packages/handshake-api/README.md) + - [webapp](https://github.com/NebulousLabs/skynet-webportal/tree/master/packages/webapp): portal frontend application - [read more](https://github.com/NebulousLabs/skynet-webportal/blob/master/packages/webapp/README.md) - discord integration - [funds-checker](funds-checker.py): script that checks wallet balance and sends status messages to discord periodically + - [health-checker](health-checker.py): script that monitors health-check service for server health issues and reports them to discord periodically - [log-checker](log-checker.py): script that scans siad logs for critical errors and reports them to discord periodically - [blacklist-skylink](blacklist-skylink.sh): script that can be run locally from a machine that has access to all your skynet portal servers that blacklists provided skylink and prunes nginx cache to ensure it's not available any more (that is a bit much but that's the best we can do right now without paid nginx version) - if you want to use it, make sure to adjust the server addresses ### Step 1: setting up server user 1. SSH in a freshly installed Debian machine on a user with sudo access (can be root) -1. `apt-get update && apt-get install sudo` to make sure `sudo` is available +1. `apt-get update && apt-get install sudo -y` to make sure `sudo` is available 1. `adduser user` to create user called `user` (creates `/home/user` directory) -1. `usermod -a -G sudo user` to add this new user to sudo group +1. `usermod -aG sudo user` to add this new user to sudo group 1. Quit the ssh session with `exit` command You a can now ssh into your machine as the user `user`. @@ -42,7 +43,7 @@ You a can now ssh into your machine as the user `user`. **Following step will be executed on remote host logged in as a `user`:** -1. `sudo apt-get install git` to install git +1. `sudo apt-get install git -y` to install git 1. `git clone https://github.com/NebulousLabs/skynet-webportal` 1. run setup scripts in the exact order and provide sudo password when asked (if one of them fails, you can retry just this one before proceeding further) 1. `/home/user/skynet-webportal/setup-scripts/setup-server.sh` @@ -74,12 +75,11 @@ At this point we have almost everything running, we just need to set up your wal ### Step 4: configuring docker services -1. generate and copy sia api token `printf ":$(cat /home/user/.sia/apipassword)" | base64` 1. edit `/home/user/skynet-webportal/.env` and configure following environment variables - `DOMAIN_NAME` (optional) is your domain name if you have it - `EMAIL_ADDRESS` (required) is your email address used for communication regarding SSL certification (required) - `SIA_WALLET_PASSWORD` (required) is your wallet password (or seed if you did not set a password) - - `HSD_API_KEY` (optional) this is a random security key for an optional handshake integration that gets generated automatically + - `HSD_API_KEY` (optional) this is a random security key for a handshake integration that gets generated automatically - `CLOUDFLARE_AUTH_TOKEN` (optional) if using cloudflare as dns loadbalancer (need to change it in Caddyfile too) - `AWS_ACCESS_KEY_ID` (optional) if using route53 as a dns loadbalancer - `AWS_SECRET_ACCESS_KEY` (optional) if using route53 as a dns loadbalancer @@ -107,11 +107,11 @@ At this point we have almost everything running, we just need to set up your wal - unsafe (force restart) > `docker-compose down sia` > `docker compose up -d sia` -- Restarting caddy gracefully after making changes to Caddyfile +- Restarting caddy gracefully after making changes to Caddyfile (no downtime) > `docker exec caddy caddy reload --config /etc/caddy/Caddyfile` -- Restarting nginx gracefully after making changes to nginx configs +- Restarting nginx gracefully after making changes to nginx configs (no downtime) > `docker exec nginx openresty -s reload` -- Checking siad service logs (last hour) +- Checking siad service logs (since last hour) > `docker logs --since 1h $(docker ps -q --filter "name=^sia$")` - Checking caddy logs (for example in case ssl certificate fails) > `docker logs caddy -f` diff --git a/setup-scripts/setup-docker-services.sh b/setup-scripts/setup-docker-services.sh index a7903c80..fbe063f9 100755 --- a/setup-scripts/setup-docker-services.sh +++ b/setup-scripts/setup-docker-services.sh @@ -12,6 +12,9 @@ sudo apt-get update sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker --version # sanity check +# add user to docker group to avoid having to use sudo for every docker command +sudo usermod -aG docker user + # Install docker-compose sudo curl -L "https://github.com/docker/compose/releases/download/1.25.5/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose From a02c27664b28400deacda1091c8b38de22c69401 Mon Sep 17 00:00:00 2001 From: Karol Wypchlo Date: Mon, 14 Sep 2020 17:40:18 +0200 Subject: [PATCH 4/5] adjust scripts --- .../blacklist-skylink.sh | 0 .../sia-stop.sh => scripts/portal-down.sh | 7 +-- scripts/portal-up.sh | 6 +++ scripts/portal-upgrade.sh | 14 +++++ setup-scripts/README.md | 20 +++---- setup-scripts/sia-restart.sh | 12 ----- setup-scripts/stats-logger.sh | 53 ------------------- 7 files changed, 30 insertions(+), 82 deletions(-) rename {setup-scripts => scripts}/blacklist-skylink.sh (100%) rename setup-scripts/sia-stop.sh => scripts/portal-down.sh (65%) create mode 100755 scripts/portal-up.sh create mode 100755 scripts/portal-upgrade.sh delete mode 100755 setup-scripts/sia-restart.sh delete mode 100755 setup-scripts/stats-logger.sh diff --git a/setup-scripts/blacklist-skylink.sh b/scripts/blacklist-skylink.sh similarity index 100% rename from setup-scripts/blacklist-skylink.sh rename to scripts/blacklist-skylink.sh diff --git a/setup-scripts/sia-stop.sh b/scripts/portal-down.sh similarity index 65% rename from setup-scripts/sia-stop.sh rename to scripts/portal-down.sh index 1bbe2ce5..b2a9593c 100755 --- a/setup-scripts/sia-stop.sh +++ b/scripts/portal-down.sh @@ -11,11 +11,8 @@ countdown() { done } -# first stop healh-check so the server is taken our of load balancer -docker-compose stop health-check +# stop healh-check so the server is taken our of load balancer +docker exec health-check cli/disable # then wait 5 minutes for the load balancer to propagate the dns records countdown 300 - -# now stop sia process -docker-compose stop sia diff --git a/scripts/portal-up.sh b/scripts/portal-up.sh new file mode 100755 index 00000000..1a287f2f --- /dev/null +++ b/scripts/portal-up.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e # exit on first error + +# start the health-checks service +docker exec health-check cli/enable diff --git a/scripts/portal-upgrade.sh b/scripts/portal-upgrade.sh new file mode 100755 index 00000000..b2973809 --- /dev/null +++ b/scripts/portal-upgrade.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e # exit on first error + +# put the server down for maintenance +. ${cwd}/portal-down.sh + +# rebuild and restart all docker containers +docker-compose build --no-cache +docker-compose down +docker-compose up -d + +# enable the server again +. ${cwd}/portal-up.sh diff --git a/setup-scripts/README.md b/setup-scripts/README.md index 5536517c..e1a743a0 100644 --- a/setup-scripts/README.md +++ b/setup-scripts/README.md @@ -23,7 +23,7 @@ You may want to fork this repository and replace ssh keys in - [funds-checker](funds-checker.py): script that checks wallet balance and sends status messages to discord periodically - [health-checker](health-checker.py): script that monitors health-check service for server health issues and reports them to discord periodically - [log-checker](log-checker.py): script that scans siad logs for critical errors and reports them to discord periodically -- [blacklist-skylink](blacklist-skylink.sh): script that can be run locally from a machine that has access to all your skynet portal servers that blacklists provided skylink and prunes nginx cache to ensure it's not available any more (that is a bit much but that's the best we can do right now without paid nginx version) - if you want to use it, make sure to adjust the server addresses +- [blacklist-skylink](../scripts/blacklist-skylink.sh): script that can be run locally from a machine that has access to all your skynet portal servers that blacklists provided skylink and prunes nginx cache to ensure it's not available any more (that is a bit much but that's the best we can do right now without paid nginx version) - if you want to use it, make sure to adjust the server addresses ### Step 1: setting up server user @@ -96,17 +96,13 @@ At this point we have almost everything running, we just need to set up your wal > `docker-compose down` - Accessing siac > `docker exec -it sia siac` -- Stopping sia service - - safe method - stops health-check service and wait for dns propagation - > `setup-scripts/sia-stop.sh` - - unsafe (force stop) - > `docker-compose down sia` -- Restarting sia service - - safe method - stops health-check service and wait for dns propagation - > `setup-scripts/sia-restart.sh` - - unsafe (force restart) - > `docker-compose down sia` - > `docker compose up -d sia` +- Portal maintenance + - Pulling portal out for maintenance + > `scripts/portal-down.sh` + - Putting portal back into place after maintenance + > `scripts/portal-up.sh` + - Upgrading portal containers (takes care of pulling it and putting it back) + > `scripts/portal-upgrade.sh` - Restarting caddy gracefully after making changes to Caddyfile (no downtime) > `docker exec caddy caddy reload --config /etc/caddy/Caddyfile` - Restarting nginx gracefully after making changes to nginx configs (no downtime) diff --git a/setup-scripts/sia-restart.sh b/setup-scripts/sia-restart.sh deleted file mode 100755 index 967638ef..00000000 --- a/setup-scripts/sia-restart.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -e # exit on first error - -# get running script directory -cwd=$(cd -P -- "$(dirname -- "$0")" && pwd -P) - -# run the sia-stop script -. ${cwd}/sia-stop.sh - -# start sia -docker-compose up -d sia diff --git a/setup-scripts/stats-logger.sh b/setup-scripts/stats-logger.sh deleted file mode 100755 index 9fc614de..00000000 --- a/setup-scripts/stats-logger.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -LOGFILE=$1 -SIA_PORT=${SIA_PORT:-9980} - -# This is a logger service that pulls some current skynet stats and appends them to json log file. -# You should probably run it using crontab, most likely as a root due to access_log read restrictions. -# -# basic usage: -# /home/user/skynet-webportal/setup-scripts/stats-logger.sh public/logs.json -# -# usage with custom sia port: -# SIA_PORT=9970 /home/user/skynet-webportal/setup-scripts/stats-logger.sh public/logs.json -# -# configuring hourly logging with crontab (run crontab -e) -# 0 * * * * /home/user/skynet-webportal/setup-scripts/stats-logger.sh /home/user/skynet-webportal/public/stats.json >/dev/null 2>&1 - -if ! [ -x "$(command -v jq)" ]; then - echo 'Error: jq is not installed. Please install with `sudo apt-get install jq`.' >&2 - exit 1 -fi - -if ! [ -x "$(command -v sponge)" ]; then - echo 'Error: sponge is not installed. Please install with `sudo apt-get install moreutils`.' >&2 - exit 1 -fi - -if [ -z "$LOGFILE" ]; then - echo 'Error: You need to specify json log file name.' >&2 - exit 1 -fi - -# create logfile if it doesn't exist and initialize it with empty array -if [ ! -f "$LOGFILE" ]; then - mkdir -p "`dirname \"$LOGFILE\"`" 2>/dev/null - echo [] > $LOGFILE -fi - -# get downloads count from nginx logs -awk -vDate=`date -d'now-1 hours' +[%d/%b/%Y:%H:%M:%S` ' { if ($4 > Date) print $0}' /var/log/nginx/access.log > /var/log/nginx/access_last_hour.log -DOWNLOADS_COUNT_LAST_HOUR=$(awk '{print $7}' /var/log/nginx/access_last_hour.log | grep -E '/[a-zA-Z0-9_-]{46}(/.*)?' | wc -l) - -# get siac output -SKYNET_LS=$(/home/user/go/bin/siac skynet ls --addr localhost:${SIA_PORT} | grep -i listing) -SKYNET_LS_REGEX="Listing (.*) files\/dirs:\s+(.*)" -if [[ "${SKYNET_LS}" =~ $SKYNET_LS_REGEX ]]; then - SKYFILES_COUNT="${BASH_REMATCH[1]}" - SKYFILES_SIZE_BYTES=$(echo ${BASH_REMATCH[2]} | sed -r 's/[ B]//g' | numfmt --from=iec) -fi - -DATE=$(date -u +"%Y-%m-%d %H:%M:%S") -LOG_ENTRY=$(echo {\"date\":\"${DATE}\", \"skyfiles_count\":${SKYFILES_COUNT}, \"skyfiles_size_bytes\":${SKYFILES_SIZE_BYTES}, \"downloads_count_last_hour\":${DOWNLOADS_COUNT_LAST_HOUR}}) -jq -c ". += [${LOG_ENTRY}]" $LOGFILE | sponge $LOGFILE \ No newline at end of file From 1fbaee6caf133762db0fd1d65218467febe50a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Wypch=C5=82o?= Date: Mon, 14 Sep 2020 18:20:26 +0200 Subject: [PATCH 5/5] Update portal-upgrade.sh --- scripts/portal-upgrade.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/portal-upgrade.sh b/scripts/portal-upgrade.sh index b2973809..4d04f30e 100755 --- a/scripts/portal-upgrade.sh +++ b/scripts/portal-upgrade.sh @@ -2,6 +2,9 @@ set -e # exit on first error +# get current working directory (pwd doesn't cut it) +cwd=$(cd -P -- "$(dirname -- "$0")" && pwd -P) + # put the server down for maintenance . ${cwd}/portal-down.sh