clean up double siad setup

This commit is contained in:
Karol Wypchlo 2020-07-28 13:22:07 +02:00
parent 0f035fdaf7
commit 955425d321
10 changed files with 51 additions and 122 deletions

View File

@ -13,18 +13,11 @@ set_real_ip_from 172.16.0.0/12;
set_real_ip_from 192.168.0.0/16; set_real_ip_from 192.168.0.0/16;
real_ip_header X-Forwarded-For; real_ip_header X-Forwarded-For;
# note that we point uploads to port '9970', do this when you want to
# run in a configuration where you have two siad instances, one for
# downloads and one for uploads. This drastically improves the up - and
# download speed of your portal. When running your portal in this double
# siad setup, make sure only the download portal runs in 'portal mode'.
# The upload siad can be run in normal mode. Set the port to '9980' if
# you do not want to run your portal in the double siad setup.
upstream siad-upload { upstream siad-upload {
server docker-host:9980; server docker-host:9970;
} }
upstream siad-download { upstream siad {
server docker-host:9980; server docker-host:9980;
} }
@ -53,7 +46,7 @@ server {
proxy_cache skynet; proxy_cache skynet;
proxy_cache_valid any 1m; # cache blacklist for 1 minute proxy_cache_valid any 1m; # cache blacklist for 1 minute
proxy_set_header User-Agent: Sia-Agent; proxy_set_header User-Agent: Sia-Agent;
proxy_pass http://siad-download/skynet/blacklist; proxy_pass http://siad/skynet/blacklist;
} }
location /skynet/blacklist { location /skynet/blacklist {
@ -62,7 +55,7 @@ server {
proxy_cache skynet; proxy_cache skynet;
proxy_cache_valid any 1m; # cache blacklist for 1 minute proxy_cache_valid any 1m; # cache blacklist for 1 minute
proxy_set_header User-Agent: Sia-Agent; proxy_set_header User-Agent: Sia-Agent;
proxy_pass http://siad-download/skynet/blacklist; proxy_pass http://siad/skynet/blacklist;
} }
location /portals { location /portals {
@ -71,7 +64,7 @@ server {
proxy_cache skynet; proxy_cache skynet;
proxy_cache_valid any 1m; # cache portals for 1 minute proxy_cache_valid any 1m; # cache portals for 1 minute
proxy_set_header User-Agent: Sia-Agent; proxy_set_header User-Agent: Sia-Agent;
proxy_pass http://siad-download/skynet/portals; proxy_pass http://siad/skynet/portals;
} }
location /skynet/portals { location /skynet/portals {
@ -80,7 +73,7 @@ server {
proxy_cache skynet; proxy_cache skynet;
proxy_cache_valid any 1m; # cache portals for 1 minute proxy_cache_valid any 1m; # cache portals for 1 minute
proxy_set_header User-Agent: Sia-Agent; proxy_set_header User-Agent: Sia-Agent;
proxy_pass http://siad-download/skynet/portals; proxy_pass http://siad/skynet/portals;
} }
location /stats { location /stats {
@ -89,7 +82,7 @@ server {
proxy_cache skynet; proxy_cache skynet;
proxy_cache_valid any 1m; # cache stats for 1 minute proxy_cache_valid any 1m; # cache stats for 1 minute
proxy_set_header User-Agent: Sia-Agent; proxy_set_header User-Agent: Sia-Agent;
proxy_pass http://siad-upload/skynet/stats; proxy_pass http://siad/skynet/stats;
} }
location /skynet/stats { location /skynet/stats {
@ -98,7 +91,7 @@ server {
proxy_cache skynet; proxy_cache skynet;
proxy_cache_valid any 1m; # cache stats for 1 minute proxy_cache_valid any 1m; # cache stats for 1 minute
proxy_set_header User-Agent: Sia-Agent; proxy_set_header User-Agent: Sia-Agent;
proxy_pass http://siad-upload/skynet/stats; proxy_pass http://siad/skynet/stats;
} }
location /statsdown { location /statsdown {
@ -107,7 +100,7 @@ server {
proxy_cache skynet; proxy_cache skynet;
proxy_cache_valid any 1m; # cache stats for 1 minute proxy_cache_valid any 1m; # cache stats for 1 minute
proxy_set_header User-Agent: Sia-Agent; proxy_set_header User-Agent: Sia-Agent;
proxy_pass http://siad-download/skynet/stats; proxy_pass http://siad/skynet/stats; # serve upload node stats temporarily
} }
location /health-check { location /health-check {
@ -153,7 +146,7 @@ server {
} }
# proxy this call to siad endpoint (make sure the ip is correct) # proxy this call to siad endpoint (make sure the ip is correct)
proxy_pass http://siad-upload/skynet/skyfile/$dir1/$dir2/$dir3/$dir4$is_args$args; proxy_pass http://siad/skynet/skyfile/$dir1/$dir2/$dir3/$dir4$is_args$args;
} }
location ~ "/skynet/skyfile/(.+)" { location ~ "/skynet/skyfile/(.+)" {
@ -172,7 +165,7 @@ server {
set $siapath $1; set $siapath $1;
# proxy this call to siad endpoint (make sure the ip is correct) # proxy this call to siad endpoint (make sure the ip is correct)
proxy_pass http://siad-upload/skynet/skyfile/$siapath$is_args$args; proxy_pass http://siad/skynet/skyfile/$siapath$is_args$args;
} }
location ~ "^/([a-zA-Z0-9-_]{46}(/.*)?)$" { location ~ "^/([a-zA-Z0-9-_]{46}(/.*)?)$" {
@ -187,7 +180,7 @@ server {
proxy_read_timeout 600; proxy_read_timeout 600;
proxy_set_header User-Agent: Sia-Agent; proxy_set_header User-Agent: Sia-Agent;
# proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct)
proxy_pass http://siad-download/skynet/skylink/$skylink$is_args$args; proxy_pass http://siad/skynet/skylink/$skylink$is_args$args;
# if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs # if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs
proxy_buffer_size 128k; proxy_buffer_size 128k;
@ -213,7 +206,7 @@ server {
proxy_set_header User-Agent: Sia-Agent; proxy_set_header User-Agent: Sia-Agent;
# proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct) # proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct)
# this alias also adds attachment=true url param to force download the file # this alias also adds attachment=true url param to force download the file
proxy_pass http://siad-download/skynet/skylink/$skylink?attachment=true&$args; proxy_pass http://siad/skynet/skylink/$skylink?attachment=true&$args;
# if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs # if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs
proxy_buffer_size 128k; proxy_buffer_size 128k;

View File

@ -17,7 +17,7 @@ You may want to fork this repository and replace ssh keys in
- [openresty](https://openresty.org) ([docker hub](https://hub.docker.com/r/openresty/openresty)): nginx custom build, acts as a cached proxy to siad (we only use it because caddy doesn't support proxy caching, otherwise we could drop it) - [openresty](https://openresty.org) ([docker hub](https://hub.docker.com/r/openresty/openresty)): nginx custom build, acts as a cached proxy to siad (we only use it because caddy doesn't support proxy caching, otherwise we could drop it)
- health-check: this is a simple service that runs periodically and collects health data about the server (status and response times) and exposes `/health-check` api endpoint that is deliberately delayed based on the response times of the server so potential load balancer could prioritize servers based on that (we use it with cloudflare) - health-check: this is a simple service that runs periodically and collects health data about the server (status and response times) and exposes `/health-check` api endpoint that is deliberately delayed based on the response times of the server so potential load balancer could prioritize servers based on that (we use it with cloudflare)
- siad setup: we use "double siad" setup that has one node solely for download and one for upload to improve performance - siad setup: we use "double siad" setup that has one node solely for download and one for upload to improve performance
- we use systemd to manage siad services - we use systemd to manage siad service
- siad is not installed as docker service for improved performance - siad is not installed as docker service for improved performance
- discord integration - discord integration
- [funds-checker](funds-checker.py): script that checks wallet balance and sends status messages to discord periodically - [funds-checker](funds-checker.py): script that checks wallet balance and sends status messages to discord periodically
@ -55,17 +55,13 @@ You a can now ssh into your machine as the user `user`.
At this point we have almost everything set up. We have 2 siad instances running as services and we need to set up the wallets and allowance on those. At this point we have almost everything set up. We have 2 siad instances running as services and we need to set up the wallets and allowance on those.
1. Create new wallet for both siad instances (remember to save the seeds) 1. Create new wallet (remember to save the seeds)
1. `siac wallet init` to init download node wallet > `siac wallet init`
1. `siac-upload wallet init` to init upload node wallet 1. Unlock wallet (use seed as password)
1. Unlock both wallets > `siac wallet unlock`
1. `siac wallet unlock` to unlock download node wallet (use seed as password) 1. Generate wallet addresse (save them for later to transfer the funds)
1. `siac-upload wallet unlock` to unlock upload node wallet (use seed as password) > `siac wallet address`
1. Generate wallet addresses for both siad instances (save them for later to transfer the funds) 1. Set up allowance by running `siac renter setallowance`
1. `siac wallet address` to generate address for download node wallet
1. `siac-upload wallet address` to generate address for upload node wallet
1. Set up allowance on both siad instances
1. `siac renter setallowance` to set allowance on download node
1. 10 KS (keep 25 KS in your wallet) 1. 10 KS (keep 25 KS in your wallet)
1. default period 1. default period
1. default number of hosts 1. default number of hosts
@ -74,20 +70,12 @@ At this point we have almost everything set up. We have 2 siad instances running
1. 500 GB expected upload 1. 500 GB expected upload
1. 5 TB expected download 1. 5 TB expected download
1. default redundancy 1. default redundancy
1. `siac-upload renter setallowance` to set allowance on upload node 1. Run `siac renter setallowance --payment-contract-initial-funding 10SC` so siad will start making 10 contracts per block with many hosts to potentially view the whole network's files
1. use the same allowance settings as download node
1. Run `siac renter setallowance --payment-contract-initial-funding 10SC` so that your download node will start making 10 contracts per block with many hosts to potentially view the whole network's files
1. Copy over apipassword from `/home/user/.sia/apipassword` and save it for the next step 1. Copy over apipassword from `/home/user/.sia/apipassword` and save it for the next step
1. Edit environment files for both siad instances 1. Edit environment file for siad `/home/user/.sia/sia.env` and set:
1. `/home/user/.sia/sia.env` for the download node
1. `SIA_API_PASSWORD` to previously copied apipassword (same for both instances) 1. `SIA_API_PASSWORD` to previously copied apipassword (same for both instances)
1. `SIA_WALLET_PASSWORD` to be the wallet seed 1. `SIA_WALLET_PASSWORD` to be the wallet seed
1. `PORTAL_NAME` xxxxed part to some meaningful name like `warsaw.siasky.net` 1. `PORTAL_NAME` (optional) only for bot utils, set it to something meaningful name like `warsaw.siasky.net`
1. `DISCORD_BOT_TOKEN` for discord health check scripts integration
1. `/home/user/.sia/sia-upload.env` for the upload node
1. `SIA_API_PASSWORD` to previously copied apipassword (same for both instances)
1. `SIA_WALLET_PASSWORD` to be the wallet seed
1. `PORTAL_NAME` xxxxed part to some meaningful name like `warsaw.siasky.net`
1. `DISCORD_BOT_TOKEN` for discord health check scripts integration 1. `DISCORD_BOT_TOKEN` for discord health check scripts integration
### Step 4: configuring docker services ### Step 4: configuring docker services
@ -97,8 +85,10 @@ At this point we have almost everything set up. We have 2 siad instances running
- `DOMAIN_NAME` (optional) is your domain name if you have it - `DOMAIN_NAME` (optional) is your domain name if you have it
- `EMAIL_ADDRESS` (required) is your email address used for communication regarding SSL certification (required) - `EMAIL_ADDRESS` (required) is your email address used for communication regarding SSL certification (required)
- `SIA_API_AUTHORIZATION` (required) is token you just generated in the previous point - `SIA_API_AUTHORIZATION` (required) is token you just generated in the previous point
- `CLOUDFLARE_AUTH_TOKEN` (optional) if using cloudflare as dns loadbalancer (it's just for siasky.net configuration)
- `HSD_API_KEY` (optional) this is a random security key for an optional handshake integration that gets generated automatically - `HSD_API_KEY` (optional) this is a random security key for an optional handshake integration that gets generated automatically
- `CLOUDFLARE_AUTH_TOKEN` (optional) if using cloudflare as dns loadbalancer (need to change it in Caddyfile too)
- `AWS_ACCESS_KEY_ID` (optional) if using route53 as a dns loadbalancer
- `AWS_SECRET_ACCESS_KEY` (optional) if using route53 as a dns loadbalancer
1. if you have a custom domain and you configured it in `DOMAIN_NAME`, edit `/home/user/skynet-webportal/docker/caddy/Caddyfile` and uncomment `import custom.domain` 1. if you have a custom domain and you configured it in `DOMAIN_NAME`, edit `/home/user/skynet-webportal/docker/caddy/Caddyfile` and uncomment `import custom.domain`
1. only for siasky.net domain instances: edit `/home/user/skynet-webportal/docker/caddy/Caddyfile`, uncomment `import siasky.net` 1. only for siasky.net domain instances: edit `/home/user/skynet-webportal/docker/caddy/Caddyfile`, uncomment `import siasky.net`
1. `sudo docker-compose up -d` to restart the services so they pick up new env variables 1. `sudo docker-compose up -d` to restart the services so they pick up new env variables
@ -106,30 +96,24 @@ At this point we have almost everything set up. We have 2 siad instances running
### Useful Commands ### Useful Commands
- Accessing siac for both nodes - Accessing siac
- `siac` for download node > `siac`
- `siac-upload` for upload node
- Checking status of siad service - Checking status of siad service
- `systemctl --user status siad` for download node > `systemctl --user status siad`
- `systemctl --user status siad-upload` for upload node
- Stopping siad service - Stopping siad service
- `systemctl --user stop siad` for download node > `systemctl --user stop siad`
- `systemctl --user stop siad-upload` for upload node
- Starting siad service - Starting siad service
- `systemctl --user start siad` for download node > `systemctl --user start siad`
- `systemctl --user start siad-upload` for upload node
- Restarting siad service - Restarting siad service
- `systemctl --user restart siad` for download node > `systemctl --user restart siad`
- `systemctl --user restart siad-upload` for upload node
- Restarting caddy gracefully after making changes to Caddyfile - Restarting caddy gracefully after making changes to Caddyfile
- `sudo docker exec caddy caddy reload --config /etc/caddy/Caddyfile` > `sudo docker exec caddy caddy reload --config /etc/caddy/Caddyfile`
- Restarting nginx gracefully after making changes to nginx configs - Restarting nginx gracefully after making changes to nginx configs
- `sudo docker exec nginx openresty -s reload` > `sudo docker exec nginx openresty -s reload`
- Checking siad service logs (follow last 50 lines) - Checking siad service logs (follow last 50 lines)
- `journalctl -f -n 50 --user-unit siad` for download node > `journalctl -f -n 50 --user-unit siad`
- `journalctl -f -n 50 --user-unit siad-upload` for upload node
- Checking caddy logs (for example in case ssl certificate fails) - Checking caddy logs (for example in case ssl certificate fails)
- `sudo docker logs caddy -f` > `sudo docker logs caddy -f`
- Checking nginx logs (nginx handles all communication to siad instances) - Checking nginx logs (nginx handles all communication to siad instances)
- `tail -n 50 docker/data/nginx/logs/access.log` to follow last 50 lines of access log > `tail -n 50 docker/data/nginx/logs/access.log` to follow last 50 lines of access log
- `tail -n 50 docker/data/nginx/logs/error.log` to follow last 50 lines of error log > `tail -n 50 docker/data/nginx/logs/error.log` to follow last 50 lines of error log

View File

@ -11,7 +11,6 @@ do
echo "⌁ Blacklisting on ${server}" echo "⌁ Blacklisting on ${server}"
ssh -q -t user@${server} 'curl -A Sia-Agent --user "":$(cat /home/user/.sia/apipassword) --data '"'"'{"add":["'$1'"]}'"'"' "localhost:9980/skynet/blacklist"' ssh -q -t user@${server} 'curl -A Sia-Agent --user "":$(cat /home/user/.sia/apipassword) --data '"'"'{"add":["'$1'"]}'"'"' "localhost:9980/skynet/blacklist"'
ssh -q -t user@${server} 'rm -rf /home/user/skynet_webportal/docker/data/nginx/cache' # remove cache from docker-managed portals ssh -q -t user@${server} 'rm -rf /home/user/skynet_webportal/docker/data/nginx/cache' # remove cache from docker-managed portals
ssh -q -t user@${server} 'sudo rm -rf /tmp/nginx' # remove cache from legacy non-docker portals
done done
echo "✓ All portals succesfully blacklisted provided skylink" echo "✓ All portals succesfully blacklisted provided skylink"

View File

@ -9,13 +9,7 @@ pip3 install discord.py
pip3 install python-dotenv pip3 install python-dotenv
downloadCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/.sia/sia.env" downloadCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/.sia/sia.env"
uploadCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/.sia/sia-upload.env"
logCheck1="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/.sia/sia.env siad 8" logCheck1="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/.sia/sia.env siad 8"
logCheck2="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/.sia/sia-upload.env siad-upload 8"
(crontab -u user -l; echo "$downloadCheck" ) | crontab -u user - (crontab -u user -l; echo "$downloadCheck" ) | crontab -u user -
(crontab -u user -l; echo "$uploadCheck" ) | crontab -u user -
(crontab -u user -l; echo "$logCheck1" ) | crontab -u user - (crontab -u user -l; echo "$logCheck1" ) | crontab -u user -
(crontab -u user -l; echo "$logCheck2" ) | crontab -u user -

View File

@ -27,17 +27,14 @@ make --directory /home/user/Sia
# Setup systemd files and restart daemon # Setup systemd files and restart daemon
mkdir -p /home/user/.config/systemd/user mkdir -p /home/user/.config/systemd/user
cp /home/user/skynet-webportal/setup-scripts/support/siad.service /home/user/.config/systemd/user/siad.service cp /home/user/skynet-webportal/setup-scripts/support/siad.service /home/user/.config/systemd/user/siad.service
cp /home/user/skynet-webportal/setup-scripts/support/siad-upload.service /home/user/.config/systemd/user/siad-upload.service
# Create siad data directories # Create siad data directories
mkdir -p /home/user/siad mkdir -p /home/user/siad
mkdir -p /home/user/siad-upload
# Setup files for storing environment variables # Setup files for storing environment variables
mkdir -p /home/user/.sia mkdir -p /home/user/.sia
# use -n flag to not override because these files store wallet information # use -n flag to not override because these files store wallet information
cp -n /home/user/skynet-webportal/setup-scripts/support/sia.env /home/user/.sia/sia.env cp -n /home/user/skynet-webportal/setup-scripts/support/sia.env /home/user/.sia/sia.env
cp -n /home/user/skynet-webportal/setup-scripts/support/sia-upload.env /home/user/.sia/sia-upload.env
# Setup persistent journal # Setup persistent journal
sudo mkdir -p /var/log/journal sudo mkdir -p /var/log/journal
@ -47,7 +44,6 @@ sudo systemctl restart systemd-journald
# Restart a daemon and enable both siad nodes (don't start yet) # Restart a daemon and enable both siad nodes (don't start yet)
systemctl --user daemon-reload systemctl --user daemon-reload
systemctl --user enable siad systemctl --user enable siad
systemctl --user enable siad-upload
# download siastats bootstrap (consensus and transactionpool) and apply it # download siastats bootstrap (consensus and transactionpool) and apply it
if ! [ -f /home/user/consensus.zip ]; then if ! [ -f /home/user/consensus.zip ]; then
@ -56,10 +52,6 @@ fi
if ! [ -f /home/user/siad/consensus/consensus.db ]; then if ! [ -f /home/user/siad/consensus/consensus.db ]; then
unzip -o /home/user/consensus.zip -d /home/user/siad unzip -o /home/user/consensus.zip -d /home/user/siad
fi fi
if ! [ -f /home/user/siad-upload/consensus/consensus.db ]; then
unzip -o /home/user/consensus.zip -d /home/user/siad-upload
fi
# start siad after the consesnsus has beed bootstraped # start siad after the consesnsus has beed bootstraped
systemctl --user start siad systemctl --user start siad
systemctl --user start siad-upload

View File

@ -116,5 +116,3 @@ export PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games:/usr/local/
set -o allexport set -o allexport
source /home/user/.sia/sia.env source /home/user/.sia/sia.env
set +o allexport set +o allexport
alias siac-upload="source /home/user/.sia/source-upload.sh; siac --addr 'localhost:9970'"

View File

@ -1,10 +0,0 @@
# siad environment variables
SIA_API_PASSWORD=""
SIA_DATA_DIR="/home/user/.sia"
SIAD_DATA_DIR="/home/user/siad-upload"
SIA_WALLET_PASSWORD=""
# portal specific environment variables
API_PORT="9970"
PORTAL_NAME="XXXXX | upload"
DISCORD_BOT_TOKEN=""

View File

@ -6,5 +6,5 @@ SIA_WALLET_PASSWORD=""
# portal specific environment variables # portal specific environment variables
API_PORT="9980" API_PORT="9980"
PORTAL_NAME="XXXXX | download" PORTAL_NAME=""
DISCORD_BOT_TOKEN="" DISCORD_BOT_TOKEN=""

View File

@ -1,15 +0,0 @@
[Unit]
Description=siad-upload
[Service]
Type=simple
WorkingDirectory=/home/user/siad-upload
EnvironmentFile=/home/user/.sia/sia-upload.env
ExecStart=/home/user/go/bin/siad --modules cgtwrf --disable-api-security --api-addr :9970 --rpc-addr :9971 --host-addr :9972 --siamux-addr :9973 --siamux-addr-ws :9974
ExecStop=/home/user/go/bin/siac --addr :9970 stop
Restart=on-failure
SyslogIdentifier=siad-upload
LimitNOFILE=10000
[Install]
WantedBy=default.target

View File

@ -1,6 +0,0 @@
#! /usr/bin/env bash
set -e
set -o allexport
source /home/user/.sia/sia-upload.env
set +o allexport