clean up double siad setup

docker/nginx/conf.d/client.conf

@@ -13,18 +13,11 @@ set_real_ip_from  172.16.0.0/12;
 set_real_ip_from  192.168.0.0/16;
 real_ip_header    X-Forwarded-For;
 
-# note that we point uploads to port '9970', do this when you want to
-# run in a configuration where you have two siad instances, one for
-# downloads and one for uploads. This drastically improves the up - and
-# download speed of your portal. When running your portal in this double
-# siad setup, make sure only the download portal runs in 'portal mode'.
-# The upload siad can be run in normal mode. Set the port to '9980' if
-# you do not want to run your portal in the double siad setup.
 upstream siad-upload {
-	server docker-host:9980;
+	server docker-host:9970;
 }
 
-upstream siad-download {
+upstream siad {
 	server docker-host:9980;
 }
 
@@ -53,7 +46,7 @@ server {
 		proxy_cache skynet;
 		proxy_cache_valid any 1m; # cache blacklist for 1 minute
 		proxy_set_header User-Agent: Sia-Agent;
-		proxy_pass http://siad-download/skynet/blacklist;
+		proxy_pass http://siad/skynet/blacklist;
 	}
 
 	location /skynet/blacklist {
@@ -62,7 +55,7 @@ server {
 		proxy_cache skynet;
 		proxy_cache_valid any 1m; # cache blacklist for 1 minute
 		proxy_set_header User-Agent: Sia-Agent;
-		proxy_pass http://siad-download/skynet/blacklist;
+		proxy_pass http://siad/skynet/blacklist;
 	}
 
 	location /portals {
@@ -71,7 +64,7 @@ server {
 		proxy_cache skynet;
 		proxy_cache_valid any 1m; # cache portals for 1 minute
 		proxy_set_header User-Agent: Sia-Agent;
-		proxy_pass http://siad-download/skynet/portals;
+		proxy_pass http://siad/skynet/portals;
 	}
 
 	location /skynet/portals {
@@ -80,7 +73,7 @@ server {
 		proxy_cache skynet;
 		proxy_cache_valid any 1m; # cache portals for 1 minute
 		proxy_set_header User-Agent: Sia-Agent;
-		proxy_pass http://siad-download/skynet/portals;
+		proxy_pass http://siad/skynet/portals;
 	}
 
 	location /stats {
@@ -89,7 +82,7 @@ server {
 		proxy_cache skynet;
 		proxy_cache_valid any 1m; # cache stats for 1 minute
 		proxy_set_header User-Agent: Sia-Agent;
-		proxy_pass http://siad-upload/skynet/stats;
+		proxy_pass http://siad/skynet/stats;
 	}
 
 	location /skynet/stats {
@@ -98,7 +91,7 @@ server {
 		proxy_cache skynet;
 		proxy_cache_valid any 1m; # cache stats for 1 minute
 		proxy_set_header User-Agent: Sia-Agent;
-		proxy_pass http://siad-upload/skynet/stats;
+		proxy_pass http://siad/skynet/stats;
 	}
 
 	location /statsdown {
@@ -107,7 +100,7 @@ server {
 		proxy_cache skynet;
 		proxy_cache_valid any 1m; # cache stats for 1 minute
 		proxy_set_header User-Agent: Sia-Agent;
-		proxy_pass http://siad-download/skynet/stats;
+		proxy_pass http://siad/skynet/stats; # serve upload node stats temporarily
 	}
 
 	location /health-check {
@@ -153,7 +146,7 @@ server {
 		}
 
 		# proxy this call to siad endpoint (make sure the ip is correct)
-		proxy_pass http://siad-upload/skynet/skyfile/$dir1/$dir2/$dir3/$dir4$is_args$args;
+		proxy_pass http://siad/skynet/skyfile/$dir1/$dir2/$dir3/$dir4$is_args$args;
 	}
 	
 	location ~ "/skynet/skyfile/(.+)" {
@@ -172,7 +165,7 @@ server {
 		set $siapath $1;
 
 		# proxy this call to siad endpoint (make sure the ip is correct)
-		proxy_pass http://siad-upload/skynet/skyfile/$siapath$is_args$args;
+		proxy_pass http://siad/skynet/skyfile/$siapath$is_args$args;
 	}
 
 	location ~ "^/([a-zA-Z0-9-_]{46}(/.*)?)$" {
@@ -187,7 +180,7 @@ server {
 		proxy_read_timeout 600;
 		proxy_set_header User-Agent: Sia-Agent;
 		# proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct)
-		proxy_pass http://siad-download/skynet/skylink/$skylink$is_args$args;
+		proxy_pass http://siad/skynet/skylink/$skylink$is_args$args;
 		
 		# if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs
 		proxy_buffer_size 128k;
@@ -213,7 +206,7 @@ server {
 		proxy_set_header User-Agent: Sia-Agent;
 		# proxy this call to siad /skynet/skylink/ endpoint (make sure the ip is correct)
 		# this alias also adds attachment=true url param to force download the file
-		proxy_pass http://siad-download/skynet/skylink/$skylink?attachment=true&$args;
+		proxy_pass http://siad/skynet/skylink/$skylink?attachment=true&$args;
 
 		# if you are expecting large headers (ie. Skynet-Skyfile-Metadata), tune these values to your needs
 		proxy_buffer_size 128k;

setup-scripts/README.md

@@ -17,7 +17,7 @@ You may want to fork this repository and replace ssh keys in
   - [openresty](https://openresty.org) ([docker hub](https://hub.docker.com/r/openresty/openresty)): nginx custom build, acts as a cached proxy to siad (we only use it because caddy doesn't support proxy caching, otherwise we could drop it)
   - health-check: this is a simple service that runs periodically and collects health data about the server (status and response times) and exposes `/health-check` api endpoint that is deliberately delayed based on the response times of the server so potential load balancer could prioritize servers based on that (we use it with cloudflare)
 - siad setup: we use "double siad" setup that has one node solely for download and one for upload to improve performance
-  - we use systemd to manage siad services
+  - we use systemd to manage siad service
   - siad is not installed as docker service for improved performance
 - discord integration
   - [funds-checker](funds-checker.py): script that checks wallet balance and sends status messages to discord periodically
@@ -55,40 +55,28 @@ You a can now ssh into your machine as the user `user`.
 
 At this point we have almost everything set up. We have 2 siad instances running as services and we need to set up the wallets and allowance on those.
 
-1. Create new wallet for both siad instances (remember to save the seeds)
+1. Create new wallet (remember to save the seeds)
-   1. `siac wallet init` to init download node wallet
+   > `siac wallet init`
-   1. `siac-upload wallet init` to init upload node wallet
+1. Unlock wallet (use seed as password)
-1. Unlock both wallets
+   > `siac wallet unlock`
-   1. `siac wallet unlock` to unlock download node wallet (use seed as password)
+1. Generate wallet addresse (save them for later to transfer the funds)
-   1. `siac-upload wallet unlock` to unlock upload node wallet (use seed as password)
+   > `siac wallet address`
-1. Generate wallet addresses for both siad instances (save them for later to transfer the funds)
+1. Set up allowance by running `siac renter setallowance`
-   1. `siac wallet address` to generate address for download node wallet
+   1. 10 KS (keep 25 KS in your wallet)
-   1. `siac-upload wallet address` to generate address for upload node wallet
+   1. default period
-1. Set up allowance on both siad instances
+   1. default number of hosts
-   1. `siac renter setallowance` to set allowance on download node
+   1. 8 week renewal time
-      1. 10 KS (keep 25 KS in your wallet)
+   1. 500 GB expected storage
-      1. default period
+   1. 500 GB expected upload
-      1. default number of hosts
+   1. 5 TB expected download
-      1. 8 week renewal time
+   1. default redundancy
-      1. 500 GB expected storage
+1. Run `siac renter setallowance --payment-contract-initial-funding 10SC` so siad will start making 10 contracts per block with many hosts to potentially view the whole network's files
-      1. 500 GB expected upload
-      1. 5 TB expected download
-      1. default redundancy
-   1. `siac-upload renter setallowance` to set allowance on upload node
-      1. use the same allowance settings as download node
-1. Run `siac renter setallowance --payment-contract-initial-funding 10SC` so that your download node will start making 10 contracts per block with many hosts to potentially view the whole network's files
 1. Copy over apipassword from `/home/user/.sia/apipassword` and save it for the next step
-1. Edit environment files for both siad instances
+1. Edit environment file for siad `/home/user/.sia/sia.env` and set:
-   1. `/home/user/.sia/sia.env` for the download node
+   1. `SIA_API_PASSWORD` to previously copied apipassword (same for both instances)
-      1. `SIA_API_PASSWORD` to previously copied apipassword (same for both instances)
+   1. `SIA_WALLET_PASSWORD` to be the wallet seed
-      1. `SIA_WALLET_PASSWORD` to be the wallet seed
+   1. `PORTAL_NAME` (optional) only for bot utils, set it to something meaningful name like `warsaw.siasky.net`
-      1. `PORTAL_NAME` xxxxed part to some meaningful name like `warsaw.siasky.net`
+   1. `DISCORD_BOT_TOKEN` for discord health check scripts integration
-      1. `DISCORD_BOT_TOKEN` for discord health check scripts integration
-   1. `/home/user/.sia/sia-upload.env` for the upload node
-      1. `SIA_API_PASSWORD` to previously copied apipassword (same for both instances)
-      1. `SIA_WALLET_PASSWORD` to be the wallet seed
-      1. `PORTAL_NAME` xxxxed part to some meaningful name like `warsaw.siasky.net`
-      1. `DISCORD_BOT_TOKEN` for discord health check scripts integration
 
 ### Step 4: configuring docker services
 
@@ -97,8 +85,10 @@ At this point we have almost everything set up. We have 2 siad instances running
    - `DOMAIN_NAME` (optional) is your domain name if you have it
    - `EMAIL_ADDRESS` (required) is your email address used for communication regarding SSL certification (required)
    - `SIA_API_AUTHORIZATION` (required) is token you just generated in the previous point
-   - `CLOUDFLARE_AUTH_TOKEN` (optional) if using cloudflare as dns loadbalancer (it's just for siasky.net configuration)
    - `HSD_API_KEY` (optional) this is a random security key for an optional handshake integration that gets generated automatically
+   - `CLOUDFLARE_AUTH_TOKEN` (optional) if using cloudflare as dns loadbalancer (need to change it in Caddyfile too)
+   - `AWS_ACCESS_KEY_ID` (optional) if using route53 as a dns loadbalancer
+   - `AWS_SECRET_ACCESS_KEY` (optional) if using route53 as a dns loadbalancer
 1. if you have a custom domain and you configured it in `DOMAIN_NAME`, edit `/home/user/skynet-webportal/docker/caddy/Caddyfile` and uncomment `import custom.domain`
 1. only for siasky.net domain instances: edit `/home/user/skynet-webportal/docker/caddy/Caddyfile`, uncomment `import siasky.net`
 1. `sudo docker-compose up -d` to restart the services so they pick up new env variables
@@ -106,30 +96,24 @@ At this point we have almost everything set up. We have 2 siad instances running
 
 ### Useful Commands
 
-- Accessing siac for both nodes
+- Accessing siac
-  - `siac` for download node
+  > `siac`
-  - `siac-upload` for upload node
 - Checking status of siad service
-  - `systemctl --user status siad` for download node
+  > `systemctl --user status siad`
-  - `systemctl --user status siad-upload` for upload node
 - Stopping siad service
-  - `systemctl --user stop siad` for download node
+  > `systemctl --user stop siad`
-  - `systemctl --user stop siad-upload` for upload node
 - Starting siad service
-  - `systemctl --user start siad` for download node
+  > `systemctl --user start siad`
-  - `systemctl --user start siad-upload` for upload node
 - Restarting siad service
-  - `systemctl --user restart siad` for download node
+  > `systemctl --user restart siad`
-  - `systemctl --user restart siad-upload` for upload node
 - Restarting caddy gracefully after making changes to Caddyfile
-  - `sudo docker exec caddy caddy reload --config /etc/caddy/Caddyfile`
+  > `sudo docker exec caddy caddy reload --config /etc/caddy/Caddyfile`
 - Restarting nginx gracefully after making changes to nginx configs
-  - `sudo docker exec nginx openresty -s reload`
+  > `sudo docker exec nginx openresty -s reload`
 - Checking siad service logs (follow last 50 lines)
-  - `journalctl -f -n 50 --user-unit siad` for download node
+  > `journalctl -f -n 50 --user-unit siad`
-  - `journalctl -f -n 50 --user-unit siad-upload` for upload node
 - Checking caddy logs (for example in case ssl certificate fails)
-  - `sudo docker logs caddy -f`
+  > `sudo docker logs caddy -f`
 - Checking nginx logs (nginx handles all communication to siad instances)
-  - `tail -n 50 docker/data/nginx/logs/access.log` to follow last 50 lines of access log
+  > `tail -n 50 docker/data/nginx/logs/access.log` to follow last 50 lines of access log
-  - `tail -n 50 docker/data/nginx/logs/error.log` to follow last 50 lines of error log
+  > `tail -n 50 docker/data/nginx/logs/error.log` to follow last 50 lines of error log

setup-scripts/blacklist-skylink.sh

@@ -11,7 +11,6 @@ do
     echo "⌁ Blacklisting on ${server}"
     ssh -q -t user@${server} 'curl -A Sia-Agent --user "":$(cat /home/user/.sia/apipassword) --data '"'"'{"add":["'$1'"]}'"'"' "localhost:9980/skynet/blacklist"'
     ssh -q -t user@${server} 'rm -rf /home/user/skynet_webportal/docker/data/nginx/cache' # remove cache from docker-managed portals
-    ssh -q -t user@${server} 'sudo rm -rf /tmp/nginx' # remove cache from legacy non-docker portals
 done
 
 echo "✓ All portals succesfully blacklisted provided skylink"

setup-scripts/setup-health-check-scripts.sh

@@ -9,13 +9,7 @@ pip3 install discord.py
 pip3 install python-dotenv
 
 downloadCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/.sia/sia.env"
-uploadCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/.sia/sia-upload.env"
-
 logCheck1="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/.sia/sia.env siad 8"
-logCheck2="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/.sia/sia-upload.env siad-upload 8"
 
 (crontab -u user -l; echo "$downloadCheck" ) | crontab -u user -
-(crontab -u user -l; echo "$uploadCheck" ) | crontab -u user -
-
 (crontab -u user -l; echo "$logCheck1" ) | crontab -u user -
-(crontab -u user -l; echo "$logCheck2" ) | crontab -u user -

setup-scripts/setup-siad.sh

@@ -27,17 +27,14 @@ make --directory /home/user/Sia
 # Setup systemd files and restart daemon
 mkdir -p /home/user/.config/systemd/user
 cp /home/user/skynet-webportal/setup-scripts/support/siad.service /home/user/.config/systemd/user/siad.service
-cp /home/user/skynet-webportal/setup-scripts/support/siad-upload.service /home/user/.config/systemd/user/siad-upload.service
 
 # Create siad data directories
 mkdir -p /home/user/siad
-mkdir -p /home/user/siad-upload
 
 # Setup files for storing environment variables
 mkdir -p /home/user/.sia
 # use -n flag to not override because these files store wallet information
 cp -n /home/user/skynet-webportal/setup-scripts/support/sia.env /home/user/.sia/sia.env
-cp -n /home/user/skynet-webportal/setup-scripts/support/sia-upload.env /home/user/.sia/sia-upload.env
 
 # Setup persistent journal
 sudo mkdir -p /var/log/journal
@@ -47,7 +44,6 @@ sudo systemctl restart systemd-journald
 # Restart a daemon and enable both siad nodes (don't start yet)
 systemctl --user daemon-reload
 systemctl --user enable siad
-systemctl --user enable siad-upload
 
 # download siastats bootstrap (consensus and transactionpool) and apply it
 if ! [ -f /home/user/consensus.zip ]; then
@@ -56,10 +52,6 @@ fi
 if ! [ -f /home/user/siad/consensus/consensus.db ]; then
     unzip -o /home/user/consensus.zip -d /home/user/siad
 fi
-if ! [ -f /home/user/siad-upload/consensus/consensus.db ]; then
-    unzip -o /home/user/consensus.zip -d /home/user/siad-upload
-fi
 
 # start siad after the consesnsus has beed bootstraped
 systemctl --user start siad
-systemctl --user start siad-upload

setup-scripts/support/bashrc

@@ -116,5 +116,3 @@ export PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games:/usr/local/
 set -o allexport
 source /home/user/.sia/sia.env
 set +o allexport
-
-alias siac-upload="source /home/user/.sia/source-upload.sh; siac --addr 'localhost:9970'"

setup-scripts/support/sia-upload.env

@@ -1,10 +0,0 @@
-# siad environment variables
-SIA_API_PASSWORD=""
-SIA_DATA_DIR="/home/user/.sia"
-SIAD_DATA_DIR="/home/user/siad-upload"
-SIA_WALLET_PASSWORD=""
-
-# portal specific environment variables
-API_PORT="9970"
-PORTAL_NAME="XXXXX | upload"
-DISCORD_BOT_TOKEN=""

setup-scripts/support/sia.env

@@ -6,5 +6,5 @@ SIA_WALLET_PASSWORD=""
 
 # portal specific environment variables
 API_PORT="9980"
-PORTAL_NAME="XXXXX | download"
+PORTAL_NAME=""
 DISCORD_BOT_TOKEN=""

setup-scripts/support/siad-upload.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=siad-upload
-
-[Service]
-Type=simple
-WorkingDirectory=/home/user/siad-upload
-EnvironmentFile=/home/user/.sia/sia-upload.env
-ExecStart=/home/user/go/bin/siad --modules cgtwrf --disable-api-security --api-addr :9970 --rpc-addr :9971 --host-addr :9972 --siamux-addr :9973 --siamux-addr-ws :9974
-ExecStop=/home/user/go/bin/siac --addr :9970 stop
-Restart=on-failure
-SyslogIdentifier=siad-upload
-LimitNOFILE=10000
-
-[Install]
-WantedBy=default.target

setup-scripts/support/source-upload.sh

@@ -1,6 +0,0 @@
-#! /usr/bin/env bash
-set -e
-
-set -o allexport
-source /home/user/.sia/sia-upload.env
-set +o allexport