docker setup

docker progress

docker works

env variables
This commit is contained in:
Karol Wypchlo 2020-07-08 14:09:54 +02:00
parent b70d4f143e
commit 259e3ddffa
13 changed files with 119 additions and 206 deletions

View File

@ -5,15 +5,33 @@ networks:
driver: bridge
services:
docker-host:
image: qoomon/docker-host
container_name: docker-host
sia:
image: nebulouslabs/sia:dev
container_name: sia
restart: unless-stopped
cap_add:
- NET_ADMIN
- NET_RAW
environment:
- SIA_MODULES=gctwr
env_file:
- .env
volumes:
- ./docker/data/sia:/sia-data
networks:
- shared
expose:
- 9980
# sia-upload-legacy:
# image: nebulouslabs/sia:dev
# container_name: sia-upload-legacy
# restart: unless-stopped
# environment:
# - SIA_MODULES=gctwr
# volumes:
# - ./docker/data/sia-upload-legacy:/sia-data
# networks:
# - shared
# expose:
# - 9980
caddy:
build:
@ -33,7 +51,6 @@ services:
- "80:80"
- "443:443"
depends_on:
- docker-host
- nginx
nginx:
@ -45,16 +62,18 @@ services:
env_file:
- .env
volumes:
- ./docker/nginx/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf:ro
- ./docker/nginx/conf.d:/etc/nginx/conf.d:ro
- ./docker/data/nginx/cache:/data/nginx/cache
- ./docker/data/nginx/logs:/usr/local/openresty/nginx/logs
- ./docker/data/sia/apipassword:/data/sia/apipassword:ro
networks:
- shared
expose:
- 80
depends_on:
- docker-host
- sia
- health-check
- handshake-api
handshake:
build:
@ -94,7 +113,6 @@ services:
- 3100
depends_on:
- handshake
- nginx
health-check:
build:
@ -111,6 +129,3 @@ services:
- PORTAL_URL=nginx
expose:
- 3100
depends_on:
- docker-host
- nginx

View File

@ -13,12 +13,13 @@ set_real_ip_from 172.16.0.0/12;
set_real_ip_from 192.168.0.0/16;
real_ip_header X-Forwarded-For;
upstream siad-upload {
server docker-host:9970;
}
# sia-upload is for legacy portals that should keep the upload node alive
# upstream siad-upload {
# server sia-upload-legacy:9980;
# }
upstream siad {
server docker-host:9980;
server sia:9980;
}
server {

View File

@ -1,10 +1,15 @@
rewrite_by_lua_block {
-- local b64 = require("ngx.base64")
-- pull apipassword from SIA_API_AUTHORIZATION environment variable
-- local apipassword = os.getenv("SIA_API_AUTHORIZATION")
local b64 = require("ngx.base64")
-- open apipassword file for reading (b flag is required for some reason)
-- (file /etc/.sia/apipassword has to be mounted from the host system)
local apipassword_file = io.open("/data/sia/apipassword", "rb")
-- read apipassword file contents and trim newline (important)
local apipassword = apipassword_file:read("*all"):gsub("%s+", "")
-- make sure to close file after reading the password
apipassword_file.close()
-- encode the user:password authorization string
-- (in our case user is empty so it is just :password)
-- local content = b64.encode_base64url(":" .. apipassword)
local content = b64.encode_base64url(":" .. apipassword)
-- set authorization header with proper base64 encoded string
ngx.req.set_header("Authorization", "Basic " .. os.getenv("SIA_API_AUTHORIZATION"))
ngx.req.set_header("Authorization", "Basic " .. content)
}

View File

@ -1,64 +0,0 @@
# nginx.conf -- docker-openresty
#
# This file is installed to:
# `/usr/local/openresty/nginx/conf/nginx.conf`
# and is the file loaded by nginx at startup,
# unless the user specifies otherwise.
#
# It tracks the upstream OpenResty's `nginx.conf`, but removes the `server`
# section and adds this directive:
# `include /etc/nginx/conf.d/*.conf;`
#
# The `docker-openresty` file `nginx.vh.default.conf` is copied to
# `/etc/nginx/conf.d/default.conf`. It contains the `server section
# of the upstream `nginx.conf`.
#
# See https://github.com/openresty/docker-openresty/blob/master/README.md#nginx-config-files
#
# expose environment variables
env SIA_API_AUTHORIZATION;
#user nobody;
worker_processes 1;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
# See Move default writable paths to a dedicated directory (#119)
# https://github.com/openresty/docker-openresty/issues/119
client_body_temp_path /var/run/openresty/nginx-client-body;
proxy_temp_path /var/run/openresty/nginx-proxy;
fastcgi_temp_path /var/run/openresty/nginx-fastcgi;
uwsgi_temp_path /var/run/openresty/nginx-uwsgi;
scgi_temp_path /var/run/openresty/nginx-scgi;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
include /etc/nginx/conf.d/*.conf;
}

46
docker/sia/Dockerfile Normal file
View File

@ -0,0 +1,46 @@
FROM golang AS builder
LABEL maintainer="NebulousLabs <devs@nebulous.tech>"
ENV GOOS linux
ENV GOARCH amd64
RUN git clone https://gitlab.com/NebulousLabs/Sia.git && \
cd Sia && \
git checkout master && \
make release
RUN git clone https://github.com/NebulousLabs/docker-sia.git /docker-sia
FROM alpine:3
LABEL maintainer="NebulousLabs <devs@nebulous.tech>"
LABEL autoheal=true
ARG SIA_DIR="/sia"
ARG SIA_DATA_DIR="/sia-data"
ARG SIAD_DATA_DIR="/sia-data"
RUN mkdir /lib64 && \
ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 && \
apk add --no-cache socat
# Workaround for backwards compatibility with old images, which hardcoded the
# Sia data directory as /mnt/sia. Creates a symbolic link so that any previous
# path references stored in the Sia host config still work.
RUN ln -s "$SIA_DATA_DIR" /mnt/sia
WORKDIR "$SIA_DIR"
ENV SIA_DATA_DIR "$SIA_DATA_DIR"
ENV SIAD_DATA_DIR "$SIAD_DATA_DIR"
ENV SIA_MODULES gctwhr
COPY --from=builder /go/bin/siac .
COPY --from=builder /go/bin/siad .
COPY --from=builder /docker-sia/scripts/healthcheck.sh .
COPY --from=builder /docker-sia/scripts/run.sh .
EXPOSE 9980 9981 9982
HEALTHCHECK --interval=10s CMD ["./healthcheck.sh"]
ENTRYPOINT ["./run.sh"]

View File

@ -12,9 +12,9 @@ You may want to fork this repository and replace ssh keys in
### Step 0: stack overview
- dockerized services inside `docker-compose.yml`
- [docker-host](https://github.com/qoomon/docker-host) ([docker hub](https://hub.docker.com/r/qoomon/docker-host)): service that exposes server ip to docker container so we could access siad from within the nginx container
- [caddy](https://caddyserver.com) ([docker hub](https://hub.docker.com/r/caddy/caddy)): reverse proxy (similar to nginx) that handles ssl out of a box and acts as an entry point
- [openresty](https://openresty.org) ([docker hub](https://hub.docker.com/r/openresty/openresty)): nginx custom build, acts as a cached proxy to siad (we only use it because caddy doesn't support proxy caching, otherwise we could drop it)
- [sia](https://sia.tech) ([docker hub](https://hub.docker.com/r/nebulouslabs/sia)): storage provider, heart of the portal setup
- [caddy](https://caddyserver.com) ([docker hub](https://hub.docker.com/r/caddy/caddy)): reverse proxy (similar to nginx) that handles ssl out of a box and acts as a transparent entry point
- [openresty](https://openresty.org) ([docker hub](https://hub.docker.com/r/openresty/openresty)): nginx custom build, acts as a cached proxy to siad and exposes all api endpoints
- health-check: this is a simple service that runs periodically and collects health data about the server (status and response times) and exposes `/health-check` api endpoint that is deliberately delayed based on the response times of the server so potential load balancer could prioritize servers based on that (we use it with cloudflare)
- siad setup: we use "double siad" setup that has one node solely for download and one for upload to improve performance
- we use systemd to manage siad service
@ -30,7 +30,6 @@ You may want to fork this repository and replace ssh keys in
1. `apt-get update && apt-get install sudo` to make sure `sudo` is available
1. `adduser user` to create user called `user` (creates `/home/user` directory)
1. `usermod -a -G sudo user` to add this new user to sudo group
1. `usermod -a -G systemd-journal user` to add this new user to systemd-journal group
1. Quit the ssh session with `exit` command
You a can now ssh into your machine as the user `user`.
@ -47,21 +46,21 @@ You a can now ssh into your machine as the user `user`.
1. `git clone https://github.com/NebulousLabs/skynet-webportal`
1. run setup scripts in the exact order and provide sudo password when asked (if one of them fails, you can retry just this one before proceeding further)
1. `/home/user/skynet-webportal/setup-scripts/setup-server.sh`
1. `/home/user/skynet-webportal/setup-scripts/setup-siad.sh`
1. `/home/user/skynet-webportal/setup-scripts/setup-docker-services.sh`
1. `/home/user/skynet-webportal/setup-scripts/setup-health-check-scripts.sh` (optional)
### Step 3: configuring siad
At this point we have almost everything set up. We have 2 siad instances running as services and we need to set up the wallets and allowance on those.
At this point we have almost everything running, we just need to set up your wallet and allowance:
1. Create new wallet (remember to save the seeds)
> `siac wallet init`
> `docker exec -it sia siac wallet init`
1. Unlock wallet (use seed as password)
> `siac wallet unlock`
> `docker exec -it sia siac wallet unlock`
1. Generate wallet addresse (save them for later to transfer the funds)
> `siac wallet address`
1. Set up allowance by running `siac renter setallowance`
> `docker exec -it sia siac wallet address`
1. Set up allowance
> `docker exec -it sia siac renter setallowance`
1. 10 KS (keep 25 KS in your wallet)
1. default period
1. default number of hosts
@ -70,13 +69,8 @@ At this point we have almost everything set up. We have 2 siad instances running
1. 500 GB expected upload
1. 5 TB expected download
1. default redundancy
1. Run `siac renter setallowance --payment-contract-initial-funding 10SC` so siad will start making 10 contracts per block with many hosts to potentially view the whole network's files
1. Copy over apipassword from `/home/user/.sia/apipassword` and save it for the next step
1. Edit environment file for siad `/home/user/.sia/sia.env` and set:
1. `SIA_API_PASSWORD` to previously copied apipassword (same for both instances)
1. `SIA_WALLET_PASSWORD` to be the wallet seed
1. `PORTAL_NAME` (optional) only for bot utils, set it to something meaningful name like `warsaw.siasky.net`
1. `DISCORD_BOT_TOKEN` for discord health check scripts integration
1. Instruct siad to start making 10 contracts per block with many hosts to potentially view the whole network's files
> `docker exec -it sia siac renter setallowance --payment-contract-initial-funding 10SC`
### Step 4: configuring docker services
@ -84,20 +78,20 @@ At this point we have almost everything set up. We have 2 siad instances running
1. edit `/home/user/skynet-webportal/.env` and configure following environment variables
- `DOMAIN_NAME` (optional) is your domain name if you have it
- `EMAIL_ADDRESS` (required) is your email address used for communication regarding SSL certification (required)
- `SIA_API_AUTHORIZATION` (required) is token you just generated in the previous point
- `SIA_WALLET_PASSWORD` (required) is your wallet password (or seed if you did not set a password)
- `HSD_API_KEY` (optional) this is a random security key for an optional handshake integration that gets generated automatically
- `CLOUDFLARE_AUTH_TOKEN` (optional) if using cloudflare as dns loadbalancer (need to change it in Caddyfile too)
- `AWS_ACCESS_KEY_ID` (optional) if using route53 as a dns loadbalancer
- `AWS_SECRET_ACCESS_KEY` (optional) if using route53 as a dns loadbalancer
1. if you have a custom domain and you configured it in `DOMAIN_NAME`, edit `/home/user/skynet-webportal/docker/caddy/Caddyfile` and uncomment `import custom.domain`
1. only for siasky.net domain instances: edit `/home/user/skynet-webportal/docker/caddy/Caddyfile`, uncomment `import siasky.net`
1. `sudo docker-compose up -d` to restart the services so they pick up new env variables
1. `sudo docker exec caddy caddy reload --config /etc/caddy/Caddyfile` to reload Caddyfile configuration
1. `docker-compose up -d` to restart the services so they pick up new env variables
1. `docker exec caddy caddy reload --config /etc/caddy/Caddyfile` to reload Caddyfile configuration
### Useful Commands
## Useful Commands
- Accessing siac
> `siac`
> `docker exec -it sia siac`
- Checking status of siad service
> `systemctl --user status siad`
- Stopping siad service
@ -107,13 +101,13 @@ At this point we have almost everything set up. We have 2 siad instances running
- Restarting siad service
> `systemctl --user restart siad`
- Restarting caddy gracefully after making changes to Caddyfile
> `sudo docker exec caddy caddy reload --config /etc/caddy/Caddyfile`
> `docker exec caddy caddy reload --config /etc/caddy/Caddyfile`
- Restarting nginx gracefully after making changes to nginx configs
> `sudo docker exec nginx openresty -s reload`
> `docker exec nginx openresty -s reload`
- Checking siad service logs (follow last 50 lines)
> `journalctl -f -n 50 --user-unit siad`
- Checking caddy logs (for example in case ssl certificate fails)
> `sudo docker logs caddy -f`
> `docker logs caddy -f`
- Checking nginx logs (nginx handles all communication to siad instances)
> `tail -n 50 docker/data/nginx/logs/access.log` to follow last 50 lines of access log
> `tail -n 50 docker/data/nginx/logs/error.log` to follow last 50 lines of error log

View File

@ -9,8 +9,8 @@ fi
for server in "germany.siasky.net" "us-east.siasky.net" "us-west.siasky.net" "helsinki.siasky.net" "siasky.dev";
do
echo "⌁ Blacklisting on ${server}"
ssh -q -t user@${server} 'curl -A Sia-Agent --user "":$(cat /home/user/.sia/apipassword) --data '"'"'{"add":["'$1'"]}'"'"' "localhost:9980/skynet/blacklist"'
ssh -q -t user@${server} 'rm -rf /home/user/skynet_webportal/docker/data/nginx/cache' # remove cache from docker-managed portals
ssh -q -t user@${server} 'docker exec sia siac skynet blacklist '$1''
ssh -q -t user@${server} 'rm -rf /home/user/skynet_webportal/docker/data/nginx/cache' # prune nginx cache
done
echo "✓ All portals succesfully blacklisted provided skylink"

View File

@ -18,14 +18,16 @@ sudo chmod +x /usr/local/bin/docker-compose
docker-compose --version # sanity check
# Create dummy .env file for docker-compose usage with veriables
# DOMAIN_NAME - the domain name your server is using ie. example.com
# EMAIL_ADDRESS - this is the administrator contact email you need to supply for communication regarding SSL certification
# SIA_API_AUTHORIZATION - the base64 encoded :apipassword string
# CLOUDFLARE_AUTH_TOKEN - cloudflare auth token for ssl generation (just for siasky.net)
# * DOMAIN_NAME - the domain name your server is using ie. example.com
# * EMAIL_ADDRESS - this is the administrator contact email you need to supply for communication regarding SSL certification
# * HSD_API_KEY - this is auto generated secure key for your handshake service integration
# * CLOUDFLARE_AUTH_TOKEN` - (optional) if using cloudflare as dns loadbalancer (need to change it in Caddyfile too)
# * AWS_ACCESS_KEY_ID - (optional) if using route53 as a dns loadbalancer
# * AWS_SECRET_ACCESS_KEY - (optional) if using route53 as a dns loadbalancer
if ! [ -f /home/user/skynet-webportal/.env ]; then
HSD_API_KEY=$(openssl rand -base64 32) # generate safe random key for handshake
printf "DOMAIN_NAME=example.com\nEMAIL_ADDRESS=email@example.com\nSIA_API_AUTHORIZATION=\nCLOUDFLARE_AUTH_TOKEN=\nHSD_API_KEY=${HSD_API_KEY}\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\n" > /home/user/skynet-webportal/.env
printf "DOMAIN_NAME=example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\n" > /home/user/skynet-webportal/.env
fi
# Start docker container with nginx and client
sudo docker-compose -f docker-compose.yml up --build -d
docker-compose -f docker-compose.yml up --build -d

View File

@ -25,13 +25,6 @@ sudo ufw --force enable # --force to make it non-interactive
sudo ufw logging low # enable logging for debugging purpose: tail -f /var/log/ufw.log
sudo ufw allow ssh # allow ssh connection to server
sudo ufw allow 80,443/tcp # allow http and https ports
sudo ufw allow proto tcp from any to 172.0.0.0/8 port 9970,9980 # expose siad api ports to local network
sudo ufw allow proto tcp from any to 192.168.0.0/16 port 9970,9980 # expose siad api ports to local network
# Setup periodical /tmp cleanup so we don't run out of disk space
# - deletes anything older than 10 days from /tmp, crontab is set to run it every day at midnight
# WARNING: if you run this job more than once, make sure to either comment this out or clean crontab from duplicates
(sudo crontab -l 2>/dev/null; echo "0 0 * * * find /tmp -type f -atime +10 -delete >/dev/null 2>&1") | sudo crontab -
# OPTIONAL: terminfo for alacritty terminal via ssh
# If you don't use the alacritty terminal you can remove this step.
@ -41,6 +34,3 @@ rm alacritty.info
# Set up file limits - siad uses a lot so we need to adjust so it doesn't choke up
sudo cp /home/user/skynet-webportal/setup-scripts/support/limits.conf /etc/security/limits.conf
# Enable lingering services, it prevents services shutdown when you log out of the server
loginctl enable-linger user

View File

@ -1,57 +0,0 @@
#! /usr/bin/env bash
set -e # exit on first error
# Setup constants
GO_VERSION=1.13.11
SIA_BRANCH_OR_TAG=v1.4.11
# Install Go
wget -c https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz
sudo tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz
rm go${GO_VERSION}.linux-amd64.tar.gz
# add gopath to PATH and persist it in /etc/profile
export PATH="${PATH}:/usr/local/go/bin:/home/user/go/bin"
echo "export PATH=${PATH}" | sudo tee /etc/profile.d/go_path.sh
# Sanity check that will pass if go was installed correctly.
go version
# Install Sia
rm -rf /home/user/Sia
git clone https://gitlab.com/NebulousLabs/Sia.git /home/user/Sia
git -C /home/user/Sia checkout ${SIA_BRANCH_OR_TAG}
make --directory /home/user/Sia
# Setup systemd files and restart daemon
mkdir -p /home/user/.config/systemd/user
cp /home/user/skynet-webportal/setup-scripts/support/siad.service /home/user/.config/systemd/user/siad.service
# Create siad data directories
mkdir -p /home/user/siad
# Setup files for storing environment variables
mkdir -p /home/user/.sia
# use -n flag to not override because these files store wallet information
cp -n /home/user/skynet-webportal/setup-scripts/support/sia.env /home/user/.sia/sia.env
# Setup persistent journal
sudo mkdir -p /var/log/journal
sudo cp /home/user/skynet-webportal/setup-scripts/support/journald.conf /etc/systemd/journald.conf
sudo systemctl restart systemd-journald
# Restart a daemon and enable both siad nodes (don't start yet)
systemctl --user daemon-reload
systemctl --user enable siad
# download siastats bootstrap (consensus and transactionpool) and apply it
if ! [ -f /home/user/consensus.zip ]; then
curl https://siastats.info/bootstrap/bootstrap.zip -o /home/user/consensus.zip
fi
if ! [ -f /home/user/siad/consensus/consensus.db ]; then
unzip -o /home/user/consensus.zip -d /home/user/siad
fi
# start siad after the consesnsus has beed bootstraped
systemctl --user start siad

View File

@ -113,6 +113,4 @@ if ! shopt -oq posix; then
fi
export PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games:/usr/local/go/bin:/home/user/go/bin
set -o allexport
source /home/user/.sia/sia.env
set +o allexport
alias siac="docker exec -it sia siac"

View File

@ -1,2 +0,0 @@
[Journal]
Storage=persistent

View File

@ -1,15 +0,0 @@
[Unit]
Description=siad
[Service]
Type=simple
WorkingDirectory=/home/user/siad
EnvironmentFile=/home/user/.sia/sia.env
ExecStart=/home/user/go/bin/siad --modules cgtwrf --disable-api-security --api-addr :9980
ExecStop=/home/user/go/bin/siac --addr :9980 stop
Restart=on-failure
SyslogIdentifier=siad
LimitNOFILE=10000
[Install]
WantedBy=default.target