diff --git a/scripts/backup-aws-s3.sh b/scripts/backup-aws-s3.sh new file mode 100644 index 00000000..64af8ab1 --- /dev/null +++ b/scripts/backup-aws-s3.sh @@ -0,0 +1,43 @@ +#! /usr/bin/env bash + +set -e # exit on first error + +echo "`date +"%Y-%m-%d %H:%M"` Starting backup process" + +# ensure aws cli is upgraded +pip3 install --quiet --upgrade awscli + +# import required environment variables from .env file +ENV_VARIABLES=("AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY" "PORTAL_DOMAIN" "SERVER_DOMAIN" "SERVER_UID") + +for ENV_VARIABLE in "${ENV_VARIABLES[@]}"; do + ENV_VARIABLE_VALUE=$(grep -v '^#' /home/user/skynet-webportal/.env | grep ${ENV_VARIABLE} || true) + if test -z "${ENV_VARIABLE_VALUE}"; then + if $ENV_VARIABLE ~= "SERVER_DOMAIN"; then + echo "Environment variable ${ENV_VARIABLE} is not set" && exit 1 + fi + fi + export ${ENV_VARIABLE_VALUE} +done + +# create bucket skynet-backup-[portaldomain] (portal domain is stripped of all non alnum characters) +# ie. siasky.net backup results in skynet-backup-siaskynet basket name +BUCKET_NAME=$(echo skynet-backup-${PORTAL_DOMAIN} | tr -cd '[[:alnum:]]') + +# create server prefix +if test -z "${SERVER_DOMAIN}"; then + SERVER_PREFIX=$(echo ${SERVER_UID} | tr -cd '[[:alnum:]]-') # if domain name is empty use just uid +else + SERVER_PREFIX=$(echo ${SERVER_UID}-${SERVER_DOMAIN} | tr -cd '[[:alnum:]]-') # use both uid and server domain if available +fi + +aws s3api create-bucket --acl private --bucket ${BUCKET_NAME} + +# sync all nginx logs +aws s3 sync --no-progress /home/user/skynet-webportal/docker/data/nginx/logs s3://${BUCKET_NAME}/${SERVER_PREFIX}/docker/data/nginx/logs + +# generate and sync skylinks +docker exec sia siac skynet ls --recursive --alert-suppress > /home/user/skynet-webportal/logs/skylinks-$(date +"%Y-%m-%d").log +aws s3 cp --no-progress /home/user/skynet-webportal/logs/skylinks.log s3://${BUCKET_NAME}/${SERVER_PREFIX}/logs/skylinks.log + +echo "`date +"%Y-%m-%d %H:%M"` Backup finished successfully" diff --git a/setup-scripts/support/crontab b/setup-scripts/support/crontab index 9bc5cd34..fb195ca5 100644 --- a/setup-scripts/support/crontab +++ b/setup-scripts/support/crontab @@ -2,6 +2,7 @@ 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1 30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env +3 44 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log 6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh