LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

incremental logs and skylinks backup

This commit is contained in:
Karol Wypchlo 2022-02-11 14:25:34 +01:00
parent 9aa21088c5
commit 3e5c576902
No known key found for this signature in database
GPG Key ID: B515DE9EEBE241E1
2 changed files with 44 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
scripts/backup-aws-s3.sh Normal file
View File

@ -0,0 +1,43 @@
#! /usr/bin/env bash
set -e # exit on first error
echo "`date +"%Y-%m-%d %H:%M"` Starting backup process"
# ensure aws cli is upgraded
pip3 install --quiet --upgrade awscli
# import required environment variables from .env file
ENV_VARIABLES=("AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY" "PORTAL_DOMAIN" "SERVER_DOMAIN" "SERVER_UID")
for ENV_VARIABLE in "${ENV_VARIABLES[@]}"; do
ENV_VARIABLE_VALUE=$(grep -v '^#' /home/user/skynet-webportal/.env | grep ${ENV_VARIABLE} || true)
if test -z "${ENV_VARIABLE_VALUE}"; then
if $ENV_VARIABLE ~= "SERVER_DOMAIN"; then
echo "Environment variable ${ENV_VARIABLE} is not set" && exit 1
fi
fi
export ${ENV_VARIABLE_VALUE}
done
# create bucket skynet-backup-[portaldomain] (portal domain is stripped of all non alnum characters)
# ie. siasky.net backup results in skynet-backup-siaskynet basket name
BUCKET_NAME=$(echo skynet-backup-${PORTAL_DOMAIN} | tr -cd '[[:alnum:]]')
# create server prefix
if test -z "${SERVER_DOMAIN}"; then
SERVER_PREFIX=$(echo ${SERVER_UID} | tr -cd '[[:alnum:]]-') # if domain name is empty use just uid
else
SERVER_PREFIX=$(echo ${SERVER_UID}-${SERVER_DOMAIN} | tr -cd '[[:alnum:]]-') # use both uid and server domain if available
fi
aws s3api create-bucket --acl private --bucket ${BUCKET_NAME}
# sync all nginx logs
aws s3 sync --no-progress /home/user/skynet-webportal/docker/data/nginx/logs s3://${BUCKET_NAME}/${SERVER_PREFIX}/docker/data/nginx/logs
# generate and sync skylinks
docker exec sia siac skynet ls --recursive --alert-suppress > /home/user/skynet-webportal/logs/skylinks-$(date +"%Y-%m-%d").log
aws s3 cp --no-progress /home/user/skynet-webportal/logs/skylinks.log s3://${BUCKET_NAME}/${SERVER_PREFIX}/logs/skylinks.log
echo "`date +"%Y-%m-%d %H:%M"` Backup finished successfully"

1
setup-scripts/support/crontab
View File

@ -2,6 +2,7 @@
0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8
0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1
30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env 30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env
3 44 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log
6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log 6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log
0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200
15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh 15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh