incremental logs and skylinks backup
This commit is contained in:
parent
9aa21088c5
commit
3e5c576902
|
@ -0,0 +1,43 @@
|
|||
#! /usr/bin/env bash
|
||||
|
||||
set -e # exit on first error
|
||||
|
||||
echo "`date +"%Y-%m-%d %H:%M"` Starting backup process"
|
||||
|
||||
# ensure aws cli is upgraded
|
||||
pip3 install --quiet --upgrade awscli
|
||||
|
||||
# import required environment variables from .env file
|
||||
ENV_VARIABLES=("AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY" "PORTAL_DOMAIN" "SERVER_DOMAIN" "SERVER_UID")
|
||||
|
||||
for ENV_VARIABLE in "${ENV_VARIABLES[@]}"; do
|
||||
ENV_VARIABLE_VALUE=$(grep -v '^#' /home/user/skynet-webportal/.env | grep ${ENV_VARIABLE} || true)
|
||||
if test -z "${ENV_VARIABLE_VALUE}"; then
|
||||
if $ENV_VARIABLE ~= "SERVER_DOMAIN"; then
|
||||
echo "Environment variable ${ENV_VARIABLE} is not set" && exit 1
|
||||
fi
|
||||
fi
|
||||
export ${ENV_VARIABLE_VALUE}
|
||||
done
|
||||
|
||||
# create bucket skynet-backup-[portaldomain] (portal domain is stripped of all non alnum characters)
|
||||
# ie. siasky.net backup results in skynet-backup-siaskynet basket name
|
||||
BUCKET_NAME=$(echo skynet-backup-${PORTAL_DOMAIN} | tr -cd '[[:alnum:]]')
|
||||
|
||||
# create server prefix
|
||||
if test -z "${SERVER_DOMAIN}"; then
|
||||
SERVER_PREFIX=$(echo ${SERVER_UID} | tr -cd '[[:alnum:]]-') # if domain name is empty use just uid
|
||||
else
|
||||
SERVER_PREFIX=$(echo ${SERVER_UID}-${SERVER_DOMAIN} | tr -cd '[[:alnum:]]-') # use both uid and server domain if available
|
||||
fi
|
||||
|
||||
aws s3api create-bucket --acl private --bucket ${BUCKET_NAME}
|
||||
|
||||
# sync all nginx logs
|
||||
aws s3 sync --no-progress /home/user/skynet-webportal/docker/data/nginx/logs s3://${BUCKET_NAME}/${SERVER_PREFIX}/docker/data/nginx/logs
|
||||
|
||||
# generate and sync skylinks
|
||||
docker exec sia siac skynet ls --recursive --alert-suppress > /home/user/skynet-webportal/logs/skylinks-$(date +"%Y-%m-%d").log
|
||||
aws s3 cp --no-progress /home/user/skynet-webportal/logs/skylinks.log s3://${BUCKET_NAME}/${SERVER_PREFIX}/logs/skylinks.log
|
||||
|
||||
echo "`date +"%Y-%m-%d %H:%M"` Backup finished successfully"
|
|
@ -2,6 +2,7 @@
|
|||
0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8
|
||||
0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1
|
||||
30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env
|
||||
3 44 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log
|
||||
6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log
|
||||
0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200
|
||||
15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh
|
||||
|
|
Reference in New Issue