LumeWeb
/
skynet-webportal
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1759 from SkynetLabs/fix-skylinks-backup 

ensure path exists and fix filename
This commit is contained in:
Karol Wypchło 2022-02-22 17:04:55 +01:00 committed by GitHub
parent 82bb85666c 38e2367902
commit 485a8e9b45
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
scripts/backup-aws-s3.sh
View File

@ -20,24 +20,29 @@ for ENV_VARIABLE in "${ENV_VARIABLES[@]}"; do
export ${ENV_VARIABLE_VALUE} export ${ENV_VARIABLE_VALUE}
done done
# create bucket skynet-backup-[portaldomain] (portal domain is stripped of all non alnum characters) # create bucket skynet-backup-[portaldomain] (replace dots with dashes and strip anything other than alnum)
# ie. siasky.net backup results in skynet-backup-siaskynet basket name # ie. siasky.net backup results in skynet-backup-siasky-net basket name
BUCKET_NAME=$(echo skynet-backup-${PORTAL_DOMAIN} | tr -cd '[[:alnum:]]-') BUCKET_NAME=$(echo skynet-backup-${PORTAL_DOMAIN} | tr '.' '-' | tr -cd '[[:alnum:]]-')
# create server prefix # create server prefix
if test -z "${SERVER_DOMAIN}"; then if test -z "${SERVER_DOMAIN}"; then
SERVER_PREFIX=$(echo ${SERVER_UID} | tr -cd '[[:alnum:]]-') # if domain name is empty use just uid # if domain name is empty use just uid (replace dots with dashes and strip anything other than alnum)
SERVER_PREFIX=$(echo ${SERVER_UID} | tr '.' '-' | tr -cd '[[:alnum:]]-')
else else
SERVER_PREFIX=$(echo ${SERVER_UID}-${SERVER_DOMAIN} | tr -cd '[[:alnum:]]-') # use both uid and server domain if available # use both uid and server domain if available (replace dots with dashes and strip anything other than alnum)
SERVER_PREFIX=$(echo ${SERVER_UID}-${SERVER_DOMAIN} | tr '.' '-' | tr -cd '[[:alnum:]]-')
fi fi
aws s3api create-bucket --acl private --bucket ${BUCKET_NAME} aws s3api create-bucket --acl private --bucket ${BUCKET_NAME}
# sync all nginx logs # sync all nginx logs
mkdir -p /home/user/skynet-webportal/docker/data/nginx/logs # ensure path exists
aws s3 sync --no-progress /home/user/skynet-webportal/docker/data/nginx/logs s3://${BUCKET_NAME}/${SERVER_PREFIX}/docker/data/nginx/logs aws s3 sync --no-progress /home/user/skynet-webportal/docker/data/nginx/logs s3://${BUCKET_NAME}/${SERVER_PREFIX}/docker/data/nginx/logs
# generate and sync skylinks # generate and sync skylinks dump
docker exec sia siac skynet ls --recursive --alert-suppress > /home/user/skynet-webportal/logs/skylinks-$(date +"%Y-%m-%d").log SKYLINKS_PATH=logs/skylinks/$(date +"%Y-%m-%d").log
aws s3 cp --no-progress /home/user/skynet-webportal/logs/skylinks.log s3://${BUCKET_NAME}/${SERVER_PREFIX}/logs/skylinks.log mkdir -p /home/user/skynet-webportal/logs/skylinks # ensure path exists
docker exec sia siac skynet ls --recursive --alert-suppress > /home/user/skynet-webportal/${SKYLINKS_PATH}
aws s3 cp --no-progress /home/user/skynet-webportal/${SKYLINKS_PATH} s3://${BUCKET_NAME}/${SERVER_PREFIX}/${SKYLINKS_PATH}
echo "`date +"%Y-%m-%d %H:%M"` Backup finished successfully" echo "`date +"%Y-%m-%d %H:%M"` Backup finished successfully"

2
setup-scripts/support/crontab
View File

@ -2,7 +2,7 @@
0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8 0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8
0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1 0 * * * * /home/user/skynet-webportal/setup-scripts/health-checker.py /home/user/skynet-webportal/.env sia 1
30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env 30 */4 * * * /home/user/skynet-webportal/setup-scripts/blocklist-airtable.py /home/user/skynet-webportal/.env
44 */8 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log 44 5 * * * /home/user/skynet-webportal/scripts/backup-aws-s3.sh 1>>/home/user/skynet-webportal/logs/backup-aws-s3.log 2>>/home/user/skynet-webportal/logs/backup-aws-s3.log
6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log 6 13 * * * /home/user/skynet-webportal/scripts/db_backup.sh 1>>/home/user/skynet-webportal/logs/db_backup.log 2>>/home/user/skynet-webportal/logs/db_backup.log
0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200 0 5 * * * /home/user/skynet-webportal/scripts/es_cleaner.py 1 http://localhost:9200
15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh 15 * * * * /home/user/skynet-webportal/scripts/nginx-prune.sh