diff --git a/setup-scripts/README.md b/setup-scripts/README.md index 6738dec3..9083dae7 100644 --- a/setup-scripts/README.md +++ b/setup-scripts/README.md @@ -104,8 +104,8 @@ At this point we have almost everything running, we just need to set up your wal > `docker exec caddy caddy reload --config /etc/caddy/Caddyfile` - Restarting nginx gracefully after making changes to nginx configs > `docker exec nginx openresty -s reload` -- Checking siad service logs (follow last 50 lines) - > `journalctl -f -n 50 --user-unit siad` +- Checking siad service logs (last hour) + > `docker logs --since 1h $(docker ps -q --filter "name=^sia$")` - Checking caddy logs (for example in case ssl certificate fails) > `docker logs caddy -f` - Checking nginx logs (nginx handles all communication to siad instances) diff --git a/setup-scripts/funds-checker.py b/setup-scripts/funds-checker.py index 4608352d..39c31dbb 100755 --- a/setup-scripts/funds-checker.py +++ b/setup-scripts/funds-checker.py @@ -5,15 +5,22 @@ health-checker runs simple health checks on a portal node using the siad API and dispatches messages to a Discord channel. """ -import discord, traceback +import discord, traceback, asyncio from bot_utils import setup, send_msg, siad, sc_precision bot_token = setup() client = discord.Client() + +async def exit_after(delay): + await asyncio.sleep(delay) + exit(0) + + @client.event async def on_ready(): await run_checks() + asyncio.create_task(exit_after(30)) await client.close() diff --git a/setup-scripts/log-checker.py b/setup-scripts/log-checker.py index e8d9b5e3..d4974048 100755 --- a/setup-scripts/log-checker.py +++ b/setup-scripts/log-checker.py @@ -1,82 +1,91 @@ #!/usr/bin/env python3 -import discord, sys, traceback, io -from bot_utils import setup, send_msg, sc_precision - +import discord, sys, traceback, io, os, asyncio +from bot_utils import setup, send_msg from datetime import datetime, timedelta from subprocess import Popen, PIPE """ -log-checker checks journal logs for siad. +log-checker checks the docker logs for siad. Arguments: 1. path to a .env file (default is none so env variables can already be preset) - 2. systemd service name (default: "siad") + 2. docker container name name (default: "sia") - 3. number of hours to look back in log (used as --since value in journalctl - command) (default: 1 hour) + 3. number of hours to look back in log (default: 1 hour) """ -DEFAULT_CHECK_INTERVAL = timedelta(hours=1) +# The default check interval in hours. +DEFAULT_CHECK_INTERVAL = 1 bot_token = setup() client = discord.Client() + +async def exit_after(delay): + await asyncio.sleep(delay) + exit(0) + + @client.event async def on_ready(): await run_checks() + asyncio.create_task(exit_after(30)) await client.close() async def run_checks(): print("Running Skynet portal log checks") try: - await check_journal() + await check_docker_logs() except: # catch all exceptions trace = traceback.format_exc() await send_msg(client, "```\n{}\n```".format(trace), force_notify=False) -# check_journal checks the journal -async def check_journal(): - print("\nChecking journal...") +# check_docker_logs checks the docker logs by filtering on the docker image name +async def check_docker_logs(): + print("\nChecking docker logs...") - # Get the systemd service name as an argument, or use "siad" as default. - service_name = "siad" + # Get the container name as an argument or use "sia" as default. + container_name = "sia" if len(sys.argv) > 2: - service_name = sys.argv[2] + container_name = sys.argv[2] - # Get the systemd service name as an argument, or use "siad" as default. - check_interval = DEFAULT_CHECK_INTERVAL + # Get the container id for siad. + stream = os.popen('docker ps -q --filter name=^{}$'.format(container_name)) + image_id = stream.read().strip() + + # Get the number of hours to look back in the logs or use 1 as default. + check_hours = DEFAULT_CHECK_INTERVAL if len(sys.argv) > 3: - check_interval = timedelta(hours=int(sys.argv[3])) + check_hours = int(sys.argv[3]) now = datetime.now() - time = now - check_interval - time_string = "{}-{}-{} {}:{}:{}".format(time.year, time.month, time.day, time.hour, time.minute, time.second) + time = now - timedelta(hours=check_hours) + time_string = "{}h".format(check_hours) - # Open the journal. - proc = Popen(["journalctl", "--user-unit", service_name, "--since", time_string], stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True) + # Read the logs. + proc = Popen(["docker", "logs", "--since", time_string, image_id], stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True) std_out, std_err = proc.communicate() if len(std_err) > 0: - await send_msg(client, "Error reading journalctl output: {}".format(std_err), force_notify=True) + await send_msg(client, "Error reading docker logs output: {}".format(std_err), force_notify=True) return # If there are any critical errors. upload the whole log file. if "Critical" in std_out or "panic" in std_out: - upload_name = "{}-{}-{}-{}-{}:{}:{}.log".format(service_name, time.year, time.month, time.day, time.hour, time.minute, time.second) + upload_name = "{}-{}-{}-{}-{}:{}:{}.log".format(container_name, time.year, time.month, time.day, time.hour, time.minute, time.second) await send_msg(client, "Critical error found in log!", file=discord.File(io.BytesIO(std_out.encode()), filename=upload_name), force_notify=True) return - # No critical errors, return a heartbeat type messagej + # No critical errors, return a heartbeat type message pretty_before = time.strftime("%I:%M%p") pretty_now = now.strftime("%I:%M%p") await send_msg(client, "No critical warnings in log from `{}` to `{}`".format(pretty_before, pretty_now)) - client.run(bot_token) diff --git a/setup-scripts/setup-docker-services.sh b/setup-scripts/setup-docker-services.sh index 57d1cf0c..a7903c80 100755 --- a/setup-scripts/setup-docker-services.sh +++ b/setup-scripts/setup-docker-services.sh @@ -17,17 +17,20 @@ sudo curl -L "https://github.com/docker/compose/releases/download/1.25.5/docker- sudo chmod +x /usr/local/bin/docker-compose docker-compose --version # sanity check -# Create dummy .env file for docker-compose usage with veriables +# Create dummy .env file for docker-compose usage with variables # * DOMAIN_NAME - the domain name your server is using ie. example.com # * EMAIL_ADDRESS - this is the administrator contact email you need to supply for communication regarding SSL certification # * HSD_API_KEY - this is auto generated secure key for your handshake service integration # * CLOUDFLARE_AUTH_TOKEN` - (optional) if using cloudflare as dns loadbalancer (need to change it in Caddyfile too) # * AWS_ACCESS_KEY_ID - (optional) if using route53 as a dns loadbalancer # * AWS_SECRET_ACCESS_KEY - (optional) if using route53 as a dns loadbalancer +# * API_PORT - (optional) the port on which siad is listening, defaults to 9980 +# * PORTAL_NAME - the name of the portal, required by the discord bot +# * DISCORD_BOT_TOKEN - required by the discord bot if ! [ -f /home/user/skynet-webportal/.env ]; then HSD_API_KEY=$(openssl rand -base64 32) # generate safe random key for handshake - printf "DOMAIN_NAME=example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\n" > /home/user/skynet-webportal/.env + printf "DOMAIN_NAME=example.com\nEMAIL_ADDRESS=email@example.com\nSIA_WALLET_PASSWORD=\nHSD_API_KEY=${HSD_API_KEY}\nCLOUDFLARE_AUTH_TOKEN=\nAWS_ACCESS_KEY_ID=\nAWS_SECRET_ACCESS_KEY=\nPORTAL_NAME=\nDISCORD_BOT_TOKEN=\n" > /home/user/skynet-webportal/.env fi # Start docker container with nginx and client -docker-compose -f docker-compose.yml up --build -d \ No newline at end of file +docker-compose -f docker-compose.yml up --build -d diff --git a/setup-scripts/setup-health-check-scripts.sh b/setup-scripts/setup-health-check-scripts.sh index 81ec5376..314a2170 100755 --- a/setup-scripts/setup-health-check-scripts.sh +++ b/setup-scripts/setup-health-check-scripts.sh @@ -8,8 +8,8 @@ sudo apt-get -y install python3-pip pip3 install discord.py pip3 install python-dotenv -fundsCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/.sia/sia.env" -logsCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/.sia/sia.env siad 8" +fundsCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/funds-checker.py /home/user/skynet-webportal/.env" +logsCheck="0 0,8,16 * * * /home/user/skynet-webportal/setup-scripts/log-checker.py /home/user/skynet-webportal/.env sia 8" (crontab -u user -l; echo "$fundsCheck" ) | crontab -u user - (crontab -u user -l; echo "$logsCheck" ) | crontab -u user -