From 0eb8ab0d7c15dc0367cc93cb370e773aca66045e Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Mon, 24 Aug 2020 12:33:03 +0300 Subject: [PATCH 1/3] Use a clearer log message and add some debug output. --- setup-scripts/log-checker.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup-scripts/log-checker.py b/setup-scripts/log-checker.py index d4974048..2de6cc8a 100755 --- a/setup-scripts/log-checker.py +++ b/setup-scripts/log-checker.py @@ -57,7 +57,9 @@ async def check_docker_logs(): container_name = sys.argv[2] # Get the container id for siad. - stream = os.popen('docker ps -q --filter name=^{}$'.format(container_name)) + cmd = 'docker ps -q --filter name=^{}$'.format(container_name) + print("[DEBUG] will run `{}`".format(cmd)) + stream = os.popen(cmd) image_id = stream.read().strip() # Get the number of hours to look back in the logs or use 1 as default. @@ -70,11 +72,12 @@ async def check_docker_logs(): time_string = "{}h".format(check_hours) # Read the logs. + print("[DEBUG] Will run `docker logs --since {} {}`".format(time_string, image_id)) proc = Popen(["docker", "logs", "--since", time_string, image_id], stdin=PIPE, stdout=PIPE, stderr=PIPE, text=True) std_out, std_err = proc.communicate() if len(std_err) > 0: - await send_msg(client, "Error reading docker logs output: {}".format(std_err), force_notify=True) + await send_msg(client, "Error(s) found in log: {}".format(std_err), force_notify=True) return # If there are any critical errors. upload the whole log file. From 3b0c59751660e88839fe7cc60eca42a9921f8cbc Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 26 Aug 2020 11:13:16 +0300 Subject: [PATCH 2/3] Limit the errors sent in a discord message to under 2000 characters (including prefix message). --- setup-scripts/log-checker.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup-scripts/log-checker.py b/setup-scripts/log-checker.py index 2de6cc8a..971ec3e8 100755 --- a/setup-scripts/log-checker.py +++ b/setup-scripts/log-checker.py @@ -77,7 +77,13 @@ async def check_docker_logs(): std_out, std_err = proc.communicate() if len(std_err) > 0: - await send_msg(client, "Error(s) found in log: {}".format(std_err), force_notify=True) + # Send at most 1900 characters of logs, rounded down to the nearest new line. + # This is a limitation in the size of Discord messages - they can be at most + # 2000 characters long (and we send some extra characters before the error log). + if len(std_err) > 1900: + pos = std_err.find("\n", -1900) + std_err = std_err[pos+1:] + await send_msg(client, "Error(s) found in log:\n{}".format(std_err), force_notify=True) return # If there are any critical errors. upload the whole log file. From 2126115df4cc8e3a51e6ecd69c46b1646747e985 Mon Sep 17 00:00:00 2001 From: Ivaylo Novakov Date: Wed, 26 Aug 2020 11:32:58 +0300 Subject: [PATCH 3/3] Send the last 1MB of error log as a file. --- setup-scripts/log-checker.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/setup-scripts/log-checker.py b/setup-scripts/log-checker.py index 971ec3e8..bfb53895 100755 --- a/setup-scripts/log-checker.py +++ b/setup-scripts/log-checker.py @@ -77,13 +77,20 @@ async def check_docker_logs(): std_out, std_err = proc.communicate() if len(std_err) > 0: + # Trim the error log to under 1MB. + one_mb = 1024*1024 + if len(std_err) > one_mb: + pos = std_err.find("\n", -one_mb) + std_err = std_err[pos+1:] + upload_name = "{}-{}-{}-{}-{}:{}:{}_err.log".format(container_name, time.year, time.month, time.day, time.hour, time.minute, time.second) + await send_msg(client, "Error(s) found in log!", file=discord.File(io.BytesIO(std_err.encode()), filename=upload_name), force_notify=True) # Send at most 1900 characters of logs, rounded down to the nearest new line. # This is a limitation in the size of Discord messages - they can be at most # 2000 characters long (and we send some extra characters before the error log). if len(std_err) > 1900: pos = std_err.find("\n", -1900) std_err = std_err[pos+1:] - await send_msg(client, "Error(s) found in log:\n{}".format(std_err), force_notify=True) + await send_msg(client, "Error(s) preview:\n{}".format(std_err), force_notify=True) return # If there are any critical errors. upload the whole log file.