From 4b707fac26345db25c3acdc8b166dc0af05a98fa Mon Sep 17 00:00:00 2001 From: Marius Date: Sun, 15 Jan 2023 21:35:08 +0100 Subject: [PATCH] docker: Add tools for preproducing load issues --- Dockerfile | 6 + docker/load-tests/1_run-test.sh | 19 ++++ docker/load-tests/2_plot_resource_usage.py | 123 +++++++++++++++++++++ docker/load-tests/README.md | 3 + docker/load-tests/docker-compose.yml | 68 ++++++++++++ docker/load-tests/resource-usage-log.txt | 69 ++++++++++++ docker/load-tests/uploader/Dockerfile | 9 ++ docker/load-tests/uploader/upload.sh | 41 +++++++ examples/hooks/plugin/hook_handler.go | 42 +++---- 9 files changed, 359 insertions(+), 21 deletions(-) create mode 100755 docker/load-tests/1_run-test.sh create mode 100755 docker/load-tests/2_plot_resource_usage.py create mode 100644 docker/load-tests/README.md create mode 100644 docker/load-tests/docker-compose.yml create mode 100644 docker/load-tests/resource-usage-log.txt create mode 100644 docker/load-tests/uploader/Dockerfile create mode 100755 docker/load-tests/uploader/upload.sh diff --git a/Dockerfile b/Dockerfile index 66f1192..6e7a928 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ RUN set -xe \ COPY cmd/ ./cmd/ COPY internal/ ./internal/ COPY pkg/ ./pkg/ +COPY examples/ ./examples/ # Get the version name and git commit as a build argument ARG GIT_VERSION @@ -24,6 +25,10 @@ RUN set -xe \ -ldflags="-X github.com/tus/tusd/cmd/tusd/cli.VersionName=${GIT_VERSION} -X github.com/tus/tusd/cmd/tusd/cli.GitCommit=${GIT_COMMIT} -X 'github.com/tus/tusd/cmd/tusd/cli.BuildDate=$(date --utc)'" \ -o /go/bin/tusd ./cmd/tusd/main.go +RUN set -xe \ + && GOOS=linux GOARCH=amd64 go build \ + -o /go/bin/hooks_handler ./examples/hooks/plugin/hook_handler.go + # start a new stage that copies in the binary built in the previous stage FROM alpine:3.16.2 WORKDIR /srv/tusd-data @@ -39,6 +44,7 @@ RUN apk add --no-cache ca-certificates jq bash \ && chmod +x /usr/local/share/docker-entrypoint.sh /usr/local/share/load-env.sh COPY --from=builder /go/bin/tusd /usr/local/bin/tusd +COPY --from=builder /go/bin/hooks_handler /usr/local/bin/hooks_handler EXPOSE 1080 USER tusd diff --git a/docker/load-tests/1_run-test.sh b/docker/load-tests/1_run-test.sh new file mode 100755 index 0000000..ea54519 --- /dev/null +++ b/docker/load-tests/1_run-test.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# Setup traps, so our background job of monitoring the containers +# exits, if the script is complete. +trap "exit" INT TERM ERR +trap "kill 0" EXIT + +# 1) Ensure that the containers are up-to-date +docker compose build + +# 2) Start the container monitoring +docker stats --format "{{ json . }}" > resource-usage-log.txt & + +# 3) Run the actual tests +docker compose up --abort-on-container-exit diff --git a/docker/load-tests/2_plot_resource_usage.py b/docker/load-tests/2_plot_resource_usage.py new file mode 100755 index 0000000..b71df67 --- /dev/null +++ b/docker/load-tests/2_plot_resource_usage.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +import json +import re +import matplotlib.pyplot as plt + +snapshots = [] + +with open("./resource-usage-log.txt") as file: + current_snapshot = None + for line in file: + # The lines might contain the reset characters before the actual JSON. + # This means that the entire resources for the current time have been + # written out, so we add the latest snapshot to our list and continue + # reading the next entries. + first_backet = line.find("{") + if first_backet == -1: + continue + + if first_backet != 0: + if current_snapshot is not None: + snapshots.append(current_snapshot) + + current_snapshot = [] + line = line[first_backet:] + + current_snapshot.append(json.loads(line)) + +def parse_percentage(string): + return float(string.strip('%')) + +units = {"B": 1, "kB": 10**3, "MB": 10**6, "GB": 10**9, "TB": 10**12, + "KiB": 2**10, "MiB": 2**20, "GiB": 2**30, "TiB": 2**40} + +def parse_byte_size(size): + number, unit = re.findall(r'([0-9\.]+)([A-Za-z]+)', size)[0] + return int(float(number)*units[unit]) + +def parse_two_bytes(string): + str1, str2 = string.split("/") + return parse_byte_size(str1), parse_byte_size(str2) + +s3_cpu = [] +s3_mem = [] +tusd_cpu = [] +tusd_mem = [] +tusd_net = [] +uploader_cpu = [] +uploader_mem = [] +uploader_net = [] +timestamp = [] + +for (i, snapshot) in enumerate(snapshots): + a_s3_cpu = None + a_s3_mem = None + a_tusd_cpu = None + a_tusd_mem = None + a_tusd_net = None + a_uploader_cpu = None + a_uploader_mem = None + a_uploader_net = None + + for entry in snapshot: + if entry["Name"] == "load-tests-tusd-1": + a_tusd_cpu = parse_percentage(entry["CPUPerc"]) + a_tusd_mem = parse_two_bytes(entry["MemUsage"])[0] + a_tusd_net = parse_two_bytes(entry["NetIO"])[0] + elif entry["Name"] == "load-tests-s3-1": + a_s3_cpu = parse_percentage(entry["CPUPerc"]) + a_s3_mem = parse_two_bytes(entry["MemUsage"])[0] + elif entry["Name"] == "load-tests-uploader-1": + a_uploader_cpu = parse_percentage(entry["CPUPerc"]) + a_uploader_mem = parse_two_bytes(entry["MemUsage"])[0] + a_uploader_net = parse_two_bytes(entry["NetIO"])[1] + + s3_cpu.append(a_s3_cpu) + s3_mem.append(a_s3_mem) + tusd_cpu.append(a_tusd_cpu) + tusd_mem.append(a_tusd_mem) + tusd_net.append(a_tusd_net) + uploader_cpu.append(a_uploader_cpu) + uploader_mem.append(a_uploader_mem) + uploader_net.append(a_uploader_net) + + # The docker stats command is hard coded to output stats every 500ms: + # https://github.com/docker/cli/blob/81c68913e4c2cb058b5a9fd5972e2989d9915b2c/cli/command/container/stats.go#L223 + timestamp.append(0.5 * i) + +fig, axs = plt.subplots(3, 3, sharex=True, sharey='row') +axs[0, 0].plot(timestamp, tusd_cpu) +axs[0, 0].set_title('tusd CPU percentage') +axs[0, 0].set(ylabel='CPU perc', xlabel='time') +axs[0, 1].plot(timestamp, s3_cpu) +axs[0, 1].set_title('s3 CPU percentage') +axs[0, 1].set(ylabel='CPU perc', xlabel='time') +axs[0, 2].plot(timestamp, uploader_cpu) +axs[0, 2].set_title('uploader CPU percentage') +axs[0, 2].set(ylabel='CPU perc', xlabel='time') + +axs[1, 0].plot(timestamp, tusd_mem) +axs[1, 0].set_title('tusd memory usage') +axs[1, 0].set(ylabel='mem perc', xlabel='time') +axs[1, 1].plot(timestamp, s3_mem) +axs[1, 1].set_title('s3 memory usage') +axs[1, 1].set(ylabel='mem perc', xlabel='time') +axs[1, 2].plot(timestamp, uploader_mem) +axs[1, 2].set_title('uploader memory usage') +axs[1, 2].set(ylabel='mem perc', xlabel='time') + +axs[2, 0].plot(timestamp, tusd_net) +axs[2, 0].set_title('tusd network input') +axs[2, 0].set(ylabel='total volume', xlabel='time') +axs[2, 1].axis('off') +axs[2, 2].plot(timestamp, uploader_net) +axs[2, 2].set_title('uploader network output') +axs[2, 2].set(ylabel='total volume', xlabel='time') + + +# Hide x labels and tick labels for top plots and y ticks for right plots. +for ax in axs.flat: + ax.label_outer() + +plt.show() diff --git a/docker/load-tests/README.md b/docker/load-tests/README.md new file mode 100644 index 0000000..586556e --- /dev/null +++ b/docker/load-tests/README.md @@ -0,0 +1,3 @@ +## Load issues with tusd & S3 + +This diff --git a/docker/load-tests/docker-compose.yml b/docker/load-tests/docker-compose.yml new file mode 100644 index 0000000..5c4d574 --- /dev/null +++ b/docker/load-tests/docker-compose.yml @@ -0,0 +1,68 @@ +version: "3.9" + +# TODO: +# - Add service for monitoring tusd +# - Add hooks +# - Use similar configuration as api2 + +services: + s3: + image: minio/minio + ports: + - "9000:9000" + - "9001:9001" + # Note: Data directory is not persistent on purpose + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + # deploy: + # resources: + # limits: + # cpus: "2" + + createbucket: + image: minio/mc + entrypoint: > + /bin/sh -c " + /usr/bin/mc config host add s3 http://s3:9000 minioadmin minioadmin; + /usr/bin/mc mb --ignore-existing s3/tusdtest.transloadit.com; + sleep infinity; + " + depends_on: + - s3 + + tusd: + build: ../../ + ports: + - "1080:1080" + # entrypoint: file /srv/tusdhook/hook_handler + entrypoint: tusd -s3-bucket "tusdtest.transloadit.com" -s3-endpoint "http://s3:9000" -hooks-plugin=/usr/local/bin/hooks_handler -hooks-enabled-events=pre-create,post-create,post-receive,post-finish -progress-hooks-interval=3000 -max-size=128849018880 -timeout=60000 -s3-disable-content-hashes=true -s3-disable-ssl=true -s3-concurrent-part-uploads=48 -s3-max-buffered-parts=1 + environment: + AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + depends_on: + - s3 + - createbucket + volumes: + - ../../examples/hooks/plugin:/srv/tusdhook + deploy: + resources: + limits: + cpus: "2" + + uploader: + build: ./uploader + # 10 MiB: 10485760 + # 100 MiB: 104857600 + # 1000 MiB: 1048576000 + command: 10485760 50 /dev/shm + tmpfs: + - /dev/shm + depends_on: + - tusd + # deploy: + # resources: + # limits: + # cpus: "1" diff --git a/docker/load-tests/resource-usage-log.txt b/docker/load-tests/resource-usage-log.txt new file mode 100644 index 0000000..9687702 --- /dev/null +++ b/docker/load-tests/resource-usage-log.txt @@ -0,0 +1,69 @@ +{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"0.05%","MemUsage":"8.633MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"220B / 0B","PIDs":"9"} +{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"} +{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"0.05%","MemUsage":"8.633MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"220B / 0B","PIDs":"9"} +{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"} +{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"045ce1160eb0","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"} +{"BlockIO":"115kB / 4.1kB","CPUPerc":"34.90%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"0.57%","MemUsage":"90.34MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"10.2kB / 3.43kB","PIDs":"13"} +{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.01%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"2.41kB / 1.76kB","PIDs":"2"} +{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.08%","MemUsage":"12.74MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"90B / 0B","PIDs":"19"} +{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"e4e5ae23b118","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"} +{"BlockIO":"115kB / 4.1kB","CPUPerc":"34.90%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"0.57%","MemUsage":"90.34MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"10.2kB / 3.43kB","PIDs":"13"} +{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.01%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"2.41kB / 1.76kB","PIDs":"2"} +{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.08%","MemUsage":"12.74MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"90B / 0B","PIDs":"19"} +{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"e4e5ae23b118","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"} +{"BlockIO":"225kB / 209kB","CPUPerc":"97.82%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"1.34%","MemUsage":"212.4MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"212kB / 128kB","PIDs":"15"} +{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"3.76kB / 1.76kB","PIDs":"2"} +{"BlockIO":"0B / 0B","CPUPerc":"17.92%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.16%","MemUsage":"25.36MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"180kB / 237kB","PIDs":"19"} +{"BlockIO":"0B / 0B","CPUPerc":"0.06%","Container":"e4e5ae23b118","ID":"e4e5ae23b118","MemPerc":"3.75%","MemUsage":"593.7MiB / 15.47GiB","Name":"load-tests-uploader-1","NetIO":"36kB / 54.2kB","PIDs":"51"} +{"BlockIO":"225kB / 209kB","CPUPerc":"97.82%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"1.34%","MemUsage":"212.4MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"212kB / 128kB","PIDs":"15"} +{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"3.76kB / 1.76kB","PIDs":"2"} +{"BlockIO":"0B / 0B","CPUPerc":"17.92%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.16%","MemUsage":"25.36MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"180kB / 237kB","PIDs":"19"} +{"BlockIO":"0B / 0B","CPUPerc":"0.06%","Container":"e4e5ae23b118","ID":"e4e5ae23b118","MemPerc":"3.75%","MemUsage":"593.7MiB / 15.47GiB","Name":"load-tests-uploader-1","NetIO":"36kB / 54.2kB","PIDs":"51"} +{"BlockIO":"373kB / 189MB","CPUPerc":"637.26%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.59%","MemUsage":"568.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"254MB / 303kB","PIDs":"28"} +{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.26kB / 1.76kB","PIDs":"2"} +{"BlockIO":"0B / 0B","CPUPerc":"55.52%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.53%","MemUsage":"84.67MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"349MB / 258MB","PIDs":"27"} +{"BlockIO":"0B / 0B","CPUPerc":"17.23%","Container":"e4e5ae23b118","ID":"e4e5ae23b118","MemPerc":"3.59%","MemUsage":"569.1MiB / 15.47GiB","Name":"load-tests-uploader-1","NetIO":"334kB / 349MB","PIDs":"48"} +{"BlockIO":"373kB / 189MB","CPUPerc":"637.26%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.59%","MemUsage":"568.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"254MB / 303kB","PIDs":"28"} +{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.26kB / 1.76kB","PIDs":"2"} +{"BlockIO":"729kB / 526MB","CPUPerc":"172.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.47%","MemUsage":"550.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.46kB / 1.76kB","PIDs":"2"} +{"BlockIO":"729kB / 526MB","CPUPerc":"172.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.47%","MemUsage":"550.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.46kB / 1.76kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.27%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.47%","MemUsage":"550.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.85kB / 1.76kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.27%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.47%","MemUsage":"550.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.85kB / 1.76kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.55%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.55%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.02%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.02%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.10%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.10%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.02%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.02%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.03%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} +{"BlockIO":"733kB / 526MB","CPUPerc":"0.03%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"} diff --git a/docker/load-tests/uploader/Dockerfile b/docker/load-tests/uploader/Dockerfile new file mode 100644 index 0000000..f153c94 --- /dev/null +++ b/docker/load-tests/uploader/Dockerfile @@ -0,0 +1,9 @@ +FROM alpine:3.16.2 + +RUN apk add --no-cache curl bash openssl + +COPY ./upload.sh /usr/local/share/upload.sh +RUN chmod +x /usr/local/share/upload.sh + +ENTRYPOINT [ "/usr/local/share/upload.sh" ] +CMD [] diff --git a/docker/load-tests/uploader/upload.sh b/docker/load-tests/uploader/upload.sh new file mode 100755 index 0000000..1d78964 --- /dev/null +++ b/docker/load-tests/uploader/upload.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e +set -o pipefail + +echo $@ + +if [ -z "$2" ]; then + echo "USAGE: upload.sh SIZE NUMBER [TEMP DIR]" + exit 1 +fi + +size="$1" +number="$2" +directory="${3:-/tmp}" +file="${directory}/${size}.bin" + +openssl rand -out "$file" "$size" + +# Get upload size in bytes +upload_size=$(stat -c "%s" "$file") +echo "Generated file with size: ${upload_size} bytes." + +# Create uploads +for i in $(seq 1 $number); do + # Note: I wanted to use the new feature for extracting header values + # (https://daniel.haxx.se/blog/2022/03/24/easier-header-picking-with-curl/) + # but this is not yet available on the current curl version in Alpine Linux. + upload_urls[${i}]="$(curl -X POST -H 'Tus-Resumable: 1.0.0' -H "Upload-Length: ${upload_size}" --fail --silent -i http://tusd:1080/files/ | grep -i ^Location: | cut -d: -f2- | sed 's/^ *\(.*\).*/\1/' | tr -d '\r')" +done + +# Perform the uploads in parallel +for i in $(seq 1 $number); do + curl -X PATCH -H 'Tus-Resumable: 1.0.0' -H 'Upload-Offset: 0' -H 'Content-Type: application/offset+octet-stream' --data-binary "@${file}" "${upload_urls[${i}]}" & + pids[${i}]=$! +done + +# Wait for all uploads to complete +for pid in ${pids[*]}; do + wait $pid +done diff --git a/examples/hooks/plugin/hook_handler.go b/examples/hooks/plugin/hook_handler.go index 38a3a6b..7f2e5bd 100644 --- a/examples/hooks/plugin/hook_handler.go +++ b/examples/hooks/plugin/hook_handler.go @@ -27,32 +27,32 @@ func (g *MyHookHandler) Setup() error { func (g *MyHookHandler) InvokeHook(req hooks.HookRequest) (res hooks.HookResponse, err error) { log.Println("MyHookHandler.InvokeHook is invoked") - // Prepare hook response structure - res.HTTPResponse.Headers = make(map[string]string) + // // Prepare hook response structure + // res.HTTPResponse.Headers = make(map[string]string) - // Example: Use the pre-create hook to check if a filename has been supplied - // using metadata. If not, the upload is rejected with a custom HTTP response. + // // Example: Use the pre-create hook to check if a filename has been supplied + // // using metadata. If not, the upload is rejected with a custom HTTP response. - if req.Type == hooks.HookPreCreate { - if _, ok := req.Event.Upload.MetaData["filename"]; !ok { - res.RejectUpload = true - res.HTTPResponse.StatusCode = 400 - res.HTTPResponse.Body = "no filename provided" - res.HTTPResponse.Headers["X-Some-Header"] = "yes" - } - } + // if req.Type == hooks.HookPreCreate { + // if _, ok := req.Event.Upload.MetaData["filename"]; !ok { + // res.RejectUpload = true + // res.HTTPResponse.StatusCode = 400 + // res.HTTPResponse.Body = "no filename provided" + // res.HTTPResponse.Headers["X-Some-Header"] = "yes" + // } + // } - // Example: Use the post-finish hook to print information about a completed upload, - // including its storage location. - if req.Type == hooks.HookPreFinish { - id := req.Event.Upload.ID - size := req.Event.Upload.Size - storage := req.Event.Upload.Storage + // // Example: Use the post-finish hook to print information about a completed upload, + // // including its storage location. + // if req.Type == hooks.HookPreFinish { + // id := req.Event.Upload.ID + // size := req.Event.Upload.Size + // storage := req.Event.Upload.Storage - log.Printf("Upload %s (%d bytes) is finished. Find the file at:\n", id, size) - log.Println(storage) + // log.Printf("Upload %s (%d bytes) is finished. Find the file at:\n", id, size) + // log.Println(storage) - } + // } // Return the hook response to tusd. return res, nil