docker: Add tools for preproducing load issues
This commit is contained in:
Marius
parent
621de70da4
commit
4b707fac26
|
|
@ -14,6 +14,7 @@ RUN set -xe \
|
||||||
COPY cmd/ ./cmd/
|
COPY cmd/ ./cmd/
|
||||||
COPY internal/ ./internal/
|
COPY internal/ ./internal/
|
||||||
COPY pkg/ ./pkg/
|
COPY pkg/ ./pkg/
|
||||||
|
COPY examples/ ./examples/
|
||||||
|
|
||||||
# Get the version name and git commit as a build argument
|
# Get the version name and git commit as a build argument
|
||||||
ARG GIT_VERSION
|
ARG GIT_VERSION
|
||||||
|
|
@ -24,6 +25,10 @@ RUN set -xe \
|
||||||
-ldflags="-X github.com/tus/tusd/cmd/tusd/cli.VersionName=${GIT_VERSION} -X github.com/tus/tusd/cmd/tusd/cli.GitCommit=${GIT_COMMIT} -X 'github.com/tus/tusd/cmd/tusd/cli.BuildDate=$(date --utc)'" \
|
-ldflags="-X github.com/tus/tusd/cmd/tusd/cli.VersionName=${GIT_VERSION} -X github.com/tus/tusd/cmd/tusd/cli.GitCommit=${GIT_COMMIT} -X 'github.com/tus/tusd/cmd/tusd/cli.BuildDate=$(date --utc)'" \
|
||||||
-o /go/bin/tusd ./cmd/tusd/main.go
|
-o /go/bin/tusd ./cmd/tusd/main.go
|
||||||
|
|
||||||
|
RUN set -xe \
|
||||||
|
&& GOOS=linux GOARCH=amd64 go build \
|
||||||
|
-o /go/bin/hooks_handler ./examples/hooks/plugin/hook_handler.go
|
||||||
|
|
||||||
# start a new stage that copies in the binary built in the previous stage
|
# start a new stage that copies in the binary built in the previous stage
|
||||||
FROM alpine:3.16.2
|
FROM alpine:3.16.2
|
||||||
WORKDIR /srv/tusd-data
|
WORKDIR /srv/tusd-data
|
||||||
|
|
@ -39,6 +44,7 @@ RUN apk add --no-cache ca-certificates jq bash \
|
||||||
&& chmod +x /usr/local/share/docker-entrypoint.sh /usr/local/share/load-env.sh
|
&& chmod +x /usr/local/share/docker-entrypoint.sh /usr/local/share/load-env.sh
|
||||||
|
|
||||||
COPY --from=builder /go/bin/tusd /usr/local/bin/tusd
|
COPY --from=builder /go/bin/tusd /usr/local/bin/tusd
|
||||||
|
COPY --from=builder /go/bin/hooks_handler /usr/local/bin/hooks_handler
|
||||||
|
|
||||||
EXPOSE 1080
|
EXPOSE 1080
|
||||||
USER tusd
|
USER tusd
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -o errexit
|
||||||
|
set -o nounset
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
|
# Setup traps, so our background job of monitoring the containers
|
||||||
|
# exits, if the script is complete.
|
||||||
|
trap "exit" INT TERM ERR
|
||||||
|
trap "kill 0" EXIT
|
||||||
|
|
||||||
|
# 1) Ensure that the containers are up-to-date
|
||||||
|
docker compose build
|
||||||
|
|
||||||
|
# 2) Start the container monitoring
|
||||||
|
docker stats --format "{{ json . }}" > resource-usage-log.txt &
|
||||||
|
|
||||||
|
# 3) Run the actual tests
|
||||||
|
docker compose up --abort-on-container-exit
|
||||||
|
|
@ -0,0 +1,123 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
snapshots = []
|
||||||
|
|
||||||
|
with open("./resource-usage-log.txt") as file:
|
||||||
|
current_snapshot = None
|
||||||
|
for line in file:
|
||||||
|
# The lines might contain the reset characters before the actual JSON.
|
||||||
|
# This means that the entire resources for the current time have been
|
||||||
|
# written out, so we add the latest snapshot to our list and continue
|
||||||
|
# reading the next entries.
|
||||||
|
first_backet = line.find("{")
|
||||||
|
if first_backet == -1:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if first_backet != 0:
|
||||||
|
if current_snapshot is not None:
|
||||||
|
snapshots.append(current_snapshot)
|
||||||
|
|
||||||
|
current_snapshot = []
|
||||||
|
line = line[first_backet:]
|
||||||
|
|
||||||
|
current_snapshot.append(json.loads(line))
|
||||||
|
|
||||||
|
def parse_percentage(string):
|
||||||
|
return float(string.strip('%'))
|
||||||
|
|
||||||
|
units = {"B": 1, "kB": 10**3, "MB": 10**6, "GB": 10**9, "TB": 10**12,
|
||||||
|
"KiB": 2**10, "MiB": 2**20, "GiB": 2**30, "TiB": 2**40}
|
||||||
|
|
||||||
|
def parse_byte_size(size):
|
||||||
|
number, unit = re.findall(r'([0-9\.]+)([A-Za-z]+)', size)[0]
|
||||||
|
return int(float(number)*units[unit])
|
||||||
|
|
||||||
|
def parse_two_bytes(string):
|
||||||
|
str1, str2 = string.split("/")
|
||||||
|
return parse_byte_size(str1), parse_byte_size(str2)
|
||||||
|
|
||||||
|
s3_cpu = []
|
||||||
|
s3_mem = []
|
||||||
|
tusd_cpu = []
|
||||||
|
tusd_mem = []
|
||||||
|
tusd_net = []
|
||||||
|
uploader_cpu = []
|
||||||
|
uploader_mem = []
|
||||||
|
uploader_net = []
|
||||||
|
timestamp = []
|
||||||
|
|
||||||
|
for (i, snapshot) in enumerate(snapshots):
|
||||||
|
a_s3_cpu = None
|
||||||
|
a_s3_mem = None
|
||||||
|
a_tusd_cpu = None
|
||||||
|
a_tusd_mem = None
|
||||||
|
a_tusd_net = None
|
||||||
|
a_uploader_cpu = None
|
||||||
|
a_uploader_mem = None
|
||||||
|
a_uploader_net = None
|
||||||
|
|
||||||
|
for entry in snapshot:
|
||||||
|
if entry["Name"] == "load-tests-tusd-1":
|
||||||
|
a_tusd_cpu = parse_percentage(entry["CPUPerc"])
|
||||||
|
a_tusd_mem = parse_two_bytes(entry["MemUsage"])[0]
|
||||||
|
a_tusd_net = parse_two_bytes(entry["NetIO"])[0]
|
||||||
|
elif entry["Name"] == "load-tests-s3-1":
|
||||||
|
a_s3_cpu = parse_percentage(entry["CPUPerc"])
|
||||||
|
a_s3_mem = parse_two_bytes(entry["MemUsage"])[0]
|
||||||
|
elif entry["Name"] == "load-tests-uploader-1":
|
||||||
|
a_uploader_cpu = parse_percentage(entry["CPUPerc"])
|
||||||
|
a_uploader_mem = parse_two_bytes(entry["MemUsage"])[0]
|
||||||
|
a_uploader_net = parse_two_bytes(entry["NetIO"])[1]
|
||||||
|
|
||||||
|
s3_cpu.append(a_s3_cpu)
|
||||||
|
s3_mem.append(a_s3_mem)
|
||||||
|
tusd_cpu.append(a_tusd_cpu)
|
||||||
|
tusd_mem.append(a_tusd_mem)
|
||||||
|
tusd_net.append(a_tusd_net)
|
||||||
|
uploader_cpu.append(a_uploader_cpu)
|
||||||
|
uploader_mem.append(a_uploader_mem)
|
||||||
|
uploader_net.append(a_uploader_net)
|
||||||
|
|
||||||
|
# The docker stats command is hard coded to output stats every 500ms:
|
||||||
|
# https://github.com/docker/cli/blob/81c68913e4c2cb058b5a9fd5972e2989d9915b2c/cli/command/container/stats.go#L223
|
||||||
|
timestamp.append(0.5 * i)
|
||||||
|
|
||||||
|
fig, axs = plt.subplots(3, 3, sharex=True, sharey='row')
|
||||||
|
axs[0, 0].plot(timestamp, tusd_cpu)
|
||||||
|
axs[0, 0].set_title('tusd CPU percentage')
|
||||||
|
axs[0, 0].set(ylabel='CPU perc', xlabel='time')
|
||||||
|
axs[0, 1].plot(timestamp, s3_cpu)
|
||||||
|
axs[0, 1].set_title('s3 CPU percentage')
|
||||||
|
axs[0, 1].set(ylabel='CPU perc', xlabel='time')
|
||||||
|
axs[0, 2].plot(timestamp, uploader_cpu)
|
||||||
|
axs[0, 2].set_title('uploader CPU percentage')
|
||||||
|
axs[0, 2].set(ylabel='CPU perc', xlabel='time')
|
||||||
|
|
||||||
|
axs[1, 0].plot(timestamp, tusd_mem)
|
||||||
|
axs[1, 0].set_title('tusd memory usage')
|
||||||
|
axs[1, 0].set(ylabel='mem perc', xlabel='time')
|
||||||
|
axs[1, 1].plot(timestamp, s3_mem)
|
||||||
|
axs[1, 1].set_title('s3 memory usage')
|
||||||
|
axs[1, 1].set(ylabel='mem perc', xlabel='time')
|
||||||
|
axs[1, 2].plot(timestamp, uploader_mem)
|
||||||
|
axs[1, 2].set_title('uploader memory usage')
|
||||||
|
axs[1, 2].set(ylabel='mem perc', xlabel='time')
|
||||||
|
|
||||||
|
axs[2, 0].plot(timestamp, tusd_net)
|
||||||
|
axs[2, 0].set_title('tusd network input')
|
||||||
|
axs[2, 0].set(ylabel='total volume', xlabel='time')
|
||||||
|
axs[2, 1].axis('off')
|
||||||
|
axs[2, 2].plot(timestamp, uploader_net)
|
||||||
|
axs[2, 2].set_title('uploader network output')
|
||||||
|
axs[2, 2].set(ylabel='total volume', xlabel='time')
|
||||||
|
|
||||||
|
|
||||||
|
# Hide x labels and tick labels for top plots and y ticks for right plots.
|
||||||
|
for ax in axs.flat:
|
||||||
|
ax.label_outer()
|
||||||
|
|
||||||
|
plt.show()
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
## Load issues with tusd & S3
|
||||||
|
|
||||||
|
This
|
||||||
|
|
@ -0,0 +1,68 @@
|
||||||
|
version: "3.9"
|
||||||
|
|
||||||
|
# TODO:
|
||||||
|
# - Add service for monitoring tusd
|
||||||
|
# - Add hooks
|
||||||
|
# - Use similar configuration as api2
|
||||||
|
|
||||||
|
services:
|
||||||
|
s3:
|
||||||
|
image: minio/minio
|
||||||
|
ports:
|
||||||
|
- "9000:9000"
|
||||||
|
- "9001:9001"
|
||||||
|
# Note: Data directory is not persistent on purpose
|
||||||
|
command: server /data --console-address ":9001"
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minioadmin
|
||||||
|
MINIO_ROOT_PASSWORD: minioadmin
|
||||||
|
# deploy:
|
||||||
|
# resources:
|
||||||
|
# limits:
|
||||||
|
# cpus: "2"
|
||||||
|
|
||||||
|
createbucket:
|
||||||
|
image: minio/mc
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
/usr/bin/mc config host add s3 http://s3:9000 minioadmin minioadmin;
|
||||||
|
/usr/bin/mc mb --ignore-existing s3/tusdtest.transloadit.com;
|
||||||
|
sleep infinity;
|
||||||
|
"
|
||||||
|
depends_on:
|
||||||
|
- s3
|
||||||
|
|
||||||
|
tusd:
|
||||||
|
build: ../../
|
||||||
|
ports:
|
||||||
|
- "1080:1080"
|
||||||
|
# entrypoint: file /srv/tusdhook/hook_handler
|
||||||
|
entrypoint: tusd -s3-bucket "tusdtest.transloadit.com" -s3-endpoint "http://s3:9000" -hooks-plugin=/usr/local/bin/hooks_handler -hooks-enabled-events=pre-create,post-create,post-receive,post-finish -progress-hooks-interval=3000 -max-size=128849018880 -timeout=60000 -s3-disable-content-hashes=true -s3-disable-ssl=true -s3-concurrent-part-uploads=48 -s3-max-buffered-parts=1
|
||||||
|
environment:
|
||||||
|
AWS_REGION: us-east-1
|
||||||
|
AWS_ACCESS_KEY_ID: minioadmin
|
||||||
|
AWS_SECRET_ACCESS_KEY: minioadmin
|
||||||
|
depends_on:
|
||||||
|
- s3
|
||||||
|
- createbucket
|
||||||
|
volumes:
|
||||||
|
- ../../examples/hooks/plugin:/srv/tusdhook
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: "2"
|
||||||
|
|
||||||
|
uploader:
|
||||||
|
build: ./uploader
|
||||||
|
# 10 MiB: 10485760
|
||||||
|
# 100 MiB: 104857600
|
||||||
|
# 1000 MiB: 1048576000
|
||||||
|
command: 10485760 50 /dev/shm
|
||||||
|
tmpfs:
|
||||||
|
- /dev/shm
|
||||||
|
depends_on:
|
||||||
|
- tusd
|
||||||
|
# deploy:
|
||||||
|
# resources:
|
||||||
|
# limits:
|
||||||
|
# cpus: "1"
|
||||||
|
|
@ -0,0 +1,69 @@
|
||||||
|
[2J[H[2J[H[2J[H{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"0.05%","MemUsage":"8.633MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"220B / 0B","PIDs":"9"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"}
|
||||||
|
[2J[H{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"0.05%","MemUsage":"8.633MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"220B / 0B","PIDs":"9"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"045ce1160eb0","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"}
|
||||||
|
[2J[H{"BlockIO":"115kB / 4.1kB","CPUPerc":"34.90%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"0.57%","MemUsage":"90.34MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"10.2kB / 3.43kB","PIDs":"13"}
|
||||||
|
{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.01%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"2.41kB / 1.76kB","PIDs":"2"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.08%","MemUsage":"12.74MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"90B / 0B","PIDs":"19"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"e4e5ae23b118","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"}
|
||||||
|
[2J[H{"BlockIO":"115kB / 4.1kB","CPUPerc":"34.90%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"0.57%","MemUsage":"90.34MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"10.2kB / 3.43kB","PIDs":"13"}
|
||||||
|
{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.01%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"2.41kB / 1.76kB","PIDs":"2"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.08%","MemUsage":"12.74MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"90B / 0B","PIDs":"19"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"e4e5ae23b118","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"}
|
||||||
|
[2J[H{"BlockIO":"225kB / 209kB","CPUPerc":"97.82%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"1.34%","MemUsage":"212.4MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"212kB / 128kB","PIDs":"15"}
|
||||||
|
{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"3.76kB / 1.76kB","PIDs":"2"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"17.92%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.16%","MemUsage":"25.36MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"180kB / 237kB","PIDs":"19"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"0.06%","Container":"e4e5ae23b118","ID":"e4e5ae23b118","MemPerc":"3.75%","MemUsage":"593.7MiB / 15.47GiB","Name":"load-tests-uploader-1","NetIO":"36kB / 54.2kB","PIDs":"51"}
|
||||||
|
[2J[H{"BlockIO":"225kB / 209kB","CPUPerc":"97.82%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"1.34%","MemUsage":"212.4MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"212kB / 128kB","PIDs":"15"}
|
||||||
|
{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"3.76kB / 1.76kB","PIDs":"2"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"17.92%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.16%","MemUsage":"25.36MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"180kB / 237kB","PIDs":"19"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"0.06%","Container":"e4e5ae23b118","ID":"e4e5ae23b118","MemPerc":"3.75%","MemUsage":"593.7MiB / 15.47GiB","Name":"load-tests-uploader-1","NetIO":"36kB / 54.2kB","PIDs":"51"}
|
||||||
|
[2J[H{"BlockIO":"373kB / 189MB","CPUPerc":"637.26%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.59%","MemUsage":"568.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"254MB / 303kB","PIDs":"28"}
|
||||||
|
{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.26kB / 1.76kB","PIDs":"2"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"55.52%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.53%","MemUsage":"84.67MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"349MB / 258MB","PIDs":"27"}
|
||||||
|
{"BlockIO":"0B / 0B","CPUPerc":"17.23%","Container":"e4e5ae23b118","ID":"e4e5ae23b118","MemPerc":"3.59%","MemUsage":"569.1MiB / 15.47GiB","Name":"load-tests-uploader-1","NetIO":"334kB / 349MB","PIDs":"48"}
|
||||||
|
[2J[H{"BlockIO":"373kB / 189MB","CPUPerc":"637.26%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.59%","MemUsage":"568.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"254MB / 303kB","PIDs":"28"}
|
||||||
|
{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.26kB / 1.76kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"729kB / 526MB","CPUPerc":"172.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.47%","MemUsage":"550.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.46kB / 1.76kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"729kB / 526MB","CPUPerc":"172.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.47%","MemUsage":"550.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.46kB / 1.76kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.27%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.47%","MemUsage":"550.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.85kB / 1.76kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.27%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.47%","MemUsage":"550.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.85kB / 1.76kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.55%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.55%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.02%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.02%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.10%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.10%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.02%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.02%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.03%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.03%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
FROM alpine:3.16.2
|
||||||
|
|
||||||
|
RUN apk add --no-cache curl bash openssl
|
||||||
|
|
||||||
|
COPY ./upload.sh /usr/local/share/upload.sh
|
||||||
|
RUN chmod +x /usr/local/share/upload.sh
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/usr/local/share/upload.sh" ]
|
||||||
|
CMD []
|
||||||
|
|
@ -0,0 +1,41 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
|
echo $@
|
||||||
|
|
||||||
|
if [ -z "$2" ]; then
|
||||||
|
echo "USAGE: upload.sh SIZE NUMBER [TEMP DIR]"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
size="$1"
|
||||||
|
number="$2"
|
||||||
|
directory="${3:-/tmp}"
|
||||||
|
file="${directory}/${size}.bin"
|
||||||
|
|
||||||
|
openssl rand -out "$file" "$size"
|
||||||
|
|
||||||
|
# Get upload size in bytes
|
||||||
|
upload_size=$(stat -c "%s" "$file")
|
||||||
|
echo "Generated file with size: ${upload_size} bytes."
|
||||||
|
|
||||||
|
# Create uploads
|
||||||
|
for i in $(seq 1 $number); do
|
||||||
|
# Note: I wanted to use the new feature for extracting header values
|
||||||
|
# (https://daniel.haxx.se/blog/2022/03/24/easier-header-picking-with-curl/)
|
||||||
|
# but this is not yet available on the current curl version in Alpine Linux.
|
||||||
|
upload_urls[${i}]="$(curl -X POST -H 'Tus-Resumable: 1.0.0' -H "Upload-Length: ${upload_size}" --fail --silent -i http://tusd:1080/files/ | grep -i ^Location: | cut -d: -f2- | sed 's/^ *\(.*\).*/\1/' | tr -d '\r')"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Perform the uploads in parallel
|
||||||
|
for i in $(seq 1 $number); do
|
||||||
|
curl -X PATCH -H 'Tus-Resumable: 1.0.0' -H 'Upload-Offset: 0' -H 'Content-Type: application/offset+octet-stream' --data-binary "@${file}" "${upload_urls[${i}]}" &
|
||||||
|
pids[${i}]=$!
|
||||||
|
done
|
||||||
|
|
||||||
|
# Wait for all uploads to complete
|
||||||
|
for pid in ${pids[*]}; do
|
||||||
|
wait $pid
|
||||||
|
done
|
||||||
|
|
@ -27,32 +27,32 @@ func (g *MyHookHandler) Setup() error {
|
||||||
func (g *MyHookHandler) InvokeHook(req hooks.HookRequest) (res hooks.HookResponse, err error) {
|
func (g *MyHookHandler) InvokeHook(req hooks.HookRequest) (res hooks.HookResponse, err error) {
|
||||||
log.Println("MyHookHandler.InvokeHook is invoked")
|
log.Println("MyHookHandler.InvokeHook is invoked")
|
||||||
|
|
||||||
// Prepare hook response structure
|
// // Prepare hook response structure
|
||||||
res.HTTPResponse.Headers = make(map[string]string)
|
// res.HTTPResponse.Headers = make(map[string]string)
|
||||||
|
|
||||||
// Example: Use the pre-create hook to check if a filename has been supplied
|
// // Example: Use the pre-create hook to check if a filename has been supplied
|
||||||
// using metadata. If not, the upload is rejected with a custom HTTP response.
|
// // using metadata. If not, the upload is rejected with a custom HTTP response.
|
||||||
|
|
||||||
if req.Type == hooks.HookPreCreate {
|
// if req.Type == hooks.HookPreCreate {
|
||||||
if _, ok := req.Event.Upload.MetaData["filename"]; !ok {
|
// if _, ok := req.Event.Upload.MetaData["filename"]; !ok {
|
||||||
res.RejectUpload = true
|
// res.RejectUpload = true
|
||||||
res.HTTPResponse.StatusCode = 400
|
// res.HTTPResponse.StatusCode = 400
|
||||||
res.HTTPResponse.Body = "no filename provided"
|
// res.HTTPResponse.Body = "no filename provided"
|
||||||
res.HTTPResponse.Headers["X-Some-Header"] = "yes"
|
// res.HTTPResponse.Headers["X-Some-Header"] = "yes"
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
// Example: Use the post-finish hook to print information about a completed upload,
|
// // Example: Use the post-finish hook to print information about a completed upload,
|
||||||
// including its storage location.
|
// // including its storage location.
|
||||||
if req.Type == hooks.HookPreFinish {
|
// if req.Type == hooks.HookPreFinish {
|
||||||
id := req.Event.Upload.ID
|
// id := req.Event.Upload.ID
|
||||||
size := req.Event.Upload.Size
|
// size := req.Event.Upload.Size
|
||||||
storage := req.Event.Upload.Storage
|
// storage := req.Event.Upload.Storage
|
||||||
|
|
||||||
log.Printf("Upload %s (%d bytes) is finished. Find the file at:\n", id, size)
|
// log.Printf("Upload %s (%d bytes) is finished. Find the file at:\n", id, size)
|
||||||
log.Println(storage)
|
// log.Println(storage)
|
||||||
|
|
||||||
}
|
// }
|
||||||
|
|
||||||
// Return the hook response to tusd.
|
// Return the hook response to tusd.
|
||||||
return res, nil
|
return res, nil
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue