docker: Add tools for preproducing load issues
This commit is contained in:
Marius
parent
621de70da4
commit
4b707fac26
|
|
@ -14,6 +14,7 @@ RUN set -xe \
|
|||
COPY cmd/ ./cmd/
|
||||
COPY internal/ ./internal/
|
||||
COPY pkg/ ./pkg/
|
||||
COPY examples/ ./examples/
|
||||
|
||||
# Get the version name and git commit as a build argument
|
||||
ARG GIT_VERSION
|
||||
|
|
@ -24,6 +25,10 @@ RUN set -xe \
|
|||
-ldflags="-X github.com/tus/tusd/cmd/tusd/cli.VersionName=${GIT_VERSION} -X github.com/tus/tusd/cmd/tusd/cli.GitCommit=${GIT_COMMIT} -X 'github.com/tus/tusd/cmd/tusd/cli.BuildDate=$(date --utc)'" \
|
||||
-o /go/bin/tusd ./cmd/tusd/main.go
|
||||
|
||||
RUN set -xe \
|
||||
&& GOOS=linux GOARCH=amd64 go build \
|
||||
-o /go/bin/hooks_handler ./examples/hooks/plugin/hook_handler.go
|
||||
|
||||
# start a new stage that copies in the binary built in the previous stage
|
||||
FROM alpine:3.16.2
|
||||
WORKDIR /srv/tusd-data
|
||||
|
|
@ -39,6 +44,7 @@ RUN apk add --no-cache ca-certificates jq bash \
|
|||
&& chmod +x /usr/local/share/docker-entrypoint.sh /usr/local/share/load-env.sh
|
||||
|
||||
COPY --from=builder /go/bin/tusd /usr/local/bin/tusd
|
||||
COPY --from=builder /go/bin/hooks_handler /usr/local/bin/hooks_handler
|
||||
|
||||
EXPOSE 1080
|
||||
USER tusd
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
# Setup traps, so our background job of monitoring the containers
|
||||
# exits, if the script is complete.
|
||||
trap "exit" INT TERM ERR
|
||||
trap "kill 0" EXIT
|
||||
|
||||
# 1) Ensure that the containers are up-to-date
|
||||
docker compose build
|
||||
|
||||
# 2) Start the container monitoring
|
||||
docker stats --format "{{ json . }}" > resource-usage-log.txt &
|
||||
|
||||
# 3) Run the actual tests
|
||||
docker compose up --abort-on-container-exit
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import re
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
snapshots = []
|
||||
|
||||
with open("./resource-usage-log.txt") as file:
|
||||
current_snapshot = None
|
||||
for line in file:
|
||||
# The lines might contain the reset characters before the actual JSON.
|
||||
# This means that the entire resources for the current time have been
|
||||
# written out, so we add the latest snapshot to our list and continue
|
||||
# reading the next entries.
|
||||
first_backet = line.find("{")
|
||||
if first_backet == -1:
|
||||
continue
|
||||
|
||||
if first_backet != 0:
|
||||
if current_snapshot is not None:
|
||||
snapshots.append(current_snapshot)
|
||||
|
||||
current_snapshot = []
|
||||
line = line[first_backet:]
|
||||
|
||||
current_snapshot.append(json.loads(line))
|
||||
|
||||
def parse_percentage(string):
|
||||
return float(string.strip('%'))
|
||||
|
||||
units = {"B": 1, "kB": 10**3, "MB": 10**6, "GB": 10**9, "TB": 10**12,
|
||||
"KiB": 2**10, "MiB": 2**20, "GiB": 2**30, "TiB": 2**40}
|
||||
|
||||
def parse_byte_size(size):
|
||||
number, unit = re.findall(r'([0-9\.]+)([A-Za-z]+)', size)[0]
|
||||
return int(float(number)*units[unit])
|
||||
|
||||
def parse_two_bytes(string):
|
||||
str1, str2 = string.split("/")
|
||||
return parse_byte_size(str1), parse_byte_size(str2)
|
||||
|
||||
s3_cpu = []
|
||||
s3_mem = []
|
||||
tusd_cpu = []
|
||||
tusd_mem = []
|
||||
tusd_net = []
|
||||
uploader_cpu = []
|
||||
uploader_mem = []
|
||||
uploader_net = []
|
||||
timestamp = []
|
||||
|
||||
for (i, snapshot) in enumerate(snapshots):
|
||||
a_s3_cpu = None
|
||||
a_s3_mem = None
|
||||
a_tusd_cpu = None
|
||||
a_tusd_mem = None
|
||||
a_tusd_net = None
|
||||
a_uploader_cpu = None
|
||||
a_uploader_mem = None
|
||||
a_uploader_net = None
|
||||
|
||||
for entry in snapshot:
|
||||
if entry["Name"] == "load-tests-tusd-1":
|
||||
a_tusd_cpu = parse_percentage(entry["CPUPerc"])
|
||||
a_tusd_mem = parse_two_bytes(entry["MemUsage"])[0]
|
||||
a_tusd_net = parse_two_bytes(entry["NetIO"])[0]
|
||||
elif entry["Name"] == "load-tests-s3-1":
|
||||
a_s3_cpu = parse_percentage(entry["CPUPerc"])
|
||||
a_s3_mem = parse_two_bytes(entry["MemUsage"])[0]
|
||||
elif entry["Name"] == "load-tests-uploader-1":
|
||||
a_uploader_cpu = parse_percentage(entry["CPUPerc"])
|
||||
a_uploader_mem = parse_two_bytes(entry["MemUsage"])[0]
|
||||
a_uploader_net = parse_two_bytes(entry["NetIO"])[1]
|
||||
|
||||
s3_cpu.append(a_s3_cpu)
|
||||
s3_mem.append(a_s3_mem)
|
||||
tusd_cpu.append(a_tusd_cpu)
|
||||
tusd_mem.append(a_tusd_mem)
|
||||
tusd_net.append(a_tusd_net)
|
||||
uploader_cpu.append(a_uploader_cpu)
|
||||
uploader_mem.append(a_uploader_mem)
|
||||
uploader_net.append(a_uploader_net)
|
||||
|
||||
# The docker stats command is hard coded to output stats every 500ms:
|
||||
# https://github.com/docker/cli/blob/81c68913e4c2cb058b5a9fd5972e2989d9915b2c/cli/command/container/stats.go#L223
|
||||
timestamp.append(0.5 * i)
|
||||
|
||||
fig, axs = plt.subplots(3, 3, sharex=True, sharey='row')
|
||||
axs[0, 0].plot(timestamp, tusd_cpu)
|
||||
axs[0, 0].set_title('tusd CPU percentage')
|
||||
axs[0, 0].set(ylabel='CPU perc', xlabel='time')
|
||||
axs[0, 1].plot(timestamp, s3_cpu)
|
||||
axs[0, 1].set_title('s3 CPU percentage')
|
||||
axs[0, 1].set(ylabel='CPU perc', xlabel='time')
|
||||
axs[0, 2].plot(timestamp, uploader_cpu)
|
||||
axs[0, 2].set_title('uploader CPU percentage')
|
||||
axs[0, 2].set(ylabel='CPU perc', xlabel='time')
|
||||
|
||||
axs[1, 0].plot(timestamp, tusd_mem)
|
||||
axs[1, 0].set_title('tusd memory usage')
|
||||
axs[1, 0].set(ylabel='mem perc', xlabel='time')
|
||||
axs[1, 1].plot(timestamp, s3_mem)
|
||||
axs[1, 1].set_title('s3 memory usage')
|
||||
axs[1, 1].set(ylabel='mem perc', xlabel='time')
|
||||
axs[1, 2].plot(timestamp, uploader_mem)
|
||||
axs[1, 2].set_title('uploader memory usage')
|
||||
axs[1, 2].set(ylabel='mem perc', xlabel='time')
|
||||
|
||||
axs[2, 0].plot(timestamp, tusd_net)
|
||||
axs[2, 0].set_title('tusd network input')
|
||||
axs[2, 0].set(ylabel='total volume', xlabel='time')
|
||||
axs[2, 1].axis('off')
|
||||
axs[2, 2].plot(timestamp, uploader_net)
|
||||
axs[2, 2].set_title('uploader network output')
|
||||
axs[2, 2].set(ylabel='total volume', xlabel='time')
|
||||
|
||||
|
||||
# Hide x labels and tick labels for top plots and y ticks for right plots.
|
||||
for ax in axs.flat:
|
||||
ax.label_outer()
|
||||
|
||||
plt.show()
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
## Load issues with tusd & S3
|
||||
|
||||
This
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
version: "3.9"
|
||||
|
||||
# TODO:
|
||||
# - Add service for monitoring tusd
|
||||
# - Add hooks
|
||||
# - Use similar configuration as api2
|
||||
|
||||
services:
|
||||
s3:
|
||||
image: minio/minio
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
# Note: Data directory is not persistent on purpose
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
# deploy:
|
||||
# resources:
|
||||
# limits:
|
||||
# cpus: "2"
|
||||
|
||||
createbucket:
|
||||
image: minio/mc
|
||||
entrypoint: >
|
||||
/bin/sh -c "
|
||||
/usr/bin/mc config host add s3 http://s3:9000 minioadmin minioadmin;
|
||||
/usr/bin/mc mb --ignore-existing s3/tusdtest.transloadit.com;
|
||||
sleep infinity;
|
||||
"
|
||||
depends_on:
|
||||
- s3
|
||||
|
||||
tusd:
|
||||
build: ../../
|
||||
ports:
|
||||
- "1080:1080"
|
||||
# entrypoint: file /srv/tusdhook/hook_handler
|
||||
entrypoint: tusd -s3-bucket "tusdtest.transloadit.com" -s3-endpoint "http://s3:9000" -hooks-plugin=/usr/local/bin/hooks_handler -hooks-enabled-events=pre-create,post-create,post-receive,post-finish -progress-hooks-interval=3000 -max-size=128849018880 -timeout=60000 -s3-disable-content-hashes=true -s3-disable-ssl=true -s3-concurrent-part-uploads=48 -s3-max-buffered-parts=1
|
||||
environment:
|
||||
AWS_REGION: us-east-1
|
||||
AWS_ACCESS_KEY_ID: minioadmin
|
||||
AWS_SECRET_ACCESS_KEY: minioadmin
|
||||
depends_on:
|
||||
- s3
|
||||
- createbucket
|
||||
volumes:
|
||||
- ../../examples/hooks/plugin:/srv/tusdhook
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "2"
|
||||
|
||||
uploader:
|
||||
build: ./uploader
|
||||
# 10 MiB: 10485760
|
||||
# 100 MiB: 104857600
|
||||
# 1000 MiB: 1048576000
|
||||
command: 10485760 50 /dev/shm
|
||||
tmpfs:
|
||||
- /dev/shm
|
||||
depends_on:
|
||||
- tusd
|
||||
# deploy:
|
||||
# resources:
|
||||
# limits:
|
||||
# cpus: "1"
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
[2J[H[2J[H[2J[H{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"0.05%","MemUsage":"8.633MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"220B / 0B","PIDs":"9"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"}
|
||||
[2J[H{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"0.05%","MemUsage":"8.633MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"220B / 0B","PIDs":"9"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"045ce1160eb0","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"}
|
||||
[2J[H{"BlockIO":"115kB / 4.1kB","CPUPerc":"34.90%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"0.57%","MemUsage":"90.34MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"10.2kB / 3.43kB","PIDs":"13"}
|
||||
{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.01%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"2.41kB / 1.76kB","PIDs":"2"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.08%","MemUsage":"12.74MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"90B / 0B","PIDs":"19"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"e4e5ae23b118","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"}
|
||||
[2J[H{"BlockIO":"115kB / 4.1kB","CPUPerc":"34.90%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"0.57%","MemUsage":"90.34MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"10.2kB / 3.43kB","PIDs":"13"}
|
||||
{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.01%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"2.41kB / 1.76kB","PIDs":"2"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.08%","MemUsage":"12.74MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"90B / 0B","PIDs":"19"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"0.00%","Container":"e4e5ae23b118","ID":"","MemPerc":"0.00%","MemUsage":"0B / 0B","Name":"--","NetIO":"0B / 0B","PIDs":"0"}
|
||||
[2J[H{"BlockIO":"225kB / 209kB","CPUPerc":"97.82%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"1.34%","MemUsage":"212.4MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"212kB / 128kB","PIDs":"15"}
|
||||
{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"3.76kB / 1.76kB","PIDs":"2"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"17.92%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.16%","MemUsage":"25.36MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"180kB / 237kB","PIDs":"19"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"0.06%","Container":"e4e5ae23b118","ID":"e4e5ae23b118","MemPerc":"3.75%","MemUsage":"593.7MiB / 15.47GiB","Name":"load-tests-uploader-1","NetIO":"36kB / 54.2kB","PIDs":"51"}
|
||||
[2J[H{"BlockIO":"225kB / 209kB","CPUPerc":"97.82%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"1.34%","MemUsage":"212.4MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"212kB / 128kB","PIDs":"15"}
|
||||
{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"3.76kB / 1.76kB","PIDs":"2"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"17.92%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.16%","MemUsage":"25.36MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"180kB / 237kB","PIDs":"19"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"0.06%","Container":"e4e5ae23b118","ID":"e4e5ae23b118","MemPerc":"3.75%","MemUsage":"593.7MiB / 15.47GiB","Name":"load-tests-uploader-1","NetIO":"36kB / 54.2kB","PIDs":"51"}
|
||||
[2J[H{"BlockIO":"373kB / 189MB","CPUPerc":"637.26%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.59%","MemUsage":"568.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"254MB / 303kB","PIDs":"28"}
|
||||
{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.26kB / 1.76kB","PIDs":"2"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"55.52%","Container":"045ce1160eb0","ID":"045ce1160eb0","MemPerc":"0.53%","MemUsage":"84.67MiB / 15.47GiB","Name":"load-tests-tusd-1","NetIO":"349MB / 258MB","PIDs":"27"}
|
||||
{"BlockIO":"0B / 0B","CPUPerc":"17.23%","Container":"e4e5ae23b118","ID":"e4e5ae23b118","MemPerc":"3.59%","MemUsage":"569.1MiB / 15.47GiB","Name":"load-tests-uploader-1","NetIO":"334kB / 349MB","PIDs":"48"}
|
||||
[2J[H{"BlockIO":"373kB / 189MB","CPUPerc":"637.26%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.59%","MemUsage":"568.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"254MB / 303kB","PIDs":"28"}
|
||||
{"BlockIO":"213kB / 4.1kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.26kB / 1.76kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"729kB / 526MB","CPUPerc":"172.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.47%","MemUsage":"550.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.46kB / 1.76kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"729kB / 526MB","CPUPerc":"172.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.47%","MemUsage":"550.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.46kB / 1.76kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.27%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.47%","MemUsage":"550.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.85kB / 1.76kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.27%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.47%","MemUsage":"550.3MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.85kB / 1.76kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.55%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.55%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.02%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.02%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"4.96kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.10%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.10%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.02%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.02%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.00%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
{"BlockIO":"213kB / 16.4kB","CPUPerc":"0.00%","Container":"6f791422d2d9","ID":"6f791422d2d9","MemPerc":"0.01%","MemUsage":"1.168MiB / 15.47GiB","Name":"load-tests-createbucket-1","NetIO":"5.14kB / 1.8kB","PIDs":"2"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.03%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
[2J[H{"BlockIO":"733kB / 526MB","CPUPerc":"0.03%","Container":"ae420f5bad2a","ID":"ae420f5bad2a","MemPerc":"3.23%","MemUsage":"511.1MiB / 15.47GiB","Name":"load-tests-s3-1","NetIO":"528MB / 485kB","PIDs":"51"}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
FROM alpine:3.16.2
|
||||
|
||||
RUN apk add --no-cache curl bash openssl
|
||||
|
||||
COPY ./upload.sh /usr/local/share/upload.sh
|
||||
RUN chmod +x /usr/local/share/upload.sh
|
||||
|
||||
ENTRYPOINT [ "/usr/local/share/upload.sh" ]
|
||||
CMD []
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
echo $@
|
||||
|
||||
if [ -z "$2" ]; then
|
||||
echo "USAGE: upload.sh SIZE NUMBER [TEMP DIR]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
size="$1"
|
||||
number="$2"
|
||||
directory="${3:-/tmp}"
|
||||
file="${directory}/${size}.bin"
|
||||
|
||||
openssl rand -out "$file" "$size"
|
||||
|
||||
# Get upload size in bytes
|
||||
upload_size=$(stat -c "%s" "$file")
|
||||
echo "Generated file with size: ${upload_size} bytes."
|
||||
|
||||
# Create uploads
|
||||
for i in $(seq 1 $number); do
|
||||
# Note: I wanted to use the new feature for extracting header values
|
||||
# (https://daniel.haxx.se/blog/2022/03/24/easier-header-picking-with-curl/)
|
||||
# but this is not yet available on the current curl version in Alpine Linux.
|
||||
upload_urls[${i}]="$(curl -X POST -H 'Tus-Resumable: 1.0.0' -H "Upload-Length: ${upload_size}" --fail --silent -i http://tusd:1080/files/ | grep -i ^Location: | cut -d: -f2- | sed 's/^ *\(.*\).*/\1/' | tr -d '\r')"
|
||||
done
|
||||
|
||||
# Perform the uploads in parallel
|
||||
for i in $(seq 1 $number); do
|
||||
curl -X PATCH -H 'Tus-Resumable: 1.0.0' -H 'Upload-Offset: 0' -H 'Content-Type: application/offset+octet-stream' --data-binary "@${file}" "${upload_urls[${i}]}" &
|
||||
pids[${i}]=$!
|
||||
done
|
||||
|
||||
# Wait for all uploads to complete
|
||||
for pid in ${pids[*]}; do
|
||||
wait $pid
|
||||
done
|
||||
|
|
@ -27,32 +27,32 @@ func (g *MyHookHandler) Setup() error {
|
|||
func (g *MyHookHandler) InvokeHook(req hooks.HookRequest) (res hooks.HookResponse, err error) {
|
||||
log.Println("MyHookHandler.InvokeHook is invoked")
|
||||
|
||||
// Prepare hook response structure
|
||||
res.HTTPResponse.Headers = make(map[string]string)
|
||||
// // Prepare hook response structure
|
||||
// res.HTTPResponse.Headers = make(map[string]string)
|
||||
|
||||
// Example: Use the pre-create hook to check if a filename has been supplied
|
||||
// using metadata. If not, the upload is rejected with a custom HTTP response.
|
||||
// // Example: Use the pre-create hook to check if a filename has been supplied
|
||||
// // using metadata. If not, the upload is rejected with a custom HTTP response.
|
||||
|
||||
if req.Type == hooks.HookPreCreate {
|
||||
if _, ok := req.Event.Upload.MetaData["filename"]; !ok {
|
||||
res.RejectUpload = true
|
||||
res.HTTPResponse.StatusCode = 400
|
||||
res.HTTPResponse.Body = "no filename provided"
|
||||
res.HTTPResponse.Headers["X-Some-Header"] = "yes"
|
||||
}
|
||||
}
|
||||
// if req.Type == hooks.HookPreCreate {
|
||||
// if _, ok := req.Event.Upload.MetaData["filename"]; !ok {
|
||||
// res.RejectUpload = true
|
||||
// res.HTTPResponse.StatusCode = 400
|
||||
// res.HTTPResponse.Body = "no filename provided"
|
||||
// res.HTTPResponse.Headers["X-Some-Header"] = "yes"
|
||||
// }
|
||||
// }
|
||||
|
||||
// Example: Use the post-finish hook to print information about a completed upload,
|
||||
// including its storage location.
|
||||
if req.Type == hooks.HookPreFinish {
|
||||
id := req.Event.Upload.ID
|
||||
size := req.Event.Upload.Size
|
||||
storage := req.Event.Upload.Storage
|
||||
// // Example: Use the post-finish hook to print information about a completed upload,
|
||||
// // including its storage location.
|
||||
// if req.Type == hooks.HookPreFinish {
|
||||
// id := req.Event.Upload.ID
|
||||
// size := req.Event.Upload.Size
|
||||
// storage := req.Event.Upload.Storage
|
||||
|
||||
log.Printf("Upload %s (%d bytes) is finished. Find the file at:\n", id, size)
|
||||
log.Println(storage)
|
||||
// log.Printf("Upload %s (%d bytes) is finished. Find the file at:\n", id, size)
|
||||
// log.Println(storage)
|
||||
|
||||
}
|
||||
// }
|
||||
|
||||
// Return the hook response to tusd.
|
||||
return res, nil
|
||||
|
|
|
|||
Loading…
Reference in New Issue