From 6795bd7669f09c371aef44ed2ead0d3ab202dd0b Mon Sep 17 00:00:00 2001 From: kiloreux Date: Thu, 11 Jan 2018 11:08:20 +0100 Subject: [PATCH 1/7] Added NFS support for easier blue/green deployments --- .infra/kube/tusd-kube.yaml | 29 ++++++++++++++++++++++------- .scripts/deploy_gcloud.sh | 2 +- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.infra/kube/tusd-kube.yaml b/.infra/kube/tusd-kube.yaml index c44db1a..027e52d 100644 --- a/.infra/kube/tusd-kube.yaml +++ b/.infra/kube/tusd-kube.yaml @@ -1,15 +1,30 @@ apiVersion: v1 -kind: PersistentVolumeClaim +kind: PersistentVolume metadata: - name: tusd - namespace: tus + name: nfs-tusd + namespace: tusd +spec: + capacity: + storage: 50Gi + accessModes: + - ReadWriteMany + nfs: + server: NFS_SERVER_IP + path: "/tusd" + +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: nfs-tusd + namespace: tusd spec: accessModes: - - ReadWriteOnce + - ReadWriteMany + storageClassName: "" resources: requests: storage: 50Gi - storageClassName: standard --- apiVersion: extensions/v1beta1 kind: Deployment @@ -22,7 +37,7 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 + maxSurge: 2 maxUnavailable: 0 template: metadata: @@ -52,7 +67,7 @@ spec: volumes: - name: tusd-disk persistentVolumeClaim: - claimName: tusd + claimName: nfs-tusd --- apiVersion: v1 kind: Service diff --git a/.scripts/deploy_gcloud.sh b/.scripts/deploy_gcloud.sh index edc720c..b2b45b4 100755 --- a/.scripts/deploy_gcloud.sh +++ b/.scripts/deploy_gcloud.sh @@ -30,7 +30,7 @@ kubectl config set-credentials travis --token=$SA_TOKEN kubectl config set-context travis --cluster=$CLUSTER_NAME --user=travis --namespace=tus kubectl config use-context travis - +sed -i 's#NFS_SERVER_IP#${NFS_SERVER_IP}#' ./.infra/kube/*.yaml kubectl apply --validate=false -f "${__root}/.infra/kube/tusd-kube.yaml" From bf1b270797d459eced203f09d349ad6eb10eca01 Mon Sep 17 00:00:00 2001 From: kiloreux Date: Thu, 11 Jan 2018 12:03:14 +0100 Subject: [PATCH 2/7] tus namespace --- .infra/kube/tusd-kube.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.infra/kube/tusd-kube.yaml b/.infra/kube/tusd-kube.yaml index 027e52d..8f1a6b0 100644 --- a/.infra/kube/tusd-kube.yaml +++ b/.infra/kube/tusd-kube.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: PersistentVolume metadata: name: nfs-tusd - namespace: tusd + namespace: tus spec: capacity: storage: 50Gi @@ -17,7 +17,7 @@ kind: PersistentVolumeClaim apiVersion: v1 metadata: name: nfs-tusd - namespace: tusd + namespace: tus spec: accessModes: - ReadWriteMany From 25c1b78e9dd50533dca6a8ed3ad64d1f2e935765 Mon Sep 17 00:00:00 2001 From: kiloreux Date: Thu, 11 Jan 2018 12:24:40 +0100 Subject: [PATCH 3/7] Not literal --- .infra/kube/tusd-kube.yaml | 1 - .scripts/deploy_gcloud.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.infra/kube/tusd-kube.yaml b/.infra/kube/tusd-kube.yaml index 8f1a6b0..63a8e15 100644 --- a/.infra/kube/tusd-kube.yaml +++ b/.infra/kube/tusd-kube.yaml @@ -11,7 +11,6 @@ spec: nfs: server: NFS_SERVER_IP path: "/tusd" - --- kind: PersistentVolumeClaim apiVersion: v1 diff --git a/.scripts/deploy_gcloud.sh b/.scripts/deploy_gcloud.sh index b2b45b4..55147cb 100755 --- a/.scripts/deploy_gcloud.sh +++ b/.scripts/deploy_gcloud.sh @@ -30,7 +30,7 @@ kubectl config set-credentials travis --token=$SA_TOKEN kubectl config set-context travis --cluster=$CLUSTER_NAME --user=travis --namespace=tus kubectl config use-context travis -sed -i 's#NFS_SERVER_IP#${NFS_SERVER_IP}#' ./.infra/kube/*.yaml +sed -i "s#NFS_SERVER_IP#${NFS_SERVER_IP}#" ./.infra/kube/*.yaml kubectl apply --validate=false -f "${__root}/.infra/kube/tusd-kube.yaml" From 367ba5a8b3cf9a6bfaa7ecb0913ce7a590c9b8f5 Mon Sep 17 00:00:00 2001 From: kiloreux Date: Fri, 12 Jan 2018 00:58:59 +0100 Subject: [PATCH 4/7] Take another test of NFS --- .infra/kube/nfs.yaml | 14 ++++++++++++++ .infra/kube/tusd-kube.yaml | 20 +++----------------- .scripts/deploy_gcloud.sh | 4 +++- 3 files changed, 20 insertions(+), 18 deletions(-) create mode 100644 .infra/kube/nfs.yaml diff --git a/.infra/kube/nfs.yaml b/.infra/kube/nfs.yaml new file mode 100644 index 0000000..68a4fae --- /dev/null +++ b/.infra/kube/nfs.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nfs-tusd + namespace: tus +spec: + capacity: + storage: 20Gi + accessModes: + - ReadWriteMany + nfs: + # FIXME: use the right IP + server: NFS_SERVER_IP + path: "/" diff --git a/.infra/kube/tusd-kube.yaml b/.infra/kube/tusd-kube.yaml index 63a8e15..22ae98d 100644 --- a/.infra/kube/tusd-kube.yaml +++ b/.infra/kube/tusd-kube.yaml @@ -1,17 +1,3 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: nfs-tusd - namespace: tus -spec: - capacity: - storage: 50Gi - accessModes: - - ReadWriteMany - nfs: - server: NFS_SERVER_IP - path: "/tusd" ---- kind: PersistentVolumeClaim apiVersion: v1 metadata: @@ -23,7 +9,7 @@ spec: storageClassName: "" resources: requests: - storage: 50Gi + storage: 20Gi --- apiVersion: extensions/v1beta1 kind: Deployment @@ -58,8 +44,8 @@ spec: ports: - containerPort: 8080 securityContext: - runAsUser: 1000 - fsGroup: 1000 + runAsUser: 0 + fsGroup: 0 volumeMounts: - name: tusd-disk mountPath: /srv/tusd-data diff --git a/.scripts/deploy_gcloud.sh b/.scripts/deploy_gcloud.sh index 55147cb..0316ede 100755 --- a/.scripts/deploy_gcloud.sh +++ b/.scripts/deploy_gcloud.sh @@ -30,7 +30,9 @@ kubectl config set-credentials travis --token=$SA_TOKEN kubectl config set-context travis --cluster=$CLUSTER_NAME --user=travis --namespace=tus kubectl config use-context travis -sed -i "s#NFS_SERVER_IP#${NFS_SERVER_IP}#" ./.infra/kube/*.yaml +sed -i "s#NFS_SERVER_IP#${NFS_SERVER_IP}#" ./.infra/kube/nfs.yaml +kubectl apply -f "${__root}/.infra/kube/nfs.yaml" +sleep 10s # This cost me some precious debugging time. kubectl apply --validate=false -f "${__root}/.infra/kube/tusd-kube.yaml" From ca69a48f69c8ddc5031371ce656040ef4e11489f Mon Sep 17 00:00:00 2001 From: kiloreux Date: Tue, 16 Jan 2018 16:09:02 +0100 Subject: [PATCH 5/7] Do not limit CPU --- .infra/kube/tusd-kube.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.infra/kube/tusd-kube.yaml b/.infra/kube/tusd-kube.yaml index 22ae98d..8cbf98a 100644 --- a/.infra/kube/tusd-kube.yaml +++ b/.infra/kube/tusd-kube.yaml @@ -36,10 +36,8 @@ spec: name: tusd resources: limits: - cpu: 0.7 memory: "2Gi" requests: - cpu: 0.5 memory: "1Gi" ports: - containerPort: 8080 From 755e892e3039f333f2738d44749e6934af8c189e Mon Sep 17 00:00:00 2001 From: kiloreux Date: Fri, 16 Feb 2018 14:46:20 +0100 Subject: [PATCH 6/7] Fix mimetype filter @kvz --- .infra/kube/tusd-kube.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.infra/kube/tusd-kube.yaml b/.infra/kube/tusd-kube.yaml index 8cbf98a..cffd91f 100644 --- a/.infra/kube/tusd-kube.yaml +++ b/.infra/kube/tusd-kube.yaml @@ -40,7 +40,8 @@ spec: requests: memory: "1Gi" ports: - - containerPort: 8080 + - name: tusd-web + containerPort: 8080 securityContext: runAsUser: 0 fsGroup: 0 @@ -59,7 +60,8 @@ metadata: namespace: tus spec: ports: - - port: 80 + - name: tusd-web + port: 80 targetPort: 8080 protocol: TCP selector: @@ -73,6 +75,14 @@ metadata: annotations: kubernetes.io/tls-acme: "true" kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/configuration-snippet: | + # Security: Don't allow people to upload html and then have browsers parse it + if ($uri ~ \.(?!(jpe?g|gif|png|webp|webm|mp4|mpg|avi|3gp|wav|mp3))$) { + add_header Content-Type application/octet-stream; + add_header Content-Disposition "attachment; filename=$basename"; + add_header X-Download-Options noopen; + add_header X-Content-Type-Options nosniff; + } nginx.ingress.kubernetes.io/proxy-body-size: 0m nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" nginx.ingress.kubernetes.io/proxy-read-timeout: "300" From cdafea14dcef2d4b6181141a52661bdd562d78d0 Mon Sep 17 00:00:00 2001 From: Marius Date: Fri, 16 Feb 2018 23:17:13 +0100 Subject: [PATCH 7/7] Update README.md --- README.md | 104 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 94 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 1133d4f..6f6a41d 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,11 @@ specifies a flexible method to upload files to remote servers using HTTP. The special feature is the ability to pause and resume uploads at any moment allowing to continue seamlessly after e.g. network interruptions. +It is capable of accepting uploads with arbitrary sizes and storing them locally +on disk, on Google Cloud Storage or on AWS S3 (or any other S3-compatible +storage system). Due to its modularization and extensibility, support for +nearly any other cloud provider could easily be added to tusd. + **Protocol version:** 1.0.0 ## Getting started @@ -26,23 +31,102 @@ Windows in various formats of the ### Compile from source -**Requirements:** - -* [Go](http://golang.org/doc/install) (1.5 or newer) - -**Running tusd from source:** - -Clone the git repository and `cd` into it. +The only requirement for building tusd is [Go](http://golang.org/doc/install) 1.5 or newer. +If you meet this criteria, you can clone the git repository and build the binary: ```bash git clone git@github.com:tus/tusd.git cd tusd +go build -o tusd cmd/tusd/main ``` -Now you can run tusd: +## Running tusd -```bash -go run cmd/tusd/main.go +Start the tusd upload server is as simple as invoking a single command. For example, following +snippet demostrates how to start a tusd process which accepts tus uploads at +`http://localhost:1080/files/` and stores them locally in the `./data` directory: + +``` +$ tusd -dir ./data +[tusd] Using './data' as directory storage. +[tusd] Using 0.00MB as maximum size. +[tusd] Using 0.0.0.0:1080 as address to listen. +[tusd] Using /files/ as the base path. +[tusd] Using /metrics as the metrics path. +``` + +Alternatively, if you want to store the uploads on an AWS S3 bucket, you only have to specify +the bucket and provide the corresponding access credentials and region information using +environment variables (if you want to use a S3-compatible store, use can use the `-s3-endpoint` +option): + +``` +$ export AWS_ACCESS_KEY_ID=xxxxx +$ export AWS_SECRET_ACCESS_KEY=xxxxx +$ export AWS_REGION=eu-west-1 +$ tusd -s3-bucket my-test-bucket.com +[tusd] Using 's3://my-test-bucket.com' as S3 bucket for storage. +[tusd] Using 0.00MB as maximum size. +[tusd] Using 0.0.0.0:1080 as address to listen. +[tusd] Using /files/ as the base path. +[tusd] Using /metrics as the metrics path. +``` + +Furthermore, tusd also has support for storing uploads on Google Cloud Storage. In order to +enable this feature, supply the path to your account file containing the necessary credentials: + +``` +$ export GCS_SERVICE_ACCOUNT_FILE=./account.json +$ tusd -gcs-bucket my-test-bucket.com +[tusd] Using 'gcs://my-test-bucket.com' as GCS bucket for storage. +[tusd] Using 0.00MB as maximum size. +[tusd] Using 0.0.0.0:1080 as address to listen. +[tusd] Using /files/ as the base path. +[tusd] Using /metrics as the metrics path. +``` + +Besides these simple examples, tusd can be easily configured using a variety of command line +options: + +``` +$ tusd -help +Usage of tusd: + -base-path string + Basepath of the HTTP server (default "/files/") + -behind-proxy + Respect X-Forwarded-* and similar headers which may be set by proxies + -dir string + Directory to store uploads in (default "./data") + -expose-metrics + Expose metrics about tusd usage (default true) + -gcs-bucket string + Use Google Cloud Storage with this bucket as storage backend (requires the GCS_SERVICE_ACCOUNT_FILE environment variable to be set) + -hooks-dir string + Directory to search for available hooks scripts + -hooks-http string + An HTTP endpoint to which hook events will be sent to + -hooks-http-backoff int + Number of seconds to wait before retrying each retry (default 1) + -hooks-http-retry int + Number of times to retry on a 500 or network timeout (default 3) + -host string + Host to bind HTTP server to (default "0.0.0.0") + -max-size int + Maximum size of a single upload in bytes + -metrics-path string + Path under which the metrics endpoint will be accessible (default "/metrics") + -port string + Port to bind HTTP server to (default "1080") + -s3-bucket string + Use AWS S3 with this bucket as storage backend (requires the AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION environment variables to be set) + -s3-endpoint string + Endpoint to use S3 compatible implementations like minio (requires s3-bucket to be pass) + -store-size int + Size of space allowed for storage + -timeout int + Read timeout for connections in milliseconds. A zero value means that reads will not timeout (default 30000) + -version + Print tusd version information ``` ## Using tusd manually