diff --git a/.infra/kube/tusd-kube.yaml b/.infra/kube/tusd-kube.yaml index cf0d648..f01c763 100644 --- a/.infra/kube/tusd-kube.yaml +++ b/.infra/kube/tusd-kube.yaml @@ -10,13 +10,6 @@ spec: labels: app: tusd spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.google.com/gke-preemptible - operator: DoesNotExist containers: - image: docker.io/tusproject/tusd:latest imagePullPolicy: Always @@ -24,27 +17,15 @@ spec: name: tusd resources: limits: - memory: "2Gi" - # requests: - # memory: "1Gi" + memory: "1Gi" + requests: + memory: "1Gi" ports: - name: tusd-web containerPort: 8080 envFrom: - - configMapRef: - name: tusd-env - secretRef: - name: tusd-s3 - securityContext: - runAsUser: 0 - fsGroup: 0 - volumeMounts: - - name: tusd-account - mountPath: /gcs - volumes: - - name: tusd-account - secret: - secretName: gcs-account + name: tusd-env --- apiVersion: v1 kind: Service @@ -75,6 +56,7 @@ metadata: nginx.ingress.kubernetes.io/proxy-read-timeout: "300" nginx.ingress.kubernetes.io/proxy-request-buffering: "off" nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/ssl-redirect: "true" spec: tls: - hosts: @@ -98,25 +80,3 @@ spec: backend: serviceName: tusd servicePort: 80 ---- -apiVersion: autoscaling/v1 -kind: HorizontalPodAutoscaler -metadata: - name: tusd - namespace: tus -spec: - scaleTargetRef: - apiVersion: apps/v1beta1 - kind: Deployment - name: tusd - minReplicas: 1 - maxReplicas: 5 - metrics: - - type: Resource - resource: - name: cpu - targetAverageUtilization: 80 - - type: Resource - resource: - name: memory - targetAverageValue: 1800Mi diff --git a/.scripts/build_all.sh b/.scripts/build_all.sh index e51f0cb..04f885f 100755 --- a/.scripts/build_all.sh +++ b/.scripts/build_all.sh @@ -11,14 +11,14 @@ compile linux amd64 compile linux arm compile darwin 386 compile darwin amd64 -compile windows 386 .exe -compile windows amd64 .exe +#compile windows 386 .exe +#compile windows amd64 .exe maketar linux 386 maketar linux amd64 maketar linux arm makezip darwin 386 makezip darwin amd64 -makezip windows 386 .exe -makezip windows amd64 .exe +#makezip windows 386 .exe +#makezip windows amd64 .exe makedep amd64 diff --git a/.scripts/deploy_gcloud.sh b/.scripts/deploy_kube.sh similarity index 53% rename from .scripts/deploy_gcloud.sh rename to .scripts/deploy_kube.sh index 612934b..abd52d6 100755 --- a/.scripts/deploy_gcloud.sh +++ b/.scripts/deploy_kube.sh @@ -8,30 +8,25 @@ set -o nounset __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" __root="$(cd "$(dirname "${__dir}")" && pwd)" - -echo "Storing ca.crt inside HOME" -echo $CA_CRT | base64 --decode -i > ${HOME}/ca.crt -echo "ca.crt is saved" +curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl +chmod +x ./kubectl +sudo mv ./kubectl /usr/local/bin/kubectl #Store the new image in docker hub docker build --quiet -t tusproject/tusd:latest -t tusproject/tusd:$TRAVIS_COMMIT ${__root}; -docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"; +docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"; docker push tusproject/tusd:$TRAVIS_COMMIT; docker push tusproject/tusd:latest; - - -gcloud config set container/use_client_certificate True -export CLOUDSDK_CONTAINER_USE_CLIENT_CERTIFICATE=True - -kubectl config set-cluster transloadit-gke-cluster --embed-certs=true --server=${CLUSTER_ENDPOINT} --certificate-authority=${HOME}/ca.crt -kubectl config set-credentials travis --token=$SA_TOKEN -kubectl config set-context travis --cluster=$CLUSTER_NAME --user=travis --namespace=tus -kubectl config use-context travis +echo "Create directory..." +mkdir ${HOME}/.kube +echo "Writing KUBECONFIG to file..." +echo $KUBECONFIGVAR | python -m base64 -d > ${HOME}/.kube/config +echo "KUBECONFIG file written" sleep 10s # This cost me some precious debugging time. -kubectl apply --validate=false -f "${__root}/.infra/kube/tusd-kube.yaml" +kubectl apply -f "${__root}/.infra/kube/tusd-kube.yaml" kubectl set image deployment/tusd --namespace=tus tusd=docker.io/tusproject/tusd:$TRAVIS_COMMIT @@ -43,7 +38,7 @@ kubectl get deployment --namespace=tus function cleanup { printf "Cleaning up...\n" - rm -f ${HOME}/ca.crt + rm -f ${HOME}/.kube/config printf "Cleaning done." } diff --git a/.scripts/generate-docker-library.sh b/.scripts/generate-docker-library.sh new file mode 100755 index 0000000..58a77cd --- /dev/null +++ b/.scripts/generate-docker-library.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# +# official-images tag file generator +# +# usage: ./generate-docker-library.sh > [official-images-folder]/library/tusd +# + +cat <<-EOH +# This file is generated via https://github.com/tus/tusd/blob/master/generate-docker-library.sh +Maintainers: tus.io (@tus), Thomas A. Hirsch (@thirsch) +GitRepo: https://github.com/tus/tusd.git +EOH + +skipBeforeVersion="0.13.0" +previousVersions=(); + +function printVersion() { + version=( ${1//./ } ) + majorMinor="${version[0]}.${version[1]}" + + match=$(echo "${previousVersions[@]:0}" | grep -oE "\s?$majorMinor\s?$") + previousVersionCount=${#previousVersions[@]} + + # add the majorMinor-Version only, if it is not present yet. + if [[ ! -z $match ]] ; then + versionString=$1 + else + versionString="$1 $majorMinor" + previousVersions+=($majorMinor) + fi + + # as the versions are sorted, the very first version gets latest. + if [[ ${previousVersionCount} -eq 0 ]]; then + versionString="$versionString, latest" + fi + + cat <<-EOE + +Tags: $versionString +GitCommit: $2 + EOE +} + +for version in `git tag -l --sort=-v:refname | grep "^[0-9.]\+$"`; do + commit=`git rev-parse ${version}` + + # no official release before this version + if [[ ${version} = ${skipBeforeVersion} ]] ; then + exit 0 + fi + + printVersion "${version}" ${commit} +done \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 9df40d5..d846996 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,12 +22,6 @@ script: - ./.scripts/test_all.sh before_deploy: - if [[ "$TRAVIS_TAG" != "" ]]; then ./.scripts/build_all.sh; fi -- if [ ! -d "$HOME/google-cloud-sdk/bin" ]; then rm -rf $HOME/google-cloud-sdk; curl https://sdk.cloud.google.com | bash; fi -- source /home/travis/google-cloud-sdk/path.bash.inc -- gcloud --quiet version -- gcloud --quiet components update -- gcloud --quiet components update kubectl -- curl https://raw.githubusercontent.com/kubernetes/helm/d87ce93e1e287ece84d940dbfe09b0de493d9953/scripts/get | bash deploy: - provider: releases api_key: @@ -41,7 +35,7 @@ deploy: repo: tus/tusd os: linux - provider: script - script: .scripts/deploy_gcloud.sh + script: .scripts/deploy_kube.sh on: branch: master go: 1.12 diff --git a/Dockerfile b/Dockerfile index 1a0652e..1af9f27 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ COPY . /go/src/github.com/tus/tusd/ WORKDIR /go/src/github.com/tus/tusd RUN apk add --no-cache \ - git \ + git gcc libc-dev \ && go get -d -v ./... \ && version="$(git tag -l --points-at HEAD)" \ && commit=$(git log --format="%H" -n 1) \ @@ -18,11 +18,11 @@ RUN apk add --no-cache \ && apk del git # start a new stage that copies in the binary built in the previous stage -FROM alpine:3.8 +FROM alpine:3.9 COPY --from=builder /go/bin/tusd /usr/local/bin/tusd -RUN apk add --no-cache ca-certificates jq \ +RUN apk add --no-cache ca-certificates jq gcc \ && addgroup -g 1000 tusd \ && adduser -u 1000 -G tusd -s /bin/sh -D tusd \ && mkdir -p /srv/tusd-hooks \ diff --git a/README.md b/README.md index c09006a..f4a24ea 100644 --- a/README.md +++ b/README.md @@ -267,11 +267,11 @@ Yes, this is made possible by the [hook system](/docs/hooks.md) inside the tusd ### Can I run tusd inside a VM/Vagrant/VirtualBox? -Yes, you can absolutely do so without any modifications. However, there is one known problem: If you are using tusd inside VirtualBox (the default provider for Vagrant) and are storing the files inside a shared/synced folder, you might get TemporaryErrors (Lockfile created, but doesn't exist) when trying to upload. This happens because shared folders do not support symbolic links which are necessary for tusd. Please use another non-shared folder for storing files (see https://github.com/tus/tusd/issues/201). +Yes, you can absolutely do so without any modifications. However, there is one known problem: If you are using tusd inside VirtualBox (the default provider for Vagrant) and are storing the files inside a shared/synced folder, you might get TemporaryErrors (Lockfile created, but doesn't exist) when trying to upload. This happens because shared folders do not support hard links which are necessary for tusd. Please use another non-shared folder for storing files (see https://github.com/tus/tusd/issues/201). ### I am getting TemporaryErrors (Lockfile created, but doesn't exist)! What can I do? -This error can occur when you are running tusd's disk storage on a file system which does not support symbolic links. These symbolic links are used to create lock files for ensuring that an upload's data is consistent. For example, this problem can happen when running tusd inside VirtualBox (see the answer above for more details) or when using file system interfaces to cloud storage providers (see https://github.com/tus/tusd/issues/257). We recommend you to ensure that your file system supports symbolic links, use a different file system, or use one of tusd's cloud storage abilities. If the problem still persists, please open a bug report. +This error can occur when you are running tusd's disk storage on a file system which does not support hard links. These hard links are used to create lock files for ensuring that an upload's data is consistent. For example, this problem can happen when running tusd inside VirtualBox (see the answer above for more details) or when using file system interfaces to cloud storage providers (see https://github.com/tus/tusd/issues/257). We recommend you to ensure that your file system supports hard links, use a different file system, or use one of tusd's cloud storage abilities. If the problem still persists, please open a bug report. ### How can I prevent users from downloading the uploaded files? diff --git a/cmd/tusd/cli/flags.go b/cmd/tusd/cli/flags.go index 9720809..23f20a6 100644 --- a/cmd/tusd/cli/flags.go +++ b/cmd/tusd/cli/flags.go @@ -29,6 +29,7 @@ var Flags struct { ExposeMetrics bool MetricsPath string BehindProxy bool + VerboseOutput bool FileHooksInstalled bool HttpHooksInstalled bool @@ -57,6 +58,7 @@ func ParseFlags() { flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage") flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible") flag.BoolVar(&Flags.BehindProxy, "behind-proxy", false, "Respect X-Forwarded-* and similar headers which may be set by proxies") + flag.BoolVar(&Flags.VerboseOutput, "verbose", true, "Enable verbose logging output") flag.Parse() diff --git a/cmd/tusd/cli/hooks.go b/cmd/tusd/cli/hooks.go index a96c107..2902ec3 100644 --- a/cmd/tusd/cli/hooks.go +++ b/cmd/tusd/cli/hooks.go @@ -103,14 +103,16 @@ func invokeHookSync(typ hooks.HookType, info handler.FileInfo, captureOutput boo } name := string(typ) - logEv(stdout, "HookInvocationStart", "type", name, "id", info.ID) + if Flags.VerboseOutput { + logEv(stdout, "HookInvocationStart", "type", name, "id", info.ID) + } output, returnCode, err := hookHandler.InvokeHook(typ, info, captureOutput) if err != nil { logEv(stderr, "HookInvocationError", "type", string(typ), "id", info.ID, "error", err.Error()) MetricsHookErrorsTotal.WithLabelValues(string(typ)).Add(1) - } else { + } else if Flags.VerboseOutput { logEv(stdout, "HookInvocationFinish", "type", string(typ), "id", info.ID) } diff --git a/pkg/handler/unrouted_handler.go b/pkg/handler/unrouted_handler.go index 6ff14e1..0dbb8d4 100644 --- a/pkg/handler/unrouted_handler.go +++ b/pkg/handler/unrouted_handler.go @@ -814,6 +814,13 @@ func (handler *UnroutedHandler) sendError(w http.ResponseWriter, r *http.Request err = errors.New("read tcp: i/o timeout") } + // Errors for connnection resets also contain TCP details, we don't need, e.g: + // read tcp 127.0.0.1:1080->127.0.0.1:10023: read: connection reset by peer + // Therefore, we also trim those down. + if strings.HasSuffix(err.Error(), "read: connection reset by peer") { + err = errors.New("read tcp: connection reset by peer") + } + statusErr, ok := err.(HTTPError) if !ok { statusErr = NewHTTPError(err, http.StatusInternalServerError) diff --git a/pkg/s3store/s3store.go b/pkg/s3store/s3store.go index fd5e871..f27118f 100644 --- a/pkg/s3store/s3store.go +++ b/pkg/s3store/s3store.go @@ -90,7 +90,9 @@ import ( // This regular expression matches every character which is not defined in the // ASCII tables which range from 00 to 7F, inclusive. -var nonASCIIRegexp = regexp.MustCompile(`([^\x00-\x7F])`) +// It also matches the \r and \n characters which are not allowed in values +// for HTTP headers. +var nonASCIIRegexp = regexp.MustCompile(`([^\x00-\x7F]|[\r\n])`) // See the handler.DataStore interface for documentation about the different // methods. @@ -257,8 +259,8 @@ func (store S3Store) WriteChunk(id string, offset int64, src io.Reader) (int64, return 0, err } if incompletePartFile != nil { - defer incompletePartFile.Close() defer os.Remove(incompletePartFile.Name()) + defer incompletePartFile.Close() if err := store.deleteIncompletePartForUpload(uploadId); err != nil { return 0, err diff --git a/pkg/s3store/s3store_test.go b/pkg/s3store/s3store_test.go index 7a317eb..7c3c670 100644 --- a/pkg/s3store/s3store_test.go +++ b/pkg/s3store/s3store_test.go @@ -37,7 +37,7 @@ func TestNewUpload(t *testing.T) { assert.Equal(s3obj, store.Service) s1 := "hello" - s2 := "men?" + s2 := "men???hi" gomock.InOrder( s3obj.EXPECT().CreateMultipartUpload(&s3.CreateMultipartUploadInput{ @@ -53,8 +53,8 @@ func TestNewUpload(t *testing.T) { s3obj.EXPECT().PutObject(&s3.PutObjectInput{ Bucket: aws.String("bucket"), Key: aws.String("uploadId.info"), - Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)), - ContentLength: aws.Int64(int64(171)), + Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü\r\nhi","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)), + ContentLength: aws.Int64(int64(177)), }), ) @@ -63,7 +63,7 @@ func TestNewUpload(t *testing.T) { Size: 500, MetaData: map[string]string{ "foo": "hello", - "bar": "menü", + "bar": "menü\r\nhi", }, }