Merge branch 'master' of github.com:tus/tusd into v1

.infra/kube/tusd-kube.yaml

@@ -10,13 +10,6 @@ spec:
       labels:
         app: tusd
     spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-              nodeSelectorTerms:
-              - matchExpressions:
-                - key: cloud.google.com/gke-preemptible
-                  operator: DoesNotExist   
       containers:
       - image: docker.io/tusproject/tusd:latest
         imagePullPolicy: Always
@@ -24,27 +17,15 @@ spec:
         name: tusd
         resources:
           limits:
-            memory: "2Gi"
+            memory: "1Gi"
-          # requests:
+          requests:
-          #   memory: "1Gi"
+            memory: "1Gi"
         ports:
         - name: tusd-web
           containerPort: 8080
         envFrom:
-        - configMapRef:
-            name: tusd-env
         - secretRef:
-            name: tusd-s3
+            name: tusd-env
-        securityContext:
-          runAsUser: 0
-          fsGroup: 0
-        volumeMounts:
-        - name: tusd-account
-          mountPath: /gcs
-      volumes:
-      - name: tusd-account
-        secret:
-          secretName: gcs-account
 ---
 apiVersion: v1
 kind: Service
@@ -75,6 +56,7 @@ metadata:
     nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
     nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
     nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
 spec:
   tls:
   - hosts:
@@ -98,25 +80,3 @@ spec:
         backend:
           serviceName: tusd
           servicePort: 80
----
-apiVersion: autoscaling/v1
-kind: HorizontalPodAutoscaler
-metadata:
-  name: tusd
-  namespace: tus
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1beta1
-    kind: Deployment
-    name: tusd
-  minReplicas: 1
-  maxReplicas: 5
-  metrics:
-  - type: Resource
-    resource:
-      name: cpu
-      targetAverageUtilization: 80
-  - type: Resource
-    resource:
-      name: memory
-      targetAverageValue: 1800Mi

.scripts/build_all.sh

@@ -11,14 +11,14 @@ compile linux   amd64
 compile linux   arm
 compile darwin  386
 compile darwin  amd64
-compile windows 386   .exe
+#compile windows 386   .exe
-compile windows amd64 .exe
+#compile windows amd64 .exe
 
 maketar linux   386
 maketar linux   amd64
 maketar linux   arm
 makezip darwin  386
 makezip darwin  amd64
-makezip windows 386   .exe
+#makezip windows 386   .exe
-makezip windows amd64 .exe
+#makezip windows amd64 .exe
 makedep amd64

.scripts/deploy_kube.sh

@@ -8,30 +8,25 @@ set -o nounset
 __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 __root="$(cd "$(dirname "${__dir}")" && pwd)"
 
-
+curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
-echo "Storing ca.crt inside HOME"
+chmod +x ./kubectl
-echo $CA_CRT | base64 --decode -i > ${HOME}/ca.crt
+sudo mv ./kubectl /usr/local/bin/kubectl
-echo "ca.crt is saved"
 
 #Store the new image in docker hub
 docker build --quiet -t tusproject/tusd:latest -t tusproject/tusd:$TRAVIS_COMMIT ${__root};
-docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
+docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD";
 docker push tusproject/tusd:$TRAVIS_COMMIT;
 docker push tusproject/tusd:latest;
 
 
-
+echo "Create directory..."
-
+mkdir ${HOME}/.kube
-gcloud config set container/use_client_certificate True
+echo "Writing KUBECONFIG to file..."
-export CLOUDSDK_CONTAINER_USE_CLIENT_CERTIFICATE=True
+echo $KUBECONFIGVAR | python -m base64 -d > ${HOME}/.kube/config
-
+echo "KUBECONFIG file written"
-kubectl config set-cluster transloadit-gke-cluster --embed-certs=true --server=${CLUSTER_ENDPOINT} --certificate-authority=${HOME}/ca.crt
-kubectl config set-credentials travis --token=$SA_TOKEN
-kubectl config set-context travis --cluster=$CLUSTER_NAME --user=travis --namespace=tus
-kubectl config use-context travis
 
 sleep 10s # This cost me some precious debugging time.
-kubectl apply --validate=false -f "${__root}/.infra/kube/tusd-kube.yaml"
+kubectl apply -f "${__root}/.infra/kube/tusd-kube.yaml"
 
 
 kubectl set image deployment/tusd --namespace=tus tusd=docker.io/tusproject/tusd:$TRAVIS_COMMIT
@@ -43,7 +38,7 @@ kubectl get deployment --namespace=tus
 
 function cleanup {
     printf "Cleaning up...\n"
-    rm -f ${HOME}/ca.crt
+    rm -f ${HOME}/.kube/config
     printf "Cleaning done."
 }
 

.scripts/generate-docker-library.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+#
+# official-images tag file generator
+#
+# usage: ./generate-docker-library.sh > [official-images-folder]/library/tusd
+#
+
+cat <<-EOH
+# This file is generated via https://github.com/tus/tusd/blob/master/generate-docker-library.sh
+Maintainers: tus.io (@tus), Thomas A. Hirsch (@thirsch)
+GitRepo: https://github.com/tus/tusd.git
+EOH
+
+skipBeforeVersion="0.13.0"
+previousVersions=();
+
+function printVersion() {
+    version=( ${1//./ } )
+    majorMinor="${version[0]}.${version[1]}"
+
+    match=$(echo "${previousVersions[@]:0}" | grep -oE "\s?$majorMinor\s?$")
+    previousVersionCount=${#previousVersions[@]}
+
+    # add the majorMinor-Version only, if it is not present yet.
+    if [[ ! -z $match ]] ; then
+        versionString=$1
+    else
+        versionString="$1 $majorMinor"
+        previousVersions+=($majorMinor)
+    fi
+
+    # as the versions are sorted, the very first version gets latest.
+    if [[ ${previousVersionCount} -eq 0 ]]; then
+        versionString="$versionString, latest"
+    fi
+
+  cat <<-EOE
+
+Tags: $versionString
+GitCommit: $2
+	EOE
+}
+
+for version in `git tag -l --sort=-v:refname | grep "^[0-9.]\+$"`; do
+  commit=`git rev-parse ${version}`
+
+  # no official release before this version
+  if [[ ${version} = ${skipBeforeVersion} ]] ; then
+    exit 0
+  fi
+
+  printVersion "${version}" ${commit}
+done

.travis.yml

@@ -22,12 +22,6 @@ script:
 - ./.scripts/test_all.sh
 before_deploy:
 - if [[ "$TRAVIS_TAG" != "" ]]; then ./.scripts/build_all.sh; fi
-- if [ ! -d "$HOME/google-cloud-sdk/bin" ]; then rm -rf $HOME/google-cloud-sdk; curl https://sdk.cloud.google.com | bash; fi
-- source /home/travis/google-cloud-sdk/path.bash.inc
-- gcloud --quiet version
-- gcloud --quiet components update
-- gcloud --quiet components update kubectl
-- curl https://raw.githubusercontent.com/kubernetes/helm/d87ce93e1e287ece84d940dbfe09b0de493d9953/scripts/get | bash
 deploy:
 - provider: releases
   api_key:
@@ -41,7 +35,7 @@ deploy:
     repo: tus/tusd
     os: linux
 - provider: script
-  script: .scripts/deploy_gcloud.sh
+  script: .scripts/deploy_kube.sh
   on:
     branch: master
     go: 1.12

Dockerfile

@@ -7,7 +7,7 @@ COPY . /go/src/github.com/tus/tusd/
 WORKDIR /go/src/github.com/tus/tusd
 
 RUN apk add --no-cache \
-        git \
+        git gcc libc-dev \
     && go get -d -v ./... \
     && version="$(git tag -l --points-at HEAD)" \
     && commit=$(git log --format="%H" -n 1) \
@@ -18,11 +18,11 @@ RUN apk add --no-cache \
     && apk del git
 
 # start a new stage that copies in the binary built in the previous stage
-FROM alpine:3.8
+FROM alpine:3.9
 
 COPY --from=builder /go/bin/tusd /usr/local/bin/tusd
 
-RUN apk add --no-cache ca-certificates jq \
+RUN apk add --no-cache ca-certificates jq gcc \
     && addgroup -g 1000 tusd \
     && adduser -u 1000 -G tusd -s /bin/sh -D tusd \
     && mkdir -p /srv/tusd-hooks \

README.md

@@ -267,11 +267,11 @@ Yes, this is made possible by the [hook system](/docs/hooks.md) inside the tusd
 
 ### Can I run tusd inside a VM/Vagrant/VirtualBox?
 
-Yes, you can absolutely do so without any modifications. However, there is one known problem: If you are using tusd inside VirtualBox (the default provider for Vagrant) and are storing the files inside a shared/synced folder, you might get TemporaryErrors (Lockfile created, but doesn't exist) when trying to upload. This happens because shared folders do not support symbolic links which are necessary for tusd. Please use another non-shared folder for storing files (see https://github.com/tus/tusd/issues/201).
+Yes, you can absolutely do so without any modifications. However, there is one known problem: If you are using tusd inside VirtualBox (the default provider for Vagrant) and are storing the files inside a shared/synced folder, you might get TemporaryErrors (Lockfile created, but doesn't exist) when trying to upload. This happens because shared folders do not support hard links which are necessary for tusd. Please use another non-shared folder for storing files (see https://github.com/tus/tusd/issues/201).
 
 ### I am getting TemporaryErrors (Lockfile created, but doesn't exist)! What can I do?
 
-This error can occur when you are running tusd's disk storage on a file system which does not support symbolic links. These symbolic links are used to create lock files for ensuring that an upload's data is consistent. For example, this problem can happen when running tusd inside VirtualBox (see the answer above for more details) or when using file system interfaces to cloud storage providers (see https://github.com/tus/tusd/issues/257). We recommend you to ensure that your file system supports symbolic links, use a different file system, or use one of tusd's cloud storage abilities. If the problem still persists, please open a bug report.
+This error can occur when you are running tusd's disk storage on a file system which does not support hard links. These hard links are used to create lock files for ensuring that an upload's data is consistent. For example, this problem can happen when running tusd inside VirtualBox (see the answer above for more details) or when using file system interfaces to cloud storage providers (see https://github.com/tus/tusd/issues/257). We recommend you to ensure that your file system supports hard links, use a different file system, or use one of tusd's cloud storage abilities. If the problem still persists, please open a bug report.
 
 ### How can I prevent users from downloading the uploaded files?
 

cmd/tusd/cli/flags.go

@@ -29,6 +29,7 @@ var Flags struct {
 	ExposeMetrics       bool
 	MetricsPath         string
 	BehindProxy         bool
+	VerboseOutput       bool
 
 	FileHooksInstalled bool
 	HttpHooksInstalled bool
@@ -57,6 +58,7 @@ func ParseFlags() {
 	flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage")
 	flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible")
 	flag.BoolVar(&Flags.BehindProxy, "behind-proxy", false, "Respect X-Forwarded-* and similar headers which may be set by proxies")
+	flag.BoolVar(&Flags.VerboseOutput, "verbose", true, "Enable verbose logging output")
 
 	flag.Parse()
 

cmd/tusd/cli/hooks.go

@@ -103,14 +103,16 @@ func invokeHookSync(typ hooks.HookType, info handler.FileInfo, captureOutput boo
 	}
 
 	name := string(typ)
-	logEv(stdout, "HookInvocationStart", "type", name, "id", info.ID)
+	if Flags.VerboseOutput {
+		logEv(stdout, "HookInvocationStart", "type", name, "id", info.ID)
+	}
 
 	output, returnCode, err := hookHandler.InvokeHook(typ, info, captureOutput)
 
 	if err != nil {
 		logEv(stderr, "HookInvocationError", "type", string(typ), "id", info.ID, "error", err.Error())
 		MetricsHookErrorsTotal.WithLabelValues(string(typ)).Add(1)
-	} else {
+	} else if Flags.VerboseOutput {
 		logEv(stdout, "HookInvocationFinish", "type", string(typ), "id", info.ID)
 	}
 

pkg/handler/unrouted_handler.go

@@ -814,6 +814,13 @@ func (handler *UnroutedHandler) sendError(w http.ResponseWriter, r *http.Request
 		err = errors.New("read tcp: i/o timeout")
 	}
 
+	// Errors for connnection resets also contain TCP details, we don't need, e.g:
+	// read tcp 127.0.0.1:1080->127.0.0.1:10023: read: connection reset by peer
+	// Therefore, we also trim those down.
+	if strings.HasSuffix(err.Error(), "read: connection reset by peer") {
+		err = errors.New("read tcp: connection reset by peer")
+	}
+
 	statusErr, ok := err.(HTTPError)
 	if !ok {
 		statusErr = NewHTTPError(err, http.StatusInternalServerError)

pkg/s3store/s3store.go

@@ -90,7 +90,9 @@ import (
 
 // This regular expression matches every character which is not defined in the
 // ASCII tables which range from 00 to 7F, inclusive.
-var nonASCIIRegexp = regexp.MustCompile(`([^\x00-\x7F])`)
+// It also matches the \r and \n characters which are not allowed in values
+// for HTTP headers.
+var nonASCIIRegexp = regexp.MustCompile(`([^\x00-\x7F]|[\r\n])`)
 
 // See the handler.DataStore interface for documentation about the different
 // methods.
@@ -257,8 +259,8 @@ func (store S3Store) WriteChunk(id string, offset int64, src io.Reader) (int64,
 		return 0, err
 	}
 	if incompletePartFile != nil {
-		defer incompletePartFile.Close()
 		defer os.Remove(incompletePartFile.Name())
+		defer incompletePartFile.Close()
 
 		if err := store.deleteIncompletePartForUpload(uploadId); err != nil {
 			return 0, err

pkg/s3store/s3store_test.go

@@ -37,7 +37,7 @@ func TestNewUpload(t *testing.T) {
 	assert.Equal(s3obj, store.Service)
 
 	s1 := "hello"
-	s2 := "men?"
+	s2 := "men???hi"
 
 	gomock.InOrder(
 		s3obj.EXPECT().CreateMultipartUpload(&s3.CreateMultipartUploadInput{
@@ -53,8 +53,8 @@ func TestNewUpload(t *testing.T) {
 		s3obj.EXPECT().PutObject(&s3.PutObjectInput{
 			Bucket:        aws.String("bucket"),
 			Key:           aws.String("uploadId.info"),
-			Body:          bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)),
+			Body:          bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü\r\nhi","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)),
-			ContentLength: aws.Int64(int64(171)),
+			ContentLength: aws.Int64(int64(177)),
 		}),
 	)
 
@@ -63,7 +63,7 @@ func TestNewUpload(t *testing.T) {
 		Size: 500,
 		MetaData: map[string]string{
 			"foo": "hello",
-			"bar": "menü",
+			"bar": "menü\r\nhi",
 		},
 	}