Merge branch 'master' of github.com:tus/tusd into v1

This commit is contained in:
Marius 2019-08-18 14:34:37 +02:00
commit e8fb3a431b
12 changed files with 101 additions and 85 deletions

View File

@ -10,13 +10,6 @@ spec:
labels: labels:
app: tusd app: tusd
spec: spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: cloud.google.com/gke-preemptible
operator: DoesNotExist
containers: containers:
- image: docker.io/tusproject/tusd:latest - image: docker.io/tusproject/tusd:latest
imagePullPolicy: Always imagePullPolicy: Always
@ -24,27 +17,15 @@ spec:
name: tusd name: tusd
resources: resources:
limits: limits:
memory: "2Gi" memory: "1Gi"
# requests: requests:
# memory: "1Gi" memory: "1Gi"
ports: ports:
- name: tusd-web - name: tusd-web
containerPort: 8080 containerPort: 8080
envFrom: envFrom:
- configMapRef:
name: tusd-env
- secretRef: - secretRef:
name: tusd-s3 name: tusd-env
securityContext:
runAsUser: 0
fsGroup: 0
volumeMounts:
- name: tusd-account
mountPath: /gcs
volumes:
- name: tusd-account
secret:
secretName: gcs-account
--- ---
apiVersion: v1 apiVersion: v1
kind: Service kind: Service
@ -75,6 +56,7 @@ metadata:
nginx.ingress.kubernetes.io/proxy-read-timeout: "300" nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-request-buffering: "off" nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300" nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/ssl-redirect: "true"
spec: spec:
tls: tls:
- hosts: - hosts:
@ -98,25 +80,3 @@ spec:
backend: backend:
serviceName: tusd serviceName: tusd
servicePort: 80 servicePort: 80
---
apiVersion: autoscaling/v1
kind: HorizontalPodAutoscaler
metadata:
name: tusd
namespace: tus
spec:
scaleTargetRef:
apiVersion: apps/v1beta1
kind: Deployment
name: tusd
minReplicas: 1
maxReplicas: 5
metrics:
- type: Resource
resource:
name: cpu
targetAverageUtilization: 80
- type: Resource
resource:
name: memory
targetAverageValue: 1800Mi

View File

@ -11,14 +11,14 @@ compile linux amd64
compile linux arm compile linux arm
compile darwin 386 compile darwin 386
compile darwin amd64 compile darwin amd64
compile windows 386 .exe #compile windows 386 .exe
compile windows amd64 .exe #compile windows amd64 .exe
maketar linux 386 maketar linux 386
maketar linux amd64 maketar linux amd64
maketar linux arm maketar linux arm
makezip darwin 386 makezip darwin 386
makezip darwin amd64 makezip darwin amd64
makezip windows 386 .exe #makezip windows 386 .exe
makezip windows amd64 .exe #makezip windows amd64 .exe
makedep amd64 makedep amd64

View File

@ -8,30 +8,25 @@ set -o nounset
__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
__root="$(cd "$(dirname "${__dir}")" && pwd)" __root="$(cd "$(dirname "${__dir}")" && pwd)"
curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
echo "Storing ca.crt inside HOME" chmod +x ./kubectl
echo $CA_CRT | base64 --decode -i > ${HOME}/ca.crt sudo mv ./kubectl /usr/local/bin/kubectl
echo "ca.crt is saved"
#Store the new image in docker hub #Store the new image in docker hub
docker build --quiet -t tusproject/tusd:latest -t tusproject/tusd:$TRAVIS_COMMIT ${__root}; docker build --quiet -t tusproject/tusd:latest -t tusproject/tusd:$TRAVIS_COMMIT ${__root};
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"; docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD";
docker push tusproject/tusd:$TRAVIS_COMMIT; docker push tusproject/tusd:$TRAVIS_COMMIT;
docker push tusproject/tusd:latest; docker push tusproject/tusd:latest;
echo "Create directory..."
mkdir ${HOME}/.kube
gcloud config set container/use_client_certificate True echo "Writing KUBECONFIG to file..."
export CLOUDSDK_CONTAINER_USE_CLIENT_CERTIFICATE=True echo $KUBECONFIGVAR | python -m base64 -d > ${HOME}/.kube/config
echo "KUBECONFIG file written"
kubectl config set-cluster transloadit-gke-cluster --embed-certs=true --server=${CLUSTER_ENDPOINT} --certificate-authority=${HOME}/ca.crt
kubectl config set-credentials travis --token=$SA_TOKEN
kubectl config set-context travis --cluster=$CLUSTER_NAME --user=travis --namespace=tus
kubectl config use-context travis
sleep 10s # This cost me some precious debugging time. sleep 10s # This cost me some precious debugging time.
kubectl apply --validate=false -f "${__root}/.infra/kube/tusd-kube.yaml" kubectl apply -f "${__root}/.infra/kube/tusd-kube.yaml"
kubectl set image deployment/tusd --namespace=tus tusd=docker.io/tusproject/tusd:$TRAVIS_COMMIT kubectl set image deployment/tusd --namespace=tus tusd=docker.io/tusproject/tusd:$TRAVIS_COMMIT
@ -43,7 +38,7 @@ kubectl get deployment --namespace=tus
function cleanup { function cleanup {
printf "Cleaning up...\n" printf "Cleaning up...\n"
rm -f ${HOME}/ca.crt rm -f ${HOME}/.kube/config
printf "Cleaning done." printf "Cleaning done."
} }

View File

@ -0,0 +1,54 @@
#!/usr/bin/env bash
#
# official-images tag file generator
#
# usage: ./generate-docker-library.sh > [official-images-folder]/library/tusd
#
cat <<-EOH
# This file is generated via https://github.com/tus/tusd/blob/master/generate-docker-library.sh
Maintainers: tus.io (@tus), Thomas A. Hirsch (@thirsch)
GitRepo: https://github.com/tus/tusd.git
EOH
skipBeforeVersion="0.13.0"
previousVersions=();
function printVersion() {
version=( ${1//./ } )
majorMinor="${version[0]}.${version[1]}"
match=$(echo "${previousVersions[@]:0}" | grep -oE "\s?$majorMinor\s?$")
previousVersionCount=${#previousVersions[@]}
# add the majorMinor-Version only, if it is not present yet.
if [[ ! -z $match ]] ; then
versionString=$1
else
versionString="$1 $majorMinor"
previousVersions+=($majorMinor)
fi
# as the versions are sorted, the very first version gets latest.
if [[ ${previousVersionCount} -eq 0 ]]; then
versionString="$versionString, latest"
fi
cat <<-EOE
Tags: $versionString
GitCommit: $2
EOE
}
for version in `git tag -l --sort=-v:refname | grep "^[0-9.]\+$"`; do
commit=`git rev-parse ${version}`
# no official release before this version
if [[ ${version} = ${skipBeforeVersion} ]] ; then
exit 0
fi
printVersion "${version}" ${commit}
done

View File

@ -22,12 +22,6 @@ script:
- ./.scripts/test_all.sh - ./.scripts/test_all.sh
before_deploy: before_deploy:
- if [[ "$TRAVIS_TAG" != "" ]]; then ./.scripts/build_all.sh; fi - if [[ "$TRAVIS_TAG" != "" ]]; then ./.scripts/build_all.sh; fi
- if [ ! -d "$HOME/google-cloud-sdk/bin" ]; then rm -rf $HOME/google-cloud-sdk; curl https://sdk.cloud.google.com | bash; fi
- source /home/travis/google-cloud-sdk/path.bash.inc
- gcloud --quiet version
- gcloud --quiet components update
- gcloud --quiet components update kubectl
- curl https://raw.githubusercontent.com/kubernetes/helm/d87ce93e1e287ece84d940dbfe09b0de493d9953/scripts/get | bash
deploy: deploy:
- provider: releases - provider: releases
api_key: api_key:
@ -41,7 +35,7 @@ deploy:
repo: tus/tusd repo: tus/tusd
os: linux os: linux
- provider: script - provider: script
script: .scripts/deploy_gcloud.sh script: .scripts/deploy_kube.sh
on: on:
branch: master branch: master
go: 1.12 go: 1.12

View File

@ -7,7 +7,7 @@ COPY . /go/src/github.com/tus/tusd/
WORKDIR /go/src/github.com/tus/tusd WORKDIR /go/src/github.com/tus/tusd
RUN apk add --no-cache \ RUN apk add --no-cache \
git \ git gcc libc-dev \
&& go get -d -v ./... \ && go get -d -v ./... \
&& version="$(git tag -l --points-at HEAD)" \ && version="$(git tag -l --points-at HEAD)" \
&& commit=$(git log --format="%H" -n 1) \ && commit=$(git log --format="%H" -n 1) \
@ -18,11 +18,11 @@ RUN apk add --no-cache \
&& apk del git && apk del git
# start a new stage that copies in the binary built in the previous stage # start a new stage that copies in the binary built in the previous stage
FROM alpine:3.8 FROM alpine:3.9
COPY --from=builder /go/bin/tusd /usr/local/bin/tusd COPY --from=builder /go/bin/tusd /usr/local/bin/tusd
RUN apk add --no-cache ca-certificates jq \ RUN apk add --no-cache ca-certificates jq gcc \
&& addgroup -g 1000 tusd \ && addgroup -g 1000 tusd \
&& adduser -u 1000 -G tusd -s /bin/sh -D tusd \ && adduser -u 1000 -G tusd -s /bin/sh -D tusd \
&& mkdir -p /srv/tusd-hooks \ && mkdir -p /srv/tusd-hooks \

View File

@ -267,11 +267,11 @@ Yes, this is made possible by the [hook system](/docs/hooks.md) inside the tusd
### Can I run tusd inside a VM/Vagrant/VirtualBox? ### Can I run tusd inside a VM/Vagrant/VirtualBox?
Yes, you can absolutely do so without any modifications. However, there is one known problem: If you are using tusd inside VirtualBox (the default provider for Vagrant) and are storing the files inside a shared/synced folder, you might get TemporaryErrors (Lockfile created, but doesn't exist) when trying to upload. This happens because shared folders do not support symbolic links which are necessary for tusd. Please use another non-shared folder for storing files (see https://github.com/tus/tusd/issues/201). Yes, you can absolutely do so without any modifications. However, there is one known problem: If you are using tusd inside VirtualBox (the default provider for Vagrant) and are storing the files inside a shared/synced folder, you might get TemporaryErrors (Lockfile created, but doesn't exist) when trying to upload. This happens because shared folders do not support hard links which are necessary for tusd. Please use another non-shared folder for storing files (see https://github.com/tus/tusd/issues/201).
### I am getting TemporaryErrors (Lockfile created, but doesn't exist)! What can I do? ### I am getting TemporaryErrors (Lockfile created, but doesn't exist)! What can I do?
This error can occur when you are running tusd's disk storage on a file system which does not support symbolic links. These symbolic links are used to create lock files for ensuring that an upload's data is consistent. For example, this problem can happen when running tusd inside VirtualBox (see the answer above for more details) or when using file system interfaces to cloud storage providers (see https://github.com/tus/tusd/issues/257). We recommend you to ensure that your file system supports symbolic links, use a different file system, or use one of tusd's cloud storage abilities. If the problem still persists, please open a bug report. This error can occur when you are running tusd's disk storage on a file system which does not support hard links. These hard links are used to create lock files for ensuring that an upload's data is consistent. For example, this problem can happen when running tusd inside VirtualBox (see the answer above for more details) or when using file system interfaces to cloud storage providers (see https://github.com/tus/tusd/issues/257). We recommend you to ensure that your file system supports hard links, use a different file system, or use one of tusd's cloud storage abilities. If the problem still persists, please open a bug report.
### How can I prevent users from downloading the uploaded files? ### How can I prevent users from downloading the uploaded files?

View File

@ -29,6 +29,7 @@ var Flags struct {
ExposeMetrics bool ExposeMetrics bool
MetricsPath string MetricsPath string
BehindProxy bool BehindProxy bool
VerboseOutput bool
FileHooksInstalled bool FileHooksInstalled bool
HttpHooksInstalled bool HttpHooksInstalled bool
@ -57,6 +58,7 @@ func ParseFlags() {
flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage") flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage")
flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible") flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible")
flag.BoolVar(&Flags.BehindProxy, "behind-proxy", false, "Respect X-Forwarded-* and similar headers which may be set by proxies") flag.BoolVar(&Flags.BehindProxy, "behind-proxy", false, "Respect X-Forwarded-* and similar headers which may be set by proxies")
flag.BoolVar(&Flags.VerboseOutput, "verbose", true, "Enable verbose logging output")
flag.Parse() flag.Parse()

View File

@ -103,14 +103,16 @@ func invokeHookSync(typ hooks.HookType, info handler.FileInfo, captureOutput boo
} }
name := string(typ) name := string(typ)
if Flags.VerboseOutput {
logEv(stdout, "HookInvocationStart", "type", name, "id", info.ID) logEv(stdout, "HookInvocationStart", "type", name, "id", info.ID)
}
output, returnCode, err := hookHandler.InvokeHook(typ, info, captureOutput) output, returnCode, err := hookHandler.InvokeHook(typ, info, captureOutput)
if err != nil { if err != nil {
logEv(stderr, "HookInvocationError", "type", string(typ), "id", info.ID, "error", err.Error()) logEv(stderr, "HookInvocationError", "type", string(typ), "id", info.ID, "error", err.Error())
MetricsHookErrorsTotal.WithLabelValues(string(typ)).Add(1) MetricsHookErrorsTotal.WithLabelValues(string(typ)).Add(1)
} else { } else if Flags.VerboseOutput {
logEv(stdout, "HookInvocationFinish", "type", string(typ), "id", info.ID) logEv(stdout, "HookInvocationFinish", "type", string(typ), "id", info.ID)
} }

View File

@ -814,6 +814,13 @@ func (handler *UnroutedHandler) sendError(w http.ResponseWriter, r *http.Request
err = errors.New("read tcp: i/o timeout") err = errors.New("read tcp: i/o timeout")
} }
// Errors for connnection resets also contain TCP details, we don't need, e.g:
// read tcp 127.0.0.1:1080->127.0.0.1:10023: read: connection reset by peer
// Therefore, we also trim those down.
if strings.HasSuffix(err.Error(), "read: connection reset by peer") {
err = errors.New("read tcp: connection reset by peer")
}
statusErr, ok := err.(HTTPError) statusErr, ok := err.(HTTPError)
if !ok { if !ok {
statusErr = NewHTTPError(err, http.StatusInternalServerError) statusErr = NewHTTPError(err, http.StatusInternalServerError)

View File

@ -90,7 +90,9 @@ import (
// This regular expression matches every character which is not defined in the // This regular expression matches every character which is not defined in the
// ASCII tables which range from 00 to 7F, inclusive. // ASCII tables which range from 00 to 7F, inclusive.
var nonASCIIRegexp = regexp.MustCompile(`([^\x00-\x7F])`) // It also matches the \r and \n characters which are not allowed in values
// for HTTP headers.
var nonASCIIRegexp = regexp.MustCompile(`([^\x00-\x7F]|[\r\n])`)
// See the handler.DataStore interface for documentation about the different // See the handler.DataStore interface for documentation about the different
// methods. // methods.
@ -257,8 +259,8 @@ func (store S3Store) WriteChunk(id string, offset int64, src io.Reader) (int64,
return 0, err return 0, err
} }
if incompletePartFile != nil { if incompletePartFile != nil {
defer incompletePartFile.Close()
defer os.Remove(incompletePartFile.Name()) defer os.Remove(incompletePartFile.Name())
defer incompletePartFile.Close()
if err := store.deleteIncompletePartForUpload(uploadId); err != nil { if err := store.deleteIncompletePartForUpload(uploadId); err != nil {
return 0, err return 0, err

View File

@ -37,7 +37,7 @@ func TestNewUpload(t *testing.T) {
assert.Equal(s3obj, store.Service) assert.Equal(s3obj, store.Service)
s1 := "hello" s1 := "hello"
s2 := "men?" s2 := "men???hi"
gomock.InOrder( gomock.InOrder(
s3obj.EXPECT().CreateMultipartUpload(&s3.CreateMultipartUploadInput{ s3obj.EXPECT().CreateMultipartUpload(&s3.CreateMultipartUploadInput{
@ -53,8 +53,8 @@ func TestNewUpload(t *testing.T) {
s3obj.EXPECT().PutObject(&s3.PutObjectInput{ s3obj.EXPECT().PutObject(&s3.PutObjectInput{
Bucket: aws.String("bucket"), Bucket: aws.String("bucket"),
Key: aws.String("uploadId.info"), Key: aws.String("uploadId.info"),
Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)), Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü\r\nhi","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)),
ContentLength: aws.Int64(int64(171)), ContentLength: aws.Int64(int64(177)),
}), }),
) )
@ -63,7 +63,7 @@ func TestNewUpload(t *testing.T) {
Size: 500, Size: 500,
MetaData: map[string]string{ MetaData: map[string]string{
"foo": "hello", "foo": "hello",
"bar": "menü", "bar": "menü\r\nhi",
}, },
} }