Merge branch 'master' of github.com:tus/tusd into v1
This commit is contained in:
Marius
commit
e8fb3a431b
|
|
@ -10,13 +10,6 @@ spec:
|
|||
labels:
|
||||
app: tusd
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: cloud.google.com/gke-preemptible
|
||||
operator: DoesNotExist
|
||||
containers:
|
||||
- image: docker.io/tusproject/tusd:latest
|
||||
imagePullPolicy: Always
|
||||
|
|
@ -24,27 +17,15 @@ spec:
|
|||
name: tusd
|
||||
resources:
|
||||
limits:
|
||||
memory: "2Gi"
|
||||
# requests:
|
||||
# memory: "1Gi"
|
||||
memory: "1Gi"
|
||||
requests:
|
||||
memory: "1Gi"
|
||||
ports:
|
||||
- name: tusd-web
|
||||
containerPort: 8080
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: tusd-env
|
||||
- secretRef:
|
||||
name: tusd-s3
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
fsGroup: 0
|
||||
volumeMounts:
|
||||
- name: tusd-account
|
||||
mountPath: /gcs
|
||||
volumes:
|
||||
- name: tusd-account
|
||||
secret:
|
||||
secretName: gcs-account
|
||||
name: tusd-env
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
|
@ -75,6 +56,7 @@ metadata:
|
|||
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
|
||||
nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
|
||||
nginx.ingress.kubernetes.io/ssl-redirect: "true"
|
||||
spec:
|
||||
tls:
|
||||
- hosts:
|
||||
|
|
@ -98,25 +80,3 @@ spec:
|
|||
backend:
|
||||
serviceName: tusd
|
||||
servicePort: 80
|
||||
---
|
||||
apiVersion: autoscaling/v1
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: tusd
|
||||
namespace: tus
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1beta1
|
||||
kind: Deployment
|
||||
name: tusd
|
||||
minReplicas: 1
|
||||
maxReplicas: 5
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
targetAverageUtilization: 80
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
targetAverageValue: 1800Mi
|
||||
|
|
|
|||
|
|
@ -11,14 +11,14 @@ compile linux amd64
|
|||
compile linux arm
|
||||
compile darwin 386
|
||||
compile darwin amd64
|
||||
compile windows 386 .exe
|
||||
compile windows amd64 .exe
|
||||
#compile windows 386 .exe
|
||||
#compile windows amd64 .exe
|
||||
|
||||
maketar linux 386
|
||||
maketar linux amd64
|
||||
maketar linux arm
|
||||
makezip darwin 386
|
||||
makezip darwin amd64
|
||||
makezip windows 386 .exe
|
||||
makezip windows amd64 .exe
|
||||
#makezip windows 386 .exe
|
||||
#makezip windows amd64 .exe
|
||||
makedep amd64
|
||||
|
|
|
|||
|
|
@ -8,30 +8,25 @@ set -o nounset
|
|||
__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
__root="$(cd "$(dirname "${__dir}")" && pwd)"
|
||||
|
||||
|
||||
echo "Storing ca.crt inside HOME"
|
||||
echo $CA_CRT | base64 --decode -i > ${HOME}/ca.crt
|
||||
echo "ca.crt is saved"
|
||||
curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
|
||||
chmod +x ./kubectl
|
||||
sudo mv ./kubectl /usr/local/bin/kubectl
|
||||
|
||||
#Store the new image in docker hub
|
||||
docker build --quiet -t tusproject/tusd:latest -t tusproject/tusd:$TRAVIS_COMMIT ${__root};
|
||||
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
|
||||
docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD";
|
||||
docker push tusproject/tusd:$TRAVIS_COMMIT;
|
||||
docker push tusproject/tusd:latest;
|
||||
|
||||
|
||||
|
||||
|
||||
gcloud config set container/use_client_certificate True
|
||||
export CLOUDSDK_CONTAINER_USE_CLIENT_CERTIFICATE=True
|
||||
|
||||
kubectl config set-cluster transloadit-gke-cluster --embed-certs=true --server=${CLUSTER_ENDPOINT} --certificate-authority=${HOME}/ca.crt
|
||||
kubectl config set-credentials travis --token=$SA_TOKEN
|
||||
kubectl config set-context travis --cluster=$CLUSTER_NAME --user=travis --namespace=tus
|
||||
kubectl config use-context travis
|
||||
echo "Create directory..."
|
||||
mkdir ${HOME}/.kube
|
||||
echo "Writing KUBECONFIG to file..."
|
||||
echo $KUBECONFIGVAR | python -m base64 -d > ${HOME}/.kube/config
|
||||
echo "KUBECONFIG file written"
|
||||
|
||||
sleep 10s # This cost me some precious debugging time.
|
||||
kubectl apply --validate=false -f "${__root}/.infra/kube/tusd-kube.yaml"
|
||||
kubectl apply -f "${__root}/.infra/kube/tusd-kube.yaml"
|
||||
|
||||
|
||||
kubectl set image deployment/tusd --namespace=tus tusd=docker.io/tusproject/tusd:$TRAVIS_COMMIT
|
||||
|
|
@ -43,7 +38,7 @@ kubectl get deployment --namespace=tus
|
|||
|
||||
function cleanup {
|
||||
printf "Cleaning up...\n"
|
||||
rm -f ${HOME}/ca.crt
|
||||
rm -f ${HOME}/.kube/config
|
||||
printf "Cleaning done."
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
#
|
||||
# official-images tag file generator
|
||||
#
|
||||
# usage: ./generate-docker-library.sh > [official-images-folder]/library/tusd
|
||||
#
|
||||
|
||||
cat <<-EOH
|
||||
# This file is generated via https://github.com/tus/tusd/blob/master/generate-docker-library.sh
|
||||
Maintainers: tus.io (@tus), Thomas A. Hirsch (@thirsch)
|
||||
GitRepo: https://github.com/tus/tusd.git
|
||||
EOH
|
||||
|
||||
skipBeforeVersion="0.13.0"
|
||||
previousVersions=();
|
||||
|
||||
function printVersion() {
|
||||
version=( ${1//./ } )
|
||||
majorMinor="${version[0]}.${version[1]}"
|
||||
|
||||
match=$(echo "${previousVersions[@]:0}" | grep -oE "\s?$majorMinor\s?$")
|
||||
previousVersionCount=${#previousVersions[@]}
|
||||
|
||||
# add the majorMinor-Version only, if it is not present yet.
|
||||
if [[ ! -z $match ]] ; then
|
||||
versionString=$1
|
||||
else
|
||||
versionString="$1 $majorMinor"
|
||||
previousVersions+=($majorMinor)
|
||||
fi
|
||||
|
||||
# as the versions are sorted, the very first version gets latest.
|
||||
if [[ ${previousVersionCount} -eq 0 ]]; then
|
||||
versionString="$versionString, latest"
|
||||
fi
|
||||
|
||||
cat <<-EOE
|
||||
|
||||
Tags: $versionString
|
||||
GitCommit: $2
|
||||
EOE
|
||||
}
|
||||
|
||||
for version in `git tag -l --sort=-v:refname | grep "^[0-9.]\+$"`; do
|
||||
commit=`git rev-parse ${version}`
|
||||
|
||||
# no official release before this version
|
||||
if [[ ${version} = ${skipBeforeVersion} ]] ; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
printVersion "${version}" ${commit}
|
||||
done
|
||||
|
|
@ -22,12 +22,6 @@ script:
|
|||
- ./.scripts/test_all.sh
|
||||
before_deploy:
|
||||
- if [[ "$TRAVIS_TAG" != "" ]]; then ./.scripts/build_all.sh; fi
|
||||
- if [ ! -d "$HOME/google-cloud-sdk/bin" ]; then rm -rf $HOME/google-cloud-sdk; curl https://sdk.cloud.google.com | bash; fi
|
||||
- source /home/travis/google-cloud-sdk/path.bash.inc
|
||||
- gcloud --quiet version
|
||||
- gcloud --quiet components update
|
||||
- gcloud --quiet components update kubectl
|
||||
- curl https://raw.githubusercontent.com/kubernetes/helm/d87ce93e1e287ece84d940dbfe09b0de493d9953/scripts/get | bash
|
||||
deploy:
|
||||
- provider: releases
|
||||
api_key:
|
||||
|
|
@ -41,7 +35,7 @@ deploy:
|
|||
repo: tus/tusd
|
||||
os: linux
|
||||
- provider: script
|
||||
script: .scripts/deploy_gcloud.sh
|
||||
script: .scripts/deploy_kube.sh
|
||||
on:
|
||||
branch: master
|
||||
go: 1.12
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ COPY . /go/src/github.com/tus/tusd/
|
|||
WORKDIR /go/src/github.com/tus/tusd
|
||||
|
||||
RUN apk add --no-cache \
|
||||
git \
|
||||
git gcc libc-dev \
|
||||
&& go get -d -v ./... \
|
||||
&& version="$(git tag -l --points-at HEAD)" \
|
||||
&& commit=$(git log --format="%H" -n 1) \
|
||||
|
|
@ -18,11 +18,11 @@ RUN apk add --no-cache \
|
|||
&& apk del git
|
||||
|
||||
# start a new stage that copies in the binary built in the previous stage
|
||||
FROM alpine:3.8
|
||||
FROM alpine:3.9
|
||||
|
||||
COPY --from=builder /go/bin/tusd /usr/local/bin/tusd
|
||||
|
||||
RUN apk add --no-cache ca-certificates jq \
|
||||
RUN apk add --no-cache ca-certificates jq gcc \
|
||||
&& addgroup -g 1000 tusd \
|
||||
&& adduser -u 1000 -G tusd -s /bin/sh -D tusd \
|
||||
&& mkdir -p /srv/tusd-hooks \
|
||||
|
|
|
|||
|
|
@ -267,11 +267,11 @@ Yes, this is made possible by the [hook system](/docs/hooks.md) inside the tusd
|
|||
|
||||
### Can I run tusd inside a VM/Vagrant/VirtualBox?
|
||||
|
||||
Yes, you can absolutely do so without any modifications. However, there is one known problem: If you are using tusd inside VirtualBox (the default provider for Vagrant) and are storing the files inside a shared/synced folder, you might get TemporaryErrors (Lockfile created, but doesn't exist) when trying to upload. This happens because shared folders do not support symbolic links which are necessary for tusd. Please use another non-shared folder for storing files (see https://github.com/tus/tusd/issues/201).
|
||||
Yes, you can absolutely do so without any modifications. However, there is one known problem: If you are using tusd inside VirtualBox (the default provider for Vagrant) and are storing the files inside a shared/synced folder, you might get TemporaryErrors (Lockfile created, but doesn't exist) when trying to upload. This happens because shared folders do not support hard links which are necessary for tusd. Please use another non-shared folder for storing files (see https://github.com/tus/tusd/issues/201).
|
||||
|
||||
### I am getting TemporaryErrors (Lockfile created, but doesn't exist)! What can I do?
|
||||
|
||||
This error can occur when you are running tusd's disk storage on a file system which does not support symbolic links. These symbolic links are used to create lock files for ensuring that an upload's data is consistent. For example, this problem can happen when running tusd inside VirtualBox (see the answer above for more details) or when using file system interfaces to cloud storage providers (see https://github.com/tus/tusd/issues/257). We recommend you to ensure that your file system supports symbolic links, use a different file system, or use one of tusd's cloud storage abilities. If the problem still persists, please open a bug report.
|
||||
This error can occur when you are running tusd's disk storage on a file system which does not support hard links. These hard links are used to create lock files for ensuring that an upload's data is consistent. For example, this problem can happen when running tusd inside VirtualBox (see the answer above for more details) or when using file system interfaces to cloud storage providers (see https://github.com/tus/tusd/issues/257). We recommend you to ensure that your file system supports hard links, use a different file system, or use one of tusd's cloud storage abilities. If the problem still persists, please open a bug report.
|
||||
|
||||
### How can I prevent users from downloading the uploaded files?
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ var Flags struct {
|
|||
ExposeMetrics bool
|
||||
MetricsPath string
|
||||
BehindProxy bool
|
||||
VerboseOutput bool
|
||||
|
||||
FileHooksInstalled bool
|
||||
HttpHooksInstalled bool
|
||||
|
|
@ -57,6 +58,7 @@ func ParseFlags() {
|
|||
flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage")
|
||||
flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible")
|
||||
flag.BoolVar(&Flags.BehindProxy, "behind-proxy", false, "Respect X-Forwarded-* and similar headers which may be set by proxies")
|
||||
flag.BoolVar(&Flags.VerboseOutput, "verbose", true, "Enable verbose logging output")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
|
|
|
|||
|
|
@ -103,14 +103,16 @@ func invokeHookSync(typ hooks.HookType, info handler.FileInfo, captureOutput boo
|
|||
}
|
||||
|
||||
name := string(typ)
|
||||
if Flags.VerboseOutput {
|
||||
logEv(stdout, "HookInvocationStart", "type", name, "id", info.ID)
|
||||
}
|
||||
|
||||
output, returnCode, err := hookHandler.InvokeHook(typ, info, captureOutput)
|
||||
|
||||
if err != nil {
|
||||
logEv(stderr, "HookInvocationError", "type", string(typ), "id", info.ID, "error", err.Error())
|
||||
MetricsHookErrorsTotal.WithLabelValues(string(typ)).Add(1)
|
||||
} else {
|
||||
} else if Flags.VerboseOutput {
|
||||
logEv(stdout, "HookInvocationFinish", "type", string(typ), "id", info.ID)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -814,6 +814,13 @@ func (handler *UnroutedHandler) sendError(w http.ResponseWriter, r *http.Request
|
|||
err = errors.New("read tcp: i/o timeout")
|
||||
}
|
||||
|
||||
// Errors for connnection resets also contain TCP details, we don't need, e.g:
|
||||
// read tcp 127.0.0.1:1080->127.0.0.1:10023: read: connection reset by peer
|
||||
// Therefore, we also trim those down.
|
||||
if strings.HasSuffix(err.Error(), "read: connection reset by peer") {
|
||||
err = errors.New("read tcp: connection reset by peer")
|
||||
}
|
||||
|
||||
statusErr, ok := err.(HTTPError)
|
||||
if !ok {
|
||||
statusErr = NewHTTPError(err, http.StatusInternalServerError)
|
||||
|
|
|
|||
|
|
@ -90,7 +90,9 @@ import (
|
|||
|
||||
// This regular expression matches every character which is not defined in the
|
||||
// ASCII tables which range from 00 to 7F, inclusive.
|
||||
var nonASCIIRegexp = regexp.MustCompile(`([^\x00-\x7F])`)
|
||||
// It also matches the \r and \n characters which are not allowed in values
|
||||
// for HTTP headers.
|
||||
var nonASCIIRegexp = regexp.MustCompile(`([^\x00-\x7F]|[\r\n])`)
|
||||
|
||||
// See the handler.DataStore interface for documentation about the different
|
||||
// methods.
|
||||
|
|
@ -257,8 +259,8 @@ func (store S3Store) WriteChunk(id string, offset int64, src io.Reader) (int64,
|
|||
return 0, err
|
||||
}
|
||||
if incompletePartFile != nil {
|
||||
defer incompletePartFile.Close()
|
||||
defer os.Remove(incompletePartFile.Name())
|
||||
defer incompletePartFile.Close()
|
||||
|
||||
if err := store.deleteIncompletePartForUpload(uploadId); err != nil {
|
||||
return 0, err
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ func TestNewUpload(t *testing.T) {
|
|||
assert.Equal(s3obj, store.Service)
|
||||
|
||||
s1 := "hello"
|
||||
s2 := "men?"
|
||||
s2 := "men???hi"
|
||||
|
||||
gomock.InOrder(
|
||||
s3obj.EXPECT().CreateMultipartUpload(&s3.CreateMultipartUploadInput{
|
||||
|
|
@ -53,8 +53,8 @@ func TestNewUpload(t *testing.T) {
|
|||
s3obj.EXPECT().PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("uploadId.info"),
|
||||
Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)),
|
||||
ContentLength: aws.Int64(int64(171)),
|
||||
Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü\r\nhi","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)),
|
||||
ContentLength: aws.Int64(int64(177)),
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -63,7 +63,7 @@ func TestNewUpload(t *testing.T) {
|
|||
Size: 500,
|
||||
MetaData: map[string]string{
|
||||
"foo": "hello",
|
||||
"bar": "menü",
|
||||
"bar": "menü\r\nhi",
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue