From bae0ffb5e57e02d526965d734d7dc7d766f881b5 Mon Sep 17 00:00:00 2001 From: Tinco Andringa Date: Sun, 17 Oct 2021 23:48:53 +0200 Subject: [PATCH 1/4] gcsstore: Allow object prefix to contain underscores (#495) * Only determine object type based on name after last separator * modify test to keep in mind directory prefixes with underscores in them * update documentation to reflect support of underscores in gcs object prefix --- cmd/tusd/cli/flags.go | 2 +- pkg/gcsstore/gcsservice.go | 6 +++++- pkg/gcsstore/gcsservice_test.go | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/tusd/cli/flags.go b/cmd/tusd/cli/flags.go index 296928f..35aaeec 100644 --- a/cmd/tusd/cli/flags.go +++ b/cmd/tusd/cli/flags.go @@ -74,7 +74,7 @@ func ParseFlags() { flag.BoolVar(&Flags.S3DisableContentHashes, "s3-disable-content-hashes", false, "Disable the calculation of MD5 and SHA256 hashes for the content that gets uploaded to S3 for minimized CPU usage (experimental and may be removed in the future)") flag.BoolVar(&Flags.S3DisableSSL, "s3-disable-ssl", false, "Disable SSL and only use HTTP for communication with S3 (experimental and may be removed in the future)") flag.StringVar(&Flags.GCSBucket, "gcs-bucket", "", "Use Google Cloud Storage with this bucket as storage backend (requires the GCS_SERVICE_ACCOUNT_FILE environment variable to be set)") - flag.StringVar(&Flags.GCSObjectPrefix, "gcs-object-prefix", "", "Prefix for GCS object names (can't contain underscore character)") + flag.StringVar(&Flags.GCSObjectPrefix, "gcs-object-prefix", "", "Prefix for GCS object names") flag.StringVar(&Flags.AzStorage, "azure-storage", "", "Use Azure BlockBlob Storage with this container name as a storage backend (requires the AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY environment variable to be set)") flag.StringVar(&Flags.AzContainerAccessType, "azure-container-access-type", "", "Access type when creating a new container if it does not exist (possible values: blob, container, '')") flag.StringVar(&Flags.AzBlobAccessTier, "azure-blob-access-tier", "", "Blob access tier when uploading new files (possible values: archive, cool, hot, '')") diff --git a/pkg/gcsstore/gcsservice.go b/pkg/gcsstore/gcsservice.go index 4667622..3acbcbd 100644 --- a/pkg/gcsstore/gcsservice.go +++ b/pkg/gcsstore/gcsservice.go @@ -351,7 +351,11 @@ loop: if strings.HasSuffix(objAttrs.Name, "info") { continue } - split := strings.Split(objAttrs.Name, "_") + + fileNameParts := strings.Split(objAttrs.Name, "/") + fileName := fileNameParts[len(fileNameParts)-1] + + split := strings.Split(fileName, "_") // If the object name does not split on "_", we have a composed object. // If the object name splits on "_" in to four pieces we diff --git a/pkg/gcsstore/gcsservice_test.go b/pkg/gcsstore/gcsservice_test.go index eeffe15..2c91430 100644 --- a/pkg/gcsstore/gcsservice_test.go +++ b/pkg/gcsstore/gcsservice_test.go @@ -447,7 +447,7 @@ func TestFilterObject(t *testing.T) { defer gock.Off() resp := googleBucketResponse{[]googleObjectResponse{ - googleObjectResponse{Name: "test-prefix_1"}, + googleObjectResponse{Name: "test_directory/test-prefix_1"}, }} gock.New("https://www.googleapis.com"). From 84faa14987da8c5bd9da9ed2cad16fe1b7941bbb Mon Sep 17 00:00:00 2001 From: Marius Date: Mon, 18 Oct 2021 00:29:13 +0200 Subject: [PATCH 2/4] core: Allow non-tus HEAD requests, Add Length to HEAD responses See https://github.com/tus/tusd/pull/480 Squashed commit of the following: commit 7439fd84a6103afdedaf94701a65ce4376789380 Author: Marius Date: Mon Oct 18 00:27:12 2021 +0200 Docs and test commit 16d9dc67e8c8eefc328b1ce12d7e7ca01a49f9f6 Merge: bae0ffb bea5183 Author: Marius Date: Mon Oct 18 00:23:13 2021 +0200 Merge branch 'head_header_check' of https://github.com/s3rius/tusd into s3rius-head_header_check commit bea5183ec3f8759efadb554b5dd1acd18187d055 Author: Pavel Kirilin Date: Thu May 20 19:53:36 2021 +0400 Fixed "Tus-Resumable" header check for HEAD request. Signed-off-by: Pavel Kirilin --- pkg/handler/head_test.go | 7 ++++--- pkg/handler/unrouted_handler.go | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/handler/head_test.go b/pkg/handler/head_test.go index 59da085..b5c83bb 100644 --- a/pkg/handler/head_test.go +++ b/pkg/handler/head_test.go @@ -48,9 +48,10 @@ func TestHead(t *testing.T) { }, Code: http.StatusOK, ResHeader: map[string]string{ - "Upload-Offset": "11", - "Upload-Length": "44", - "Cache-Control": "no-store", + "Upload-Offset": "11", + "Upload-Length": "44", + "Content-Length": "44", + "Cache-Control": "no-store", }, }).Run(handler, t) diff --git a/pkg/handler/unrouted_handler.go b/pkg/handler/unrouted_handler.go index ba2fb42..6c5f52b 100644 --- a/pkg/handler/unrouted_handler.go +++ b/pkg/handler/unrouted_handler.go @@ -260,9 +260,9 @@ func (handler *UnroutedHandler) Middleware(h http.Handler) http.Handler { } // Test if the version sent by the client is supported - // GET methods are not checked since a browser may visit this URL and does - // not include this header. This request is not part of the specification. - if r.Method != "GET" && r.Header.Get("Tus-Resumable") != "1.0.0" { + // GET and HEAD methods are not checked since a browser may visit this URL and does + // not include this header. GET requests are not part of the specification. + if r.Method != "GET" && r.Method != "HEAD" && r.Header.Get("Tus-Resumable") != "1.0.0" { handler.sendError(w, r, ErrUnsupportedVersion) return } @@ -472,6 +472,7 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request) w.Header().Set("Upload-Defer-Length", UploadLengthDeferred) } else { w.Header().Set("Upload-Length", strconv.FormatInt(info.Size, 10)) + w.Header().Set("Content-Length", strconv.FormatInt(info.Size, 10)) } w.Header().Set("Cache-Control", "no-store") From cd717e580c407a1040e240c3f4bed86a32a43be4 Mon Sep 17 00:00:00 2001 From: Marius Date: Mon, 18 Oct 2021 14:43:37 +0200 Subject: [PATCH 3/4] ci: Tag docker image with tag name See https://github.com/tus/tusd/issues/511 --- .github/workflows/main.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a3d8aca..9a6c395 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,3 +59,18 @@ jobs: run: | docker build -t tusproject/tusd:$GITHUB_SHA . docker push tusproject/tusd:$GITHUB_SHA + - name: Tag docker image with release tag + if: startsWith(github.ref, 'refs/tags/') + run: | + # Extract tag name: https://stackoverflow.com/a/58178121 + TAG_NAME="${GITHUB_REF#refs/*/}" + docker tag tusproject/tusd:$GITHUB_SHA tusproject/tusd:$TAG_NAME + docker push tusproject/tusd:$TAG_NAME + # Create latest tag if we have no pre-release + if [[ $TAG_NAME =~ ^v\d+\.\d+\.\d+$ ]]; + then + docker tag tusproject/tusd:$GITHUB_SHA tusproject/tusd:latest + docker push tusproject/tusd:latest + fi + shell: bash + From 21b3de4c7125635a18b373a09aacbca115d93832 Mon Sep 17 00:00:00 2001 From: Marius Date: Mon, 18 Oct 2021 15:04:48 +0200 Subject: [PATCH 4/4] ci: Replace version test with working regex --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9a6c395..301d317 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -67,7 +67,7 @@ jobs: docker tag tusproject/tusd:$GITHUB_SHA tusproject/tusd:$TAG_NAME docker push tusproject/tusd:$TAG_NAME # Create latest tag if we have no pre-release - if [[ $TAG_NAME =~ ^v\d+\.\d+\.\d+$ ]]; + if [[ $TAG_NAME =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then docker tag tusproject/tusd:$GITHUB_SHA tusproject/tusd:latest docker push tusproject/tusd:latest