From 53b86abc354d9a8fb8f7407529cffc2185892261 Mon Sep 17 00:00:00 2001 From: ridhozhr Date: Sun, 2 Jun 2019 15:55:41 +0200 Subject: [PATCH] gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store --- cmd/tusd/cli/composer.go | 1 + cmd/tusd/cli/flags.go | 49 ++++++++++++++++++++++----------------- gcsstore/gcsstore.go | 38 ++++++++++++++++++++---------- gcsstore/gcsstore_test.go | 32 ++++++++++++++++++++++++- 4 files changed, 86 insertions(+), 34 deletions(-) diff --git a/cmd/tusd/cli/composer.go b/cmd/tusd/cli/composer.go index 99eb962..f8c64ce 100644 --- a/cmd/tusd/cli/composer.go +++ b/cmd/tusd/cli/composer.go @@ -56,6 +56,7 @@ func CreateComposer() { stdout.Printf("Using 'gcs://%s' as GCS bucket for storage.\n", Flags.GCSBucket) store := gcsstore.New(Flags.GCSBucket, service) + store.ObjectPrefix = Flags.GCSObjectPrefix store.UseIn(Composer) locker := memorylocker.New() diff --git a/cmd/tusd/cli/flags.go b/cmd/tusd/cli/flags.go index 5bb82de..a26b325 100644 --- a/cmd/tusd/cli/flags.go +++ b/cmd/tusd/cli/flags.go @@ -3,30 +3,31 @@ package cli import ( "flag" "path/filepath" + "strings" ) var Flags struct { - HttpHost string - HttpPort string - HttpSock string - MaxSize int64 - UploadDir string - StoreSize int64 - Basepath string - Timeout int64 - S3Bucket string - S3ObjectPrefix string - S3Endpoint string - GCSBucket string - FileHooksDir string - HttpHooksEndpoint string - HttpHooksRetry int - HttpHooksBackoff int - HooksStopUploadCode int - ShowVersion bool - ExposeMetrics bool - MetricsPath string - BehindProxy bool + HttpHost string + HttpPort string + HttpSock string + MaxSize int64 + UploadDir string + StoreSize int64 + Basepath string + Timeout int64 + S3Bucket string + S3ObjectPrefix string + S3Endpoint string + GCSBucket string + GCSObjectPrefix string + FileHooksDir string + HttpHooksEndpoint string + HttpHooksRetry int + HttpHooksBackoff int + ShowVersion bool + ExposeMetrics bool + MetricsPath string + BehindProxy bool FileHooksInstalled bool HttpHooksInstalled bool @@ -45,6 +46,7 @@ func ParseFlags() { flag.StringVar(&Flags.S3ObjectPrefix, "s3-object-prefix", "", "Prefix for S3 object names") flag.StringVar(&Flags.S3Endpoint, "s3-endpoint", "", "Endpoint to use S3 compatible implementations like minio (requires s3-bucket to be pass)") flag.StringVar(&Flags.GCSBucket, "gcs-bucket", "", "Use Google Cloud Storage with this bucket as storage backend (requires the GCS_SERVICE_ACCOUNT_FILE environment variable to be set)") + flag.StringVar(&Flags.GCSObjectPrefix, "gcs-object-prefix", "", "Prefix for GCS object names (can't contain underscore character)") flag.StringVar(&Flags.FileHooksDir, "hooks-dir", "", "Directory to search for available hooks scripts") flag.StringVar(&Flags.HttpHooksEndpoint, "hooks-http", "", "An HTTP endpoint to which hook events will be sent to") flag.IntVar(&Flags.HttpHooksRetry, "hooks-http-retry", 3, "Number of times to retry on a 500 or network timeout") @@ -76,4 +78,9 @@ func ParseFlags() { "neither flag was provided. Please consult `tusd -help` for " + "more information on these options.") } + + if Flags.GCSObjectPrefix != "" && strings.Contains(Flags.GCSObjectPrefix, "_") { + stderr.Fatalf("gcs-object-prefix value (%s) can't contain underscore. "+ + "Please remove underscore from the value", Flags.GCSObjectPrefix) + } } diff --git a/gcsstore/gcsstore.go b/gcsstore/gcsstore.go index 589ef19..cf2579e 100644 --- a/gcsstore/gcsstore.go +++ b/gcsstore/gcsstore.go @@ -14,13 +14,14 @@ import ( "bytes" "encoding/json" "fmt" - "golang.org/x/net/context" "io" "strconv" "strings" "sync" "sync/atomic" + "golang.org/x/net/context" + "cloud.google.com/go/storage" "github.com/tus/tusd" "github.com/tus/tusd/uid" @@ -32,6 +33,11 @@ type GCSStore struct { // Specifies the GCS bucket that uploads will be stored in Bucket string + // ObjectPrefix is prepended to the name of each GCS object that is created. + // It can be used to create a pseudo-directory structure in the bucket, + // e.g. "path/to/my/uploads". + ObjectPrefix string + // Service specifies an interface used to communicate with the Google // cloud storage backend. Implementation can be seen in gcsservice file. Service GCSAPI @@ -59,7 +65,7 @@ func (store GCSStore) NewUpload(info tusd.FileInfo) (id string, err error) { } ctx := context.Background() - err = store.writeInfo(ctx, info.ID, info) + err = store.writeInfo(ctx, store.keyWithPrefix(info.ID), info) if err != nil { return info.ID, err } @@ -68,7 +74,7 @@ func (store GCSStore) NewUpload(info tusd.FileInfo) (id string, err error) { } func (store GCSStore) WriteChunk(id string, offset int64, src io.Reader) (int64, error) { - prefix := fmt.Sprintf("%s_", id) + prefix := fmt.Sprintf("%s_", store.keyWithPrefix(id)) filterParams := GCSFilterParams{ Bucket: store.Bucket, Prefix: prefix, @@ -94,7 +100,7 @@ func (store GCSStore) WriteChunk(id string, offset int64, src io.Reader) (int64, } } - cid := fmt.Sprintf("%s_%d", id, maxIdx+1) + cid := fmt.Sprintf("%s_%d", store.keyWithPrefix(id), maxIdx+1) objectParams := GCSObjectParams{ Bucket: store.Bucket, ID: cid, @@ -112,7 +118,7 @@ const CONCURRENT_SIZE_REQUESTS = 32 func (store GCSStore) GetInfo(id string) (tusd.FileInfo, error) { info := tusd.FileInfo{} - i := fmt.Sprintf("%s.info", id) + i := fmt.Sprintf("%s.info", store.keyWithPrefix(id)) params := GCSObjectParams{ Bucket: store.Bucket, @@ -138,7 +144,7 @@ func (store GCSStore) GetInfo(id string) (tusd.FileInfo, error) { return info, err } - prefix := fmt.Sprintf("%s", id) + prefix := fmt.Sprintf("%s", store.keyWithPrefix(id)) filterParams := GCSFilterParams{ Bucket: store.Bucket, Prefix: prefix, @@ -200,7 +206,7 @@ func (store GCSStore) GetInfo(id string) (tusd.FileInfo, error) { } info.Offset = offset - err = store.writeInfo(ctx, id, info) + err = store.writeInfo(ctx, store.keyWithPrefix(id), info) if err != nil { return info, err } @@ -231,7 +237,7 @@ func (store GCSStore) writeInfo(ctx context.Context, id string, info tusd.FileIn } func (store GCSStore) FinishUpload(id string) error { - prefix := fmt.Sprintf("%s_", id) + prefix := fmt.Sprintf("%s_", store.keyWithPrefix(id)) filterParams := GCSFilterParams{ Bucket: store.Bucket, Prefix: prefix, @@ -245,7 +251,7 @@ func (store GCSStore) FinishUpload(id string) error { composeParams := GCSComposeParams{ Bucket: store.Bucket, - Destination: id, + Destination: store.keyWithPrefix(id), Sources: names, } @@ -266,7 +272,7 @@ func (store GCSStore) FinishUpload(id string) error { objectParams := GCSObjectParams{ Bucket: store.Bucket, - ID: id, + ID: store.keyWithPrefix(id), } err = store.Service.SetObjectMetadata(ctx, objectParams, info.MetaData) @@ -280,7 +286,7 @@ func (store GCSStore) FinishUpload(id string) error { func (store GCSStore) Terminate(id string) error { filterParams := GCSFilterParams{ Bucket: store.Bucket, - Prefix: id, + Prefix: store.keyWithPrefix(id), } ctx := context.Background() @@ -295,7 +301,7 @@ func (store GCSStore) Terminate(id string) error { func (store GCSStore) GetReader(id string) (io.Reader, error) { params := GCSObjectParams{ Bucket: store.Bucket, - ID: id, + ID: store.keyWithPrefix(id), } ctx := context.Background() @@ -306,3 +312,11 @@ func (store GCSStore) GetReader(id string) (io.Reader, error) { return r, nil } + +func (store GCSStore) keyWithPrefix(key string) string { + prefix := store.ObjectPrefix + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + return prefix + key +} diff --git a/gcsstore/gcsstore_test.go b/gcsstore/gcsstore_test.go index fbefcc2..7ab0101 100644 --- a/gcsstore/gcsstore_test.go +++ b/gcsstore/gcsstore_test.go @@ -4,9 +4,10 @@ import ( "bytes" "encoding/json" "fmt" - "golang.org/x/net/context" "testing" + "golang.org/x/net/context" + "cloud.google.com/go/storage" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -64,6 +65,35 @@ func TestNewUpload(t *testing.T) { assert.Equal(id, mockID) } +func TestNewUploadWithPrefix(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + assert := assert.New(t) + + service := NewMockGCSAPI(mockCtrl) + store := gcsstore.New(mockBucket, service) + store.ObjectPrefix = "/path/to/file" + + assert.Equal(store.Bucket, mockBucket) + + data, err := json.Marshal(mockTusdInfo) + assert.Nil(err) + + r := bytes.NewReader(data) + + params := gcsstore.GCSObjectParams{ + Bucket: store.Bucket, + ID: fmt.Sprintf("%s.info", "/path/to/file/"+mockID), + } + + ctx := context.Background() + service.EXPECT().WriteObject(ctx, params, r).Return(int64(r.Len()), nil) + + id, err := store.NewUpload(mockTusdInfo) + assert.Nil(err) + assert.Equal(id, mockID) +} + type MockGetInfoReader struct{} func (r MockGetInfoReader) Close() error {