tusd/pkg/gcsstore/gcsstore.go

354 lines
7.3 KiB
Go
Raw Normal View History

// Package gcsstore provides a Google cloud storage based backend.
//
// GCSStore is a storage backend that uses the GCSAPI interface in order to store uploads
// on GCS. Uploads will be represented by two files in GCS; the data file will be stored
// as an extensionless object [uid] and the JSON info file will stored as [uid].info.
// In order to store uploads on GCS, make sure to specifiy the appropriate Google service
// account file path in the GCS_SERVICE_ACCOUNT_FILE environment variable. Also make sure that
// this service account file has the "https://www.googleapis.com/auth/devstorage.read_write"
// scope enabled so you can read and write data to the storage buckets associated with the
// service account file.
package gcsstore
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"strconv"
"strings"
"sync"
"sync/atomic"
2018-05-25 10:14:16 +00:00
"cloud.google.com/go/storage"
"github.com/tus/tusd/internal/uid"
"github.com/tus/tusd/pkg/handler"
)
// See the handler.DataStore interface for documentation about the different
// methods.
type GCSStore struct {
// Specifies the GCS bucket that uploads will be stored in
Bucket string
gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius <maerious@gmail.com> Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar <azharridho42@gmail.com> Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store
2019-06-02 13:55:41 +00:00
// ObjectPrefix is prepended to the name of each GCS object that is created.
// It can be used to create a pseudo-directory structure in the bucket,
// e.g. "path/to/my/uploads".
ObjectPrefix string
// Service specifies an interface used to communicate with the Google
// cloud storage backend. Implementation can be seen in gcsservice file.
Service GCSAPI
}
// New constructs a new GCS storage backend using the supplied GCS bucket name
// and service object.
func New(bucket string, service GCSAPI) GCSStore {
return GCSStore{
Bucket: bucket,
Service: service,
}
}
func (store GCSStore) UseIn(composer *handler.StoreComposer) {
composer.UseCore(store)
composer.UseTerminater(store)
}
func (store GCSStore) NewUpload(info handler.FileInfo) (handler.Upload, error) {
if info.ID == "" {
info.ID = uid.Uid()
}
2019-08-19 09:08:24 +00:00
info.Storage = map[string]string{
"Type": "gcsstore",
"Bucket": store.Bucket,
"Key": store.keyWithPrefix(info.ID),
}
ctx := context.Background()
err := store.writeInfo(ctx, store.keyWithPrefix(info.ID), info)
if err != nil {
return &gcsUpload{info.ID, &store}, err
}
return &gcsUpload{info.ID, &store}, nil
}
type gcsUpload struct {
id string
store *GCSStore
}
func (store GCSStore) GetUpload(id string) (handler.Upload, error) {
return &gcsUpload{id, &store}, nil
}
func (store GCSStore) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload {
return upload.(*gcsUpload)
}
func (upload gcsUpload) WriteChunk(offset int64, src io.Reader) (int64, error) {
id := upload.id
store := upload.store
gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius <maerious@gmail.com> Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar <azharridho42@gmail.com> Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store
2019-06-02 13:55:41 +00:00
prefix := fmt.Sprintf("%s_", store.keyWithPrefix(id))
filterParams := GCSFilterParams{
Bucket: store.Bucket,
Prefix: prefix,
}
ctx := context.Background()
names, err := store.Service.FilterObjects(ctx, filterParams)
if err != nil {
return 0, err
}
maxIdx := -1
for _, name := range names {
split := strings.Split(name, "_")
idx, err := strconv.Atoi(split[len(split)-1])
if err != nil {
return 0, err
}
if idx > maxIdx {
maxIdx = idx
}
}
gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius <maerious@gmail.com> Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar <azharridho42@gmail.com> Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store
2019-06-02 13:55:41 +00:00
cid := fmt.Sprintf("%s_%d", store.keyWithPrefix(id), maxIdx+1)
objectParams := GCSObjectParams{
Bucket: store.Bucket,
ID: cid,
}
n, err := store.Service.WriteObject(ctx, objectParams, src)
if err != nil {
return 0, err
}
return n, err
}
const CONCURRENT_SIZE_REQUESTS = 32
func (upload gcsUpload) GetInfo() (handler.FileInfo, error) {
id := upload.id
store := upload.store
info := handler.FileInfo{}
gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius <maerious@gmail.com> Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar <azharridho42@gmail.com> Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store
2019-06-02 13:55:41 +00:00
i := fmt.Sprintf("%s.info", store.keyWithPrefix(id))
params := GCSObjectParams{
Bucket: store.Bucket,
ID: i,
}
ctx := context.Background()
r, err := store.Service.ReadObject(ctx, params)
if err != nil {
2018-05-25 10:14:16 +00:00
if err == storage.ErrObjectNotExist {
return info, handler.ErrNotFound
2018-05-25 10:14:16 +00:00
}
return info, err
}
buf := make([]byte, r.Size())
_, err = r.Read(buf)
if err != nil {
return info, err
}
if err := json.Unmarshal(buf, &info); err != nil {
return info, err
}
gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius <maerious@gmail.com> Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar <azharridho42@gmail.com> Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store
2019-06-02 13:55:41 +00:00
prefix := fmt.Sprintf("%s", store.keyWithPrefix(id))
filterParams := GCSFilterParams{
Bucket: store.Bucket,
Prefix: prefix,
}
names, err := store.Service.FilterObjects(ctx, filterParams)
if err != nil {
return info, err
}
var offset int64 = 0
var firstError error = nil
var wg sync.WaitGroup
sem := make(chan struct{}, CONCURRENT_SIZE_REQUESTS)
errChan := make(chan error)
ctxCancel, cancel := context.WithCancel(ctx)
defer cancel()
go func() {
for err := range errChan {
if err != context.Canceled && firstError == nil {
firstError = err
cancel()
}
}
}()
for _, name := range names {
sem <- struct{}{}
wg.Add(1)
params = GCSObjectParams{
Bucket: store.Bucket,
ID: name,
}
go func(params GCSObjectParams) {
defer func() {
<-sem
wg.Done()
}()
size, err := store.Service.GetObjectSize(ctxCancel, params)
if err != nil {
errChan <- err
return
}
atomic.AddInt64(&offset, size)
}(params)
}
wg.Wait()
close(errChan)
if firstError != nil {
return info, firstError
}
info.Offset = offset
gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius <maerious@gmail.com> Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar <azharridho42@gmail.com> Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store
2019-06-02 13:55:41 +00:00
err = store.writeInfo(ctx, store.keyWithPrefix(id), info)
if err != nil {
return info, err
}
return info, nil
}
func (store GCSStore) writeInfo(ctx context.Context, id string, info handler.FileInfo) error {
data, err := json.Marshal(info)
if err != nil {
return err
}
r := bytes.NewReader(data)
i := fmt.Sprintf("%s.info", id)
params := GCSObjectParams{
Bucket: store.Bucket,
ID: i,
}
_, err = store.Service.WriteObject(ctx, params, r)
if err != nil {
return err
}
return nil
}
func (upload gcsUpload) FinishUpload() error {
id := upload.id
store := upload.store
gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius <maerious@gmail.com> Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar <azharridho42@gmail.com> Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store
2019-06-02 13:55:41 +00:00
prefix := fmt.Sprintf("%s_", store.keyWithPrefix(id))
filterParams := GCSFilterParams{
Bucket: store.Bucket,
Prefix: prefix,
}
ctx := context.Background()
names, err := store.Service.FilterObjects(ctx, filterParams)
if err != nil {
return err
}
composeParams := GCSComposeParams{
Bucket: store.Bucket,
gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius <maerious@gmail.com> Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar <azharridho42@gmail.com> Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store
2019-06-02 13:55:41 +00:00
Destination: store.keyWithPrefix(id),
Sources: names,
}
err = store.Service.ComposeObjects(ctx, composeParams)
if err != nil {
return err
}
err = store.Service.DeleteObjectsWithFilter(ctx, filterParams)
if err != nil {
return err
}
info, err := upload.GetInfo()
if err != nil {
return err
}
objectParams := GCSObjectParams{
Bucket: store.Bucket,
gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius <maerious@gmail.com> Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar <azharridho42@gmail.com> Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store
2019-06-02 13:55:41 +00:00
ID: store.keyWithPrefix(id),
}
err = store.Service.SetObjectMetadata(ctx, objectParams, info.MetaData)
if err != nil {
return err
}
return nil
}
func (upload gcsUpload) Terminate() error {
id := upload.id
store := upload.store
filterParams := GCSFilterParams{
Bucket: store.Bucket,
gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius <maerious@gmail.com> Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar <azharridho42@gmail.com> Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store
2019-06-02 13:55:41 +00:00
Prefix: store.keyWithPrefix(id),
}
ctx := context.Background()
err := store.Service.DeleteObjectsWithFilter(ctx, filterParams)
if err != nil {
return err
}
return nil
}
func (upload gcsUpload) GetReader() (io.Reader, error) {
id := upload.id
store := upload.store
params := GCSObjectParams{
Bucket: store.Bucket,
gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius <maerious@gmail.com> Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar <azharridho42@gmail.com> Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store
2019-06-02 13:55:41 +00:00
ID: store.keyWithPrefix(id),
}
ctx := context.Background()
r, err := store.Service.ReadObject(ctx, params)
if err != nil {
return nil, err
}
return r, nil
}
gcsstore: Add ability to set custom object prefix (#275) Squashed commit of the following: commit e48ca3f3fe086504aa1a97d26e2f4fe263880664 Author: Marius <maerious@gmail.com> Date: Sun Jun 2 15:54:39 2019 +0200 Format Go source code commit 477ef689d37b8904f3b79170c1b4d78a3b1fba4d Merge: 82c50f9 b89c337 Author: Ridho Azhar <azharridho42@gmail.com> Date: Mon May 27 15:56:20 2019 +0700 Merge branch 'master' into master commit 82c50f9364329432b8fac0579332afa82e249d98 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:30:57 2019 +0700 add test file with prefix commit aa8a29866fe5e9f33f360fa087199390dae5aef1 Author: ridhozhr <ridho@nodeflux.io> Date: Mon May 27 13:18:08 2019 +0700 remove object prefix gcs from parameter commit e25b36c5e95a316508c59a42116cdf5ea1d36bf0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:19:01 2019 +0700 add flags gcs object prefix validation commit 53762be170e52c3cda4bfefde4bd28c709f3622a Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 22:04:17 2019 +0700 integrate prefix with store method commit fe62533f1ea3994dd86a76b0f290bab54ebd0ef0 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 21:03:25 2019 +0700 add prefix in test file gcs store commit e824008fe22a92032236f18d6b4c02867458e194 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:57:32 2019 +0700 integrate flags with composer gcs object prefix commit bb2ee4cf4155d3a185f3750cf8dcca1306e0ee38 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:38 2019 +0700 add gcs-object-prefix flag commit 600f4fc939f3197d9485896331915556153986f3 Author: ridhozhr <ridho@nodeflux.io> Date: Wed May 22 20:54:14 2019 +0700 add object prefix in gcs store
2019-06-02 13:55:41 +00:00
func (store GCSStore) keyWithPrefix(key string) string {
prefix := store.ObjectPrefix
if prefix != "" && !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
return prefix + key
}