diff --git a/README.md b/README.md index 0fe6bd9..c09006a 100644 --- a/README.md +++ b/README.md @@ -227,7 +227,6 @@ useful tools: * [**gcsstore**](https://godoc.org/github.com/tus/tusd/gcsstore): A storage backend using Google cloud storage * [**memorylocker**](https://godoc.org/github.com/tus/tusd/memorylocker): An in-memory locker for handling concurrent uploads * [**etcd3locker**](https://godoc.org/github.com/tus/tusd/etcd3locker): A locker using the distributed KV etcd3 store -* [**limitedstore**](https://godoc.org/github.com/tus/tusd/limitedstore): A storage wrapper limiting the total used space for uploads ### 3rd-Party tusd Packages diff --git a/cmd/tusd/cli/composer.go b/cmd/tusd/cli/composer.go index f8c64ce..2927068 100644 --- a/cmd/tusd/cli/composer.go +++ b/cmd/tusd/cli/composer.go @@ -6,7 +6,6 @@ import ( "github.com/tus/tusd" "github.com/tus/tusd/filestore" "github.com/tus/tusd/gcsstore" - "github.com/tus/tusd/limitedstore" "github.com/tus/tusd/memorylocker" "github.com/tus/tusd/s3store" @@ -73,18 +72,5 @@ func CreateComposer() { store.UseIn(Composer) } - storeSize := Flags.StoreSize - maxSize := Flags.MaxSize - - if storeSize > 0 { - limitedstore.New(storeSize, Composer.Core, Composer.Terminater).UseIn(Composer) - stdout.Printf("Using %.2fMB as storage size.\n", float64(storeSize)/1024/1024) - - // We need to ensure that a single upload can fit into the storage size - if maxSize > storeSize || maxSize == 0 { - Flags.MaxSize = storeSize - } - } - stdout.Printf("Using %.2fMB as maximum size.\n", float64(Flags.MaxSize)/1024/1024) } diff --git a/cmd/tusd/cli/flags.go b/cmd/tusd/cli/flags.go index 3aa1e00..9720809 100644 --- a/cmd/tusd/cli/flags.go +++ b/cmd/tusd/cli/flags.go @@ -12,7 +12,6 @@ var Flags struct { HttpSock string MaxSize int64 UploadDir string - StoreSize int64 Basepath string Timeout int64 S3Bucket string @@ -41,7 +40,6 @@ func ParseFlags() { flag.StringVar(&Flags.HttpSock, "unix-sock", "", "If set, will listen to a UNIX socket at this location instead of a TCP socket") flag.Int64Var(&Flags.MaxSize, "max-size", 0, "Maximum size of a single upload in bytes") flag.StringVar(&Flags.UploadDir, "dir", "./data", "Directory to store uploads in") - flag.Int64Var(&Flags.StoreSize, "store-size", 0, "Size of space allowed for storage") flag.StringVar(&Flags.Basepath, "base-path", "/files/", "Basepath of the HTTP server") flag.Int64Var(&Flags.Timeout, "timeout", 30*1000, "Read timeout for connections in milliseconds. A zero value means that reads will not timeout") flag.StringVar(&Flags.S3Bucket, "s3-bucket", "", "Use AWS S3 with this bucket as storage backend (requires the AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION environment variables to be set)") diff --git a/limitedstore/limitedstore.go b/limitedstore/limitedstore.go deleted file mode 100644 index 121fd98..0000000 --- a/limitedstore/limitedstore.go +++ /dev/null @@ -1,140 +0,0 @@ -// Package limitedstore provides a storage with a limited space. -// -// This goal is achieved by using a simple wrapper around existing -// datastores (tusd.DataStore) while limiting the used storage size. -// It will start terminating existing uploads if not enough space is left in -// order to create a new upload. -// The order in which the uploads will be terminated is defined by their size, -// whereas the biggest ones are deleted first. -// This package's functionality is very limited and naive. It will terminate -// uploads whether they are finished yet or not. Only one datastore is allowed to -// access the underlying storage else the limited store will not function -// properly. Two tusd.FileStore instances using the same directory, for example. -// In addition the limited store will keep a list of the uploads' IDs in memory -// which may create a growing memory leak. -package limitedstore - -import ( - "os" - "sort" - "sync" - - "github.com/tus/tusd" -) - -type LimitedStore struct { - tusd.DataStore - terminater tusd.TerminaterDataStore - - StoreSize int64 - - uploads map[string]int64 - usedSize int64 - - mutex *sync.Mutex -} - -// pair structure to perform map-sorting -type pair struct { - key string - value int64 -} - -type pairlist []pair - -func (p pairlist) Len() int { return len(p) } -func (p pairlist) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p pairlist) Less(i, j int) bool { return p[i].value < p[j].value } - -// New creates a new limited store with the given size as the maximum storage -// size. The wrapped data store needs to implement the TerminaterDataStore -// interface, in order to provide the required Terminate method. -func New(storeSize int64, dataStore tusd.DataStore, terminater tusd.TerminaterDataStore) *LimitedStore { - return &LimitedStore{ - StoreSize: storeSize, - DataStore: dataStore, - terminater: terminater, - uploads: make(map[string]int64), - mutex: new(sync.Mutex), - } -} - -func (store *LimitedStore) UseIn(composer *tusd.StoreComposer) { - composer.UseCore(store) - composer.UseTerminater(store) -} - -func (store *LimitedStore) NewUpload(info tusd.FileInfo) (string, error) { - store.mutex.Lock() - defer store.mutex.Unlock() - - if err := store.ensureSpace(info.Size); err != nil { - return "", err - } - - id, err := store.DataStore.NewUpload(info) - if err != nil { - return "", err - } - - store.usedSize += info.Size - store.uploads[id] = info.Size - - return id, nil -} - -func (store *LimitedStore) Terminate(id string) error { - store.mutex.Lock() - defer store.mutex.Unlock() - - return store.terminate(id) -} - -func (store *LimitedStore) terminate(id string) error { - err := store.terminater.Terminate(id) - // Ignore the error if the upload could not be found. In this case, the upload - // has likely already been removed by another service (e.g. a cron job) and we - // just remove the upload from our internal list and claim the used space back. - if err != nil && err != tusd.ErrNotFound && !os.IsNotExist(err) { - return err - } - - size := store.uploads[id] - delete(store.uploads, id) - store.usedSize -= size - - return nil -} - -// Ensure enough space is available to store an upload of the specified size. -// It will terminate uploads until enough space is freed. -func (store *LimitedStore) ensureSpace(size int64) error { - if (store.usedSize + size) <= store.StoreSize { - // Enough space is available to store the new upload - return nil - } - - sortedUploads := make(pairlist, len(store.uploads)) - i := 0 - for u, h := range store.uploads { - sortedUploads[i] = pair{u, h} - i++ - } - sort.Sort(sort.Reverse(sortedUploads)) - - // Forward traversal through the uploads in terms of size, biggest upload first - for _, k := range sortedUploads { - id := k.key - - if err := store.terminate(id); err != nil { - return err - } - - if (store.usedSize + size) <= store.StoreSize { - // Enough space has been freed to store the new upload - return nil - } - } - - return nil -} diff --git a/limitedstore/limitedstore_test.go b/limitedstore/limitedstore_test.go deleted file mode 100644 index 69c61a4..0000000 --- a/limitedstore/limitedstore_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package limitedstore - -import ( - "io" - "strconv" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/tus/tusd" -) - -var _ tusd.DataStore = &LimitedStore{} -var _ tusd.TerminaterDataStore = &LimitedStore{} - -type dataStore struct { - t *assert.Assertions - numCreatedUploads int - numTerminatedUploads int -} - -func (store *dataStore) NewUpload(info tusd.FileInfo) (string, error) { - uploadId := store.numCreatedUploads - - // We expect the uploads to be created in a specific order. - // These sizes correlate to this order. - expectedSize := []int64{30, 60, 80}[uploadId] - - store.t.Equal(expectedSize, info.Size) - - store.numCreatedUploads += 1 - - return strconv.Itoa(uploadId), nil -} - -func (store *dataStore) WriteChunk(id string, offset int64, src io.Reader) (int64, error) { - return 0, nil -} - -func (store *dataStore) GetInfo(id string) (tusd.FileInfo, error) { - return tusd.FileInfo{}, nil -} - -func (store *dataStore) Terminate(id string) error { - // We expect the uploads to be terminated in a specific order (the bigger - // come first) - expectedUploadId := []string{"1", "0"}[store.numTerminatedUploads] - - store.t.Equal(expectedUploadId, id) - - store.numTerminatedUploads += 1 - - return nil -} - -func TestLimitedStore(t *testing.T) { - a := assert.New(t) - dataStore := &dataStore{ - t: a, - } - store := New(100, dataStore, dataStore) - - // Create new upload (30 bytes) - id, err := store.NewUpload(tusd.FileInfo{ - Size: 30, - }) - a.NoError(err) - a.Equal("0", id) - - // Create new upload (60 bytes) - id, err = store.NewUpload(tusd.FileInfo{ - Size: 60, - }) - a.NoError(err) - a.Equal("1", id) - - // Create new upload (80 bytes) - id, err = store.NewUpload(tusd.FileInfo{ - Size: 80, - }) - a.NoError(err) - a.Equal("2", id) - - if dataStore.numTerminatedUploads != 2 { - t.Error("expected two uploads to be terminated") - } -}