LumeWeb
/
tusd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP on DataStore refactor

This commit is contained in:
Marius 2019-08-24 15:14:51 +02:00
parent aa0280d004
commit 5acc586800
19 changed files with 521 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
pkg/filelocker/filelocker.go Normal file
View File

@ -0,0 +1,83 @@
// Package filestore provide a storage backend based on the local file system.
//
// FileStore is a storage backend used as a handler.DataStore in handler.NewHandler.
// It stores the uploads in a directory specified in two different files: The
// `[id].info` files are used to store the fileinfo in JSON format. The
// `[id]` files without an extension contain the raw binary data uploaded.
// No cleanup is performed so you may want to run a cronjob to ensure your disk
// is not filled up with old and finished uploads.
//
// In addition, it provides an exclusive upload locking mechanism using lock files
// which are stored on disk. Each of them stores the PID of the process which
// acquired the lock. This allows locks to be automatically freed when a process
// is unable to release it on its own because the process is not alive anymore.
// For more information, consult the documentation for handler.LockerDataStore
// interface, which is implemented by FileStore
package filestore
import (
"os"
"path/filepath"
"github.com/tus/tusd/pkg/handler"
"gopkg.in/Acconut/lockfile.v1"
)
var defaultFilePerm = os.FileMode(0664)
// See the handler.DataStore interface for documentation about the different
// methods.
type FileLocker struct {
// Relative or absolute path to store files in. FileStore does not check
// whether the path exists, use os.MkdirAll in this case on your own.
Path string
}
// New creates a new file based storage backend. The directory specified will
// be used as the only storage entry. This method does not check
// whether the path exists, use os.MkdirAll to ensure.
// In addition, a locking mechanism is provided.
func New(path string) FileLocker {
return FileLocker{path}
}
func (locker FileLocker) NewLock(id string) (handler.UploadLock, error) {
path, err := filepath.Abs(filepath.Join(locker.Path, id+".lock"))
if err != nil {
return lockfile.Lockfile(""), err
}
// We use Lockfile directly instead of lockfile.New to bypass the unnecessary
// check whether the provided path is absolute since we just resolved it
// on our own.
return fileUploadLock{
file: lockfile.Lockfile(path),
}, nil
}
type fileUploadLock struct {
file lockfile.Lockfile
}
func (lock fileUploadLock) Lock() error {
err = lock.file.TryLock()
if err == lockfile.ErrBusy {
return handler.ErrFileLocked
}
return err
}
func (lock fileUploadLock) Unlock() error {
err = lock.file.Unlock()
// A "no such file or directory" will be returned if no lockfile was found.
// Since this means that the file has never been locked, we drop the error
// and continue as if nothing happened.
if os.IsNotExist(err) {
err = nil
}
return err
}

26
pkg/handler/composer.go
View File

@ -8,12 +8,8 @@ type StoreComposer struct {
UsesTerminater bool
Terminater TerminaterDataStore
UsesFinisher bool
Finisher FinisherDataStore
UsesLocker bool
Locker LockerDataStore
UsesGetReader bool
GetReader GetReaderDataStore
UsesConcater bool
Concater ConcaterDataStore
UsesLengthDeferrer bool
@ -42,24 +38,12 @@ func (store *StoreComposer) Capabilities() string {
} else {
str += "✗"
}
str += ` Finisher: `
if store.UsesFinisher {
str += "✓"
} else {
str += "✗"
}
str += ` Locker: `
if store.UsesLocker {
str += "✓"
} else {
str += "✗"
}
str += ` GetReader: `
if store.UsesGetReader {
str += "✓"
} else {
str += "✗"
}
str += ` Concater: `
if store.UsesConcater {
str += "✓"
@ -86,18 +70,12 @@ func (store *StoreComposer) UseTerminater(ext TerminaterDataStore) {
store.UsesTerminater = ext != nil
store.Terminater = ext
}
func (store *StoreComposer) UseFinisher(ext FinisherDataStore) {
store.UsesFinisher = ext != nil
store.Finisher = ext
}
func (store *StoreComposer) UseLocker(ext LockerDataStore) {
store.UsesLocker = ext != nil
store.Locker = ext
}
func (store *StoreComposer) UseGetReader(ext GetReaderDataStore) {
store.UsesGetReader = ext != nil
store.GetReader = ext
}
func (store *StoreComposer) UseConcater(ext ConcaterDataStore) {
store.UsesConcater = ext != nil
store.Concater = ext

80
pkg/handler/datastore.go
View File

@ -47,12 +47,7 @@ func (f FileInfo) StopUpload() {
}
}
type DataStore interface {
// Create a new upload using the size as the file's length. The method must
// return an unique id which is used to identify the upload. If no backend
// (e.g. Riak) specifes the id you may want to use the uid package to
// generate one. The properties Size and MetaData will be filled.
NewUpload(info FileInfo) (id string, err error)
type Upload interface {
// Write the chunk read from src into the file specified by the id at the
// given offset. The handler will take care of validating the offset and
// limiting the size of the src to not overflow the file's size. It may
@ -60,31 +55,50 @@ type DataStore interface {
// It will also lock resources while they are written to ensure only one
// write happens per time.
// The function call must return the number of bytes written.
WriteChunk(id string, offset int64, src io.Reader) (int64, error)
WriteChunk(offset int64, src io.Reader) (int64, error)
// Read the fileinformation used to validate the offset and respond to HEAD
// requests. It may return an os.ErrNotExist which will be interpreted as a
// 404 Not Found.
GetInfo(id string) (FileInfo, error)
GetInfo() (FileInfo, error)
// GetReader returns a reader which allows iterating of the content of an
// upload specified by its ID. It should attempt to provide a reader even if
// the upload has not been finished yet but it's not required.
// If the returned reader also implements the io.Closer interface, the
// Close() method will be invoked once everything has been read.
// If the given upload could not be found, the error tusd.ErrNotFound should
// be returned.
GetReader() (io.Reader, error)
// FinisherDataStore is the interface which can be implemented by DataStores
// which need to do additional operations once an entire upload has been
// completed. These tasks may include but are not limited to freeing unused
// resources or notifying other services. For example, S3Store uses this
// interface for removing a temporary object.
// FinishUpload executes additional operations for the finished upload which
// is specified by its ID.
FinishUpload() error
}
type DataStore interface {
// Create a new upload using the size as the file's length. The method must
// return an unique id which is used to identify the upload. If no backend
// (e.g. Riak) specifes the id you may want to use the uid package to
// generate one. The properties Size and MetaData will be filled.
NewUpload(info FileInfo) (upload Upload, err error)
GetUpload(id string) (upload Upload, err error)
}
type TerminatableUpload interface {
// Terminate an upload so any further requests to the resource, both reading
// and writing, must return os.ErrNotExist or similar.
Terminate() error
}
// TerminaterDataStore is the interface which must be implemented by DataStores
// if they want to receive DELETE requests using the Handler. If this interface
// is not implemented, no request handler for this method is attached.
type TerminaterDataStore interface {
// Terminate an upload so any further requests to the resource, both reading
// and writing, must return os.ErrNotExist or similar.
Terminate(id string) error
}
// FinisherDataStore is the interface which can be implemented by DataStores
// which need to do additional operations once an entire upload has been
// completed. These tasks may include but are not limited to freeing unused
// resources or notifying other services. For example, S3Store uses this
// interface for removing a temporary object.
type FinisherDataStore interface {
// FinishUpload executes additional operations for the finished upload which
// is specified by its ID.
FinishUpload(id string) error
AsTerminatableUpload(upload Upload) TerminatableUpload
}
// LockerDataStore is the interface required for custom lock persisting mechanisms.
@ -106,22 +120,6 @@ type LockerDataStore interface {
UnlockUpload(id string) error
}
// GetReaderDataStore is the interface which must be implemented if handler should
// expose and support the GET route. It will allow clients to download the
// content of an upload regardless whether it's finished or not.
// Please, be aware that this feature is not part of the official tus
// specification. Instead it's a custom mechanism by tusd.
type GetReaderDataStore interface {
// GetReader returns a reader which allows iterating of the content of an
// upload specified by its ID. It should attempt to provide a reader even if
// the upload has not been finished yet but it's not required.
// If the returned reader also implements the io.Closer interface, the
// Close() method will be invoked once everything has been read.
// If the given upload could not be found, the error tusd.ErrNotFound should
// be returned.
GetReader(id string) (io.Reader, error)
}
// ConcaterDataStore is the interface required to be implemented if the
// Concatenation extension should be enabled. Only in this case, the handler
// will parse and respect the Upload-Concat header.
@ -140,5 +138,9 @@ type ConcaterDataStore interface {
// client to upload files when their total size is not yet known. Instead, the
// client must send the total size as soon as it becomes known.
type LengthDeferrerDataStore interface {
DeclareLength(id string, length int64) error
AsLengthDeclarableUpload(upload Upload) LengthDeclarableUpload
}
type LengthDeclarableUpload interface {
DeclareLength(length int64) error
}

2
pkg/handler/composer_test.go → pkg/handler/f/composer_test.go
View File

@ -1,5 +1,6 @@
package handler_test
/*
import (
"github.com/tus/tusd/pkg/filestore"
"github.com/tus/tusd/pkg/handler"
@ -21,3 +22,4 @@ func ExampleNewStoreComposer() {
_, _ = handler.NewHandler(config)
}
*/

0
pkg/handler/concat_test.go → pkg/handler/f/concat_test.go
View File

13
pkg/handler/config_test.go → pkg/handler/f/config_test.go
View File

@ -1,7 +1,6 @@
package handler
import (
"io"
"testing"
"github.com/stretchr/testify/assert"
@ -9,15 +8,11 @@ import (
type zeroStore struct{}
func (store zeroStore) NewUpload(info FileInfo) (string, error) {
return "", nil
func (store zeroStore) NewUpload(info FileInfo) (Upload, error) {
return nil, nil
}
func (store zeroStore) WriteChunk(id string, offset int64, src io.Reader) (int64, error) {
return 0, nil
}
func (store zeroStore) GetInfo(id string) (FileInfo, error) {
return FileInfo{}, nil
func (store zeroStore) GetUpload(id string) (Upload, error) {
return nil, nil
}
func TestConfig(t *testing.T) {

0
pkg/handler/cors_test.go → pkg/handler/f/cors_test.go
View File

0
pkg/handler/head_test.go → pkg/handler/f/head_test.go
View File

0
pkg/handler/options_test.go → pkg/handler/f/options_test.go
View File

0
pkg/handler/patch_test.go → pkg/handler/f/patch_test.go
View File

0
pkg/handler/post_test.go → pkg/handler/f/post_test.go
View File

93
pkg/handler/f/terminate_test.go Normal file
View File

@ -0,0 +1,93 @@
package handler_test
import (
"net/http"
"testing"
"github.com/golang/mock/gomock"
. "github.com/tus/tusd/pkg/handler"
"github.com/stretchr/testify/assert"
)
func TestTerminate(t *testing.T) {
SubTest(t, "ExtensionDiscovery", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
composer = NewStoreComposer()
composer.UseCore(store)
composer.UseTerminater(store)
handler, _ := NewHandler(Config{
StoreComposer: composer,
})
(&httpTest{
Method: "OPTIONS",
Code: http.StatusOK,
ResHeader: map[string]string{
"Tus-Extension": "creation,creation-with-upload,termination",
},
}).Run(handler, t)
})
SubTest(t, "Termination", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
locker := NewMockLocker(ctrl)
gomock.InOrder(
locker.EXPECT().LockUpload("foo"),
store.EXPECT().GetInfo("foo").Return(FileInfo{
ID: "foo",
Size: 10,
}, nil),
store.EXPECT().Terminate("foo").Return(nil),
locker.EXPECT().UnlockUpload("foo"),
)
composer = NewStoreComposer()
composer.UseCore(store)
composer.UseTerminater(store)
composer.UseLocker(locker)
handler, _ := NewHandler(Config{
StoreComposer: composer,
NotifyTerminatedUploads: true,
})
c := make(chan FileInfo, 1)
handler.TerminatedUploads = c
(&httpTest{
Method: "DELETE",
URL: "foo",
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
},
Code: http.StatusNoContent,
}).Run(handler, t)
info := <-c
a := assert.New(t)
a.Equal("foo", info.ID)
a.Equal(int64(10), info.Size)
})
SubTest(t, "NotProvided", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
composer = NewStoreComposer()
composer.UseCore(store)
handler, _ := NewUnroutedHandler(Config{
StoreComposer: composer,
})
(&httpTest{
Method: "DELETE",
URL: "foo",
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
},
Code: http.StatusNotImplemented,
}).Run(http.HandlerFunc(handler.DelFile), t)
})
}

55
pkg/handler/get_test.go
View File

@ -28,10 +28,12 @@ func TestGet(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
locker := NewMockLocker(ctrl)
upload := NewMockFullUpload(ctrl)
gomock.InOrder(
locker.EXPECT().LockUpload("yes"),
store.EXPECT().GetInfo("yes").Return(FileInfo{
store.EXPECT().GetUpload("yes").Return(upload, nil),
upload.EXPECT().GetInfo().Return(FileInfo{
Offset: 5,
Size: 20,
MetaData: map[string]string{
@ -39,13 +41,12 @@ func TestGet(t *testing.T) {
"filetype": "image/jpeg",
},
}, nil),
store.EXPECT().GetReader("yes").Return(reader, nil),
upload.EXPECT().GetReader().Return(reader, nil),
locker.EXPECT().UnlockUpload("yes"),
)
composer = NewStoreComposer()
composer.UseCore(store)
composer.UseGetReader(store)
composer.UseLocker(locker)
handler, _ := NewHandler(Config{
@ -70,9 +71,16 @@ func TestGet(t *testing.T) {
})
SubTest(t, "EmptyDownload", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
store.EXPECT().GetInfo("yes").Return(FileInfo{
ctrl := gomock.NewController(t)
defer ctrl.Finish()
upload := NewMockFullUpload(ctrl)
gomock.InOrder(
store.EXPECT().GetUpload("yes").Return(upload, nil),
upload.EXPECT().GetInfo().Return(FileInfo{
Offset: 0,
}, nil)
}, nil),
)
handler, _ := NewHandler(Config{
StoreComposer: composer,
@ -90,28 +98,20 @@ func TestGet(t *testing.T) {
}).Run(handler, t)
})
SubTest(t, "NotProvided", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
composer = NewStoreComposer()
composer.UseCore(store)
handler, _ := NewUnroutedHandler(Config{
StoreComposer: composer,
})
(&httpTest{
Method: "GET",
URL: "foo",
Code: http.StatusNotImplemented,
}).Run(http.HandlerFunc(handler.GetFile), t)
})
SubTest(t, "InvalidFileType", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
store.EXPECT().GetInfo("yes").Return(FileInfo{
ctrl := gomock.NewController(t)
defer ctrl.Finish()
upload := NewMockFullUpload(ctrl)
gomock.InOrder(
store.EXPECT().GetUpload("yes").Return(upload, nil),
upload.EXPECT().GetInfo().Return(FileInfo{
Offset: 0,
MetaData: map[string]string{
"filetype": "non-a-valid-mime-type",
},
}, nil)
}, nil),
)
handler, _ := NewHandler(Config{
StoreComposer: composer,
@ -131,13 +131,20 @@ func TestGet(t *testing.T) {
})
SubTest(t, "NotWhitelistedFileType", func(t *testing.T, store *MockFullDataStore, composer *StoreComposer) {
store.EXPECT().GetInfo("yes").Return(FileInfo{
ctrl := gomock.NewController(t)
defer ctrl.Finish()
upload := NewMockFullUpload(ctrl)
gomock.InOrder(
store.EXPECT().GetUpload("yes").Return(upload, nil),
upload.EXPECT().GetInfo().Return(FileInfo{
Offset: 0,
MetaData: map[string]string{
"filetype": "text/html",
"filename": "invoice.html",
},
}, nil)
}, nil),
)
handler, _ := NewHandler(Config{
StoreComposer: composer,

6
pkg/handler/handler.go
View File

@ -40,16 +40,12 @@ func NewHandler(config Config) (*Handler, error) {
mux.Post("", http.HandlerFunc(handler.PostFile))
mux.Head(":id", http.HandlerFunc(handler.HeadFile))
mux.Add("PATCH", ":id", http.HandlerFunc(handler.PatchFile))
mux.Get(":id", http.HandlerFunc(handler.GetFile))
// Only attach the DELETE handler if the Terminate() method is provided
if config.StoreComposer.UsesTerminater {
mux.Del(":id", http.HandlerFunc(handler.DelFile))
}
// GET handler requires the GetReader() method
if config.StoreComposer.UsesGetReader {
mux.Get(":id", http.HandlerFunc(handler.GetFile))
}
return routedHandler, nil
}

248
pkg/handler/handler_mock_test.go
View File

@ -1,156 +1,268 @@
// Automatically generated by MockGen. DO NOT EDIT!
// Code generated by MockGen. DO NOT EDIT.
// Source: utils_test.go
// Package handler_test is a generated GoMock package.
package handler_test
import (
gomock "github.com/golang/mock/gomock"
handler "github.com/tus/tusd/pkg/handler"
io "io"
reflect "reflect"
)
// Mock of FullDataStore interface
// MockFullDataStore is a mock of FullDataStore interface
type MockFullDataStore struct {
ctrl *gomock.Controller
recorder *_MockFullDataStoreRecorder
recorder *MockFullDataStoreMockRecorder
}
// Recorder for MockFullDataStore (not exported)
type _MockFullDataStoreRecorder struct {
// MockFullDataStoreMockRecorder is the mock recorder for MockFullDataStore
type MockFullDataStoreMockRecorder struct {
mock *MockFullDataStore
}
// NewMockFullDataStore creates a new mock instance
func NewMockFullDataStore(ctrl *gomock.Controller) *MockFullDataStore {
mock := &MockFullDataStore{ctrl: ctrl}
mock.recorder = &_MockFullDataStoreRecorder{mock}
mock.recorder = &MockFullDataStoreMockRecorder{mock}
return mock
}
func (_m *MockFullDataStore) EXPECT() *_MockFullDataStoreRecorder {
return _m.recorder
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockFullDataStore) EXPECT() *MockFullDataStoreMockRecorder {
return m.recorder
}
func (_m *MockFullDataStore) NewUpload(info handler.FileInfo) (string, error) {
ret := _m.ctrl.Call(_m, "NewUpload", info)
ret0, _ := ret[0].(string)
// NewUpload mocks base method
func (m *MockFullDataStore) NewUpload(info handler.FileInfo) (handler.Upload, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NewUpload", info)
ret0, _ := ret[0].(handler.Upload)
ret1, _ := ret[1].(error)
return ret0, ret1
}
func (_mr *_MockFullDataStoreRecorder) NewUpload(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "NewUpload", arg0)
// NewUpload indicates an expected call of NewUpload
func (mr *MockFullDataStoreMockRecorder) NewUpload(info interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewUpload", reflect.TypeOf((*MockFullDataStore)(nil).NewUpload), info)
}
func (_m *MockFullDataStore) WriteChunk(id string, offset int64, src io.Reader) (int64, error) {
ret := _m.ctrl.Call(_m, "WriteChunk", id, offset, src)
// GetUpload mocks base method
func (m *MockFullDataStore) GetUpload(id string) (handler.Upload, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetUpload", id)
ret0, _ := ret[0].(handler.Upload)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetUpload indicates an expected call of GetUpload
func (mr *MockFullDataStoreMockRecorder) GetUpload(id interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUpload", reflect.TypeOf((*MockFullDataStore)(nil).GetUpload), id)
}
// AsTerminatableUpload mocks base method
func (m *MockFullDataStore) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AsTerminatableUpload", upload)
ret0, _ := ret[0].(handler.TerminatableUpload)
return ret0
}
// AsTerminatableUpload indicates an expected call of AsTerminatableUpload
func (mr *MockFullDataStoreMockRecorder) AsTerminatableUpload(upload interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsTerminatableUpload", reflect.TypeOf((*MockFullDataStore)(nil).AsTerminatableUpload), upload)
}
// ConcatUploads mocks base method
func (m *MockFullDataStore) ConcatUploads(destination string, partialUploads []string) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ConcatUploads", destination, partialUploads)
ret0, _ := ret[0].(error)
return ret0
}
// ConcatUploads indicates an expected call of ConcatUploads
func (mr *MockFullDataStoreMockRecorder) ConcatUploads(destination, partialUploads interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConcatUploads", reflect.TypeOf((*MockFullDataStore)(nil).ConcatUploads), destination, partialUploads)
}
// AsLengthDeclarableUpload mocks base method
func (m *MockFullDataStore) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AsLengthDeclarableUpload", upload)
ret0, _ := ret[0].(handler.LengthDeclarableUpload)
return ret0
}
// AsLengthDeclarableUpload indicates an expected call of AsLengthDeclarableUpload
func (mr *MockFullDataStoreMockRecorder) AsLengthDeclarableUpload(upload interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsLengthDeclarableUpload", reflect.TypeOf((*MockFullDataStore)(nil).AsLengthDeclarableUpload), upload)
}
// MockFullUpload is a mock of FullUpload interface
type MockFullUpload struct {
ctrl *gomock.Controller
recorder *MockFullUploadMockRecorder
}
// MockFullUploadMockRecorder is the mock recorder for MockFullUpload
type MockFullUploadMockRecorder struct {
mock *MockFullUpload
}
// NewMockFullUpload creates a new mock instance
func NewMockFullUpload(ctrl *gomock.Controller) *MockFullUpload {
mock := &MockFullUpload{ctrl: ctrl}
mock.recorder = &MockFullUploadMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockFullUpload) EXPECT() *MockFullUploadMockRecorder {
return m.recorder
}
// WriteChunk mocks base method
func (m *MockFullUpload) WriteChunk(offset int64, src io.Reader) (int64, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "WriteChunk", offset, src)
ret0, _ := ret[0].(int64)
ret1, _ := ret[1].(error)
return ret0, ret1
}
func (_mr *_MockFullDataStoreRecorder) WriteChunk(arg0, arg1, arg2 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "WriteChunk", arg0, arg1, arg2)
// WriteChunk indicates an expected call of WriteChunk
func (mr *MockFullUploadMockRecorder) WriteChunk(offset, src interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteChunk", reflect.TypeOf((*MockFullUpload)(nil).WriteChunk), offset, src)
}
func (_m *MockFullDataStore) GetInfo(id string) (handler.FileInfo, error) {
ret := _m.ctrl.Call(_m, "GetInfo", id)
// GetInfo mocks base method
func (m *MockFullUpload) GetInfo() (handler.FileInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetInfo")
ret0, _ := ret[0].(handler.FileInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
func (_mr *_MockFullDataStoreRecorder) GetInfo(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "GetInfo", arg0)
// GetInfo indicates an expected call of GetInfo
func (mr *MockFullUploadMockRecorder) GetInfo() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInfo", reflect.TypeOf((*MockFullUpload)(nil).GetInfo))
}
func (_m *MockFullDataStore) Terminate(id string) error {
ret := _m.ctrl.Call(_m, "Terminate", id)
ret0, _ := ret[0].(error)
return ret0
}
func (_mr *_MockFullDataStoreRecorder) Terminate(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "Terminate", arg0)
}
func (_m *MockFullDataStore) ConcatUploads(destination string, partialUploads []string) error {
ret := _m.ctrl.Call(_m, "ConcatUploads", destination, partialUploads)
ret0, _ := ret[0].(error)
return ret0
}
func (_mr *_MockFullDataStoreRecorder) ConcatUploads(arg0, arg1 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "ConcatUploads", arg0, arg1)
}
func (_m *MockFullDataStore) GetReader(id string) (io.Reader, error) {
ret := _m.ctrl.Call(_m, "GetReader", id)
// GetReader mocks base method
func (m *MockFullUpload) GetReader() (io.Reader, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetReader")
ret0, _ := ret[0].(io.Reader)
ret1, _ := ret[1].(error)
return ret0, ret1
}
func (_mr *_MockFullDataStoreRecorder) GetReader(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "GetReader", arg0)
// GetReader indicates an expected call of GetReader
func (mr *MockFullUploadMockRecorder) GetReader() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetReader", reflect.TypeOf((*MockFullUpload)(nil).GetReader))
}
func (_m *MockFullDataStore) FinishUpload(id string) error {
ret := _m.ctrl.Call(_m, "FinishUpload", id)
// FinishUpload mocks base method
func (m *MockFullUpload) FinishUpload() error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "FinishUpload")
ret0, _ := ret[0].(error)
return ret0
}
func (_mr *_MockFullDataStoreRecorder) FinishUpload(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "FinishUpload", arg0)
// FinishUpload indicates an expected call of FinishUpload
func (mr *MockFullUploadMockRecorder) FinishUpload() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FinishUpload", reflect.TypeOf((*MockFullUpload)(nil).FinishUpload))
}
func (_m *MockFullDataStore) DeclareLength(id string, length int64) error {
ret := _m.ctrl.Call(_m, "DeclareLength", id, length)
// Terminate mocks base method
func (m *MockFullUpload) Terminate() error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Terminate")
ret0, _ := ret[0].(error)
return ret0
}
func (_mr *_MockFullDataStoreRecorder) DeclareLength(arg0, arg1 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "DeclareLength", arg0, arg1)
// Terminate indicates an expected call of Terminate
func (mr *MockFullUploadMockRecorder) Terminate() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Terminate", reflect.TypeOf((*MockFullUpload)(nil).Terminate))
}
// Mock of Locker interface
// DeclareLength mocks base method
func (m *MockFullUpload) DeclareLength(length int64) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "DeclareLength", length)
ret0, _ := ret[0].(error)
return ret0
}
// DeclareLength indicates an expected call of DeclareLength
func (mr *MockFullUploadMockRecorder) DeclareLength(length interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeclareLength", reflect.TypeOf((*MockFullUpload)(nil).DeclareLength), length)
}
// MockLocker is a mock of Locker interface
type MockLocker struct {
ctrl *gomock.Controller
recorder *_MockLockerRecorder
recorder *MockLockerMockRecorder
}
// Recorder for MockLocker (not exported)
type _MockLockerRecorder struct {
// MockLockerMockRecorder is the mock recorder for MockLocker
type MockLockerMockRecorder struct {
mock *MockLocker
}
// NewMockLocker creates a new mock instance
func NewMockLocker(ctrl *gomock.Controller) *MockLocker {
mock := &MockLocker{ctrl: ctrl}
mock.recorder = &_MockLockerRecorder{mock}
mock.recorder = &MockLockerMockRecorder{mock}
return mock
}
func (_m *MockLocker) EXPECT() *_MockLockerRecorder {
return _m.recorder
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockLocker) EXPECT() *MockLockerMockRecorder {
return m.recorder
}
func (_m *MockLocker) LockUpload(id string) error {
ret := _m.ctrl.Call(_m, "LockUpload", id)
// LockUpload mocks base method
func (m *MockLocker) LockUpload(id string) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "LockUpload", id)
ret0, _ := ret[0].(error)
return ret0
}
func (_mr *_MockLockerRecorder) LockUpload(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "LockUpload", arg0)
// LockUpload indicates an expected call of LockUpload
func (mr *MockLockerMockRecorder) LockUpload(id interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LockUpload", reflect.TypeOf((*MockLocker)(nil).LockUpload), id)
}
func (_m *MockLocker) UnlockUpload(id string) error {
ret := _m.ctrl.Call(_m, "UnlockUpload", id)
// UnlockUpload mocks base method
func (m *MockLocker) UnlockUpload(id string) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "UnlockUpload", id)
ret0, _ := ret[0].(error)
return ret0
}
func (_mr *_MockLockerRecorder) UnlockUpload(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "UnlockUpload", arg0)
// UnlockUpload indicates an expected call of UnlockUpload
func (mr *MockLockerMockRecorder) UnlockUpload(id interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnlockUpload", reflect.TypeOf((*MockLocker)(nil).UnlockUpload), id)
}

2
pkg/handler/subtest_test.go
View File

@ -19,8 +19,6 @@ func SubTest(t *testing.T, name string, runTest func(*testing.T, *MockFullDataSt
composer := handler.NewStoreComposer()
composer.UseCore(store)
composer.UseTerminater(store)
composer.UseFinisher(store)
composer.UseGetReader(store)
composer.UseConcater(store)
composer.UseLengthDeferrer(store)

7
pkg/handler/terminate_test.go
View File

@ -33,14 +33,17 @@ func TestTerminate(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
locker := NewMockLocker(ctrl)
upload := NewMockFullUpload(ctrl)
gomock.InOrder(
locker.EXPECT().LockUpload("foo"),
store.EXPECT().GetInfo("foo").Return(FileInfo{
store.EXPECT().GetUpload("foo").Return(upload, nil),
upload.EXPECT().GetInfo().Return(FileInfo{
ID: "foo",
Size: 10,
}, nil),
store.EXPECT().Terminate("foo").Return(nil),
store.EXPECT().AsTerminatableUpload(upload).Return(upload),
upload.EXPECT().Terminate().Return(nil),
locker.EXPECT().UnlockUpload("foo"),
)

90
pkg/handler/unrouted_handler.go
View File

@ -297,14 +297,19 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
PartialUploads: partialUploads,
}
id, err := handler.composer.Core.NewUpload(info)
upload, err := handler.composer.Core.NewUpload(info)
if err != nil {
handler.sendError(w, r, err)
return
}
// TODO: Should we use GetInfo here?
info.ID = id
info, err = upload.GetInfo()
if err != nil {
handler.sendError(w, r, err)
return
}
id := info.ID
// Add the Location header directly after creating the new resource to even
// include it in cases of failure when an error is returned
@ -341,7 +346,7 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
defer locker.UnlockUpload(id)
}
if err := handler.writeChunk(id, info, w, r); err != nil {
if err := handler.writeChunk(upload, info, w, r); err != nil {
handler.sendError(w, r, err)
return
}
@ -349,7 +354,7 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
// Directly finish the upload if the upload is empty (i.e. has a size of 0).
// This statement is in an else-if block to avoid causing duplicate calls
// to finishUploadIfComplete if an upload is empty and contains a chunk.
handler.finishUploadIfComplete(info)
handler.finishUploadIfComplete(upload, info)
}
handler.sendResp(w, r, http.StatusCreated)
@ -374,7 +379,13 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request)
defer locker.UnlockUpload(id)
}
info, err := handler.composer.Core.GetInfo(id)
upload, err := handler.composer.Core.GetUpload(id)
if err != nil {
handler.sendError(w, r, err)
return
}
info, err := upload.GetInfo()
if err != nil {
handler.sendError(w, r, err)
return
@ -444,7 +455,13 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
defer locker.UnlockUpload(id)
}
info, err := handler.composer.Core.GetInfo(id)
upload, err := handler.composer.Core.GetUpload(id)
if err != nil {
handler.sendError(w, r, err)
return
}
info, err := upload.GetInfo()
if err != nil {
handler.sendError(w, r, err)
return
@ -482,7 +499,9 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
handler.sendError(w, r, ErrInvalidUploadLength)
return
}
if err := handler.composer.LengthDeferrer.DeclareLength(id, uploadLength); err != nil {
lengthDeclarableUpload := handler.composer.LengthDeferrer.AsLengthDeclarableUpload(upload)
if err := lengthDeclarableUpload.DeclareLength(uploadLength); err != nil {
handler.sendError(w, r, err)
return
}
@ -491,7 +510,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
info.SizeIsDeferred = false
}
if err := handler.writeChunk(id, info, w, r); err != nil {
if err := handler.writeChunk(upload, info, w, r); err != nil {
handler.sendError(w, r, err)
return
}
@ -502,10 +521,11 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
// writeChunk reads the body from the requests r and appends it to the upload
// with the corresponding id. Afterwards, it will set the necessary response
// headers but will not send the response.
func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.ResponseWriter, r *http.Request) error {
func (handler *UnroutedHandler) writeChunk(upload Upload, info FileInfo, w http.ResponseWriter, r *http.Request) error {
// Get Content-Length if possible
length := r.ContentLength
offset := info.Offset
id := info.ID
// Test if this upload fits into the file's size
if !info.SizeIsDeferred && offset+length > info.Size {
@ -562,9 +582,9 @@ func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.Resp
}
var err error
bytesWritten, err = handler.composer.Core.WriteChunk(id, offset, reader)
bytesWritten, err = upload.WriteChunk(offset, reader)
if terminateUpload && handler.composer.UsesTerminater {
if terminateErr := handler.terminateUpload(id, info); terminateErr != nil {
if terminateErr := handler.terminateUpload(upload, info); terminateErr != nil {
// We only log this error and not show it to the user since this
// termination error is not relevant to the uploading client
handler.log("UploadStopTerminateError", "id", id, "error", terminateErr.Error())
@ -591,21 +611,19 @@ func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.Resp
handler.Metrics.incBytesReceived(uint64(bytesWritten))
info.Offset = newOffset
return handler.finishUploadIfComplete(info)
return handler.finishUploadIfComplete(upload, info)
}
// finishUploadIfComplete checks whether an upload is completed (i.e. upload offset
// matches upload size) and if so, it will call the data store's FinishUpload
// function and send the necessary message on the CompleteUpload channel.
func (handler *UnroutedHandler) finishUploadIfComplete(info FileInfo) error {
func (handler *UnroutedHandler) finishUploadIfComplete(upload Upload, info FileInfo) error {
// If the upload is completed, ...
if !info.SizeIsDeferred && info.Offset == info.Size {
// ... allow custom mechanism to finish and cleanup the upload
if handler.composer.UsesFinisher {
if err := handler.composer.Finisher.FinishUpload(info.ID); err != nil {
if err := upload.FinishUpload(); err != nil {
return err
}
}
// ... send the info out to the channel
if handler.config.NotifyCompleteUploads {
@ -621,11 +639,6 @@ func (handler *UnroutedHandler) finishUploadIfComplete(info FileInfo) error {
// GetFile handles requests to download a file using a GET request. This is not
// part of the specification.
func (handler *UnroutedHandler) GetFile(w http.ResponseWriter, r *http.Request) {
if !handler.composer.UsesGetReader {
handler.sendError(w, r, ErrNotImplemented)
return
}
id, err := extractIDFromPath(r.URL.Path)
if err != nil {
handler.sendError(w, r, err)
@ -642,7 +655,13 @@ func (handler *UnroutedHandler) GetFile(w http.ResponseWriter, r *http.Request)
defer locker.UnlockUpload(id)
}
info, err := handler.composer.Core.GetInfo(id)
upload, err := handler.composer.Core.GetUpload(id)
if err != nil {
handler.sendError(w, r, err)
return
}
info, err := upload.GetInfo()
if err != nil {
handler.sendError(w, r, err)
return
@ -661,7 +680,7 @@ func (handler *UnroutedHandler) GetFile(w http.ResponseWriter, r *http.Request)
return
}
src, err := handler.composer.GetReader.GetReader(id)
src, err := upload.GetReader()
if err != nil {
handler.sendError(w, r, err)
return
@ -761,16 +780,22 @@ func (handler *UnroutedHandler) DelFile(w http.ResponseWriter, r *http.Request)
defer locker.UnlockUpload(id)
}
upload, err := handler.composer.Core.GetUpload(id)
if err != nil {
handler.sendError(w, r, err)
return
}
var info FileInfo
if handler.config.NotifyTerminatedUploads {
info, err = handler.composer.Core.GetInfo(id)
info, err = upload.GetInfo()
if err != nil {
handler.sendError(w, r, err)
return
}
}
err = handler.terminateUpload(id, info)
err = handler.terminateUpload(upload, info)
if err != nil {
handler.sendError(w, r, err)
return
@ -784,8 +809,10 @@ func (handler *UnroutedHandler) DelFile(w http.ResponseWriter, r *http.Request)
// and updates the statistics.
// Note the the info argument is only needed if the terminated uploads
// notifications are enabled.
func (handler *UnroutedHandler) terminateUpload(id string, info FileInfo) error {
err := handler.composer.Terminater.Terminate(id)
func (handler *UnroutedHandler) terminateUpload(upload Upload, info FileInfo) error {
terminatableUpload := handler.composer.Terminater.AsTerminatableUpload(upload)
err := terminatableUpload.Terminate()
if err != nil {
return err
}
@ -951,7 +978,12 @@ func getHostAndProtocol(r *http.Request, allowForwarded bool) (host, proto strin
// of a final resource.
func (handler *UnroutedHandler) sizeOfUploads(ids []string) (size int64, err error) {
for _, id := range ids {
info, err := handler.composer.Core.GetInfo(id)
upload, err := handler.composer.Core.GetUpload(id)
if err != nil {
return size, err
}
info, err := upload.GetInfo()
if err != nil {
return size, err
}

10
pkg/handler/utils_test.go
View File

@ -13,7 +13,7 @@ import (
"github.com/tus/tusd/pkg/handler"
)
//go:generate mockgen -package handler_test -source utils_test.go -aux_files tusd=datastore.go -destination=handler_mock_test.go
//go:generate mockgen -package handler_test -source utils_test.go -aux_files handler=datastore.go -destination=handler_mock_test.go
// FullDataStore is an interface combining most interfaces for data stores.
// This is used by mockgen(1) to generate a mocked data store used for testing
@ -25,11 +25,15 @@ type FullDataStore interface {
handler.DataStore
handler.TerminaterDataStore
handler.ConcaterDataStore
handler.GetReaderDataStore
handler.FinisherDataStore
handler.LengthDeferrerDataStore
}
type FullUpload interface {
handler.Upload
handler.TerminatableUpload
handler.LengthDeclarableUpload
}
type Locker interface {
handler.LockerDataStore
}