diff --git a/pkg/filestore/filestore.go b/pkg/filestore/filestore.go index 89a7d46..6f58007 100644 --- a/pkg/filestore/filestore.go +++ b/pkg/filestore/filestore.go @@ -6,13 +6,6 @@ // `[id]` files without an extension contain the raw binary data uploaded. // No cleanup is performed so you may want to run a cronjob to ensure your disk // is not filled up with old and finished uploads. -// -// In addition, it provides an exclusive upload locking mechanism using lock files -// which are stored on disk. Each of them stores the PID of the process which -// acquired the lock. This allows locks to be automatically freed when a process -// is unable to release it on its own because the process is not alive anymore. -// For more information, consult the documentation for handler.LockerDataStore -// interface, which is implemented by FileStore package filestore import ( @@ -25,8 +18,6 @@ import ( "github.com/tus/tusd/internal/uid" "github.com/tus/tusd/pkg/handler" - - "gopkg.in/Acconut/lockfile.v1" ) var defaultFilePerm = os.FileMode(0664) @@ -51,15 +42,13 @@ func New(path string) FileStore { // all possible extension to it. func (store FileStore) UseIn(composer *handler.StoreComposer) { composer.UseCore(store) - composer.UseGetReader(store) composer.UseTerminater(store) - composer.UseLocker(store) composer.UseConcater(store) composer.UseLengthDeferrer(store) } -func (store FileStore) NewUpload(info handler.FileInfo) (id string, err error) { - id = uid.Uid() +func (store FileStore) NewUpload(info handler.FileInfo) (handler.Upload, error) { + id := uid.Uid() binPath := store.binPath(id) info.ID = id info.Storage = map[string]string{ @@ -73,17 +62,84 @@ func (store FileStore) NewUpload(info handler.FileInfo) (id string, err error) { if os.IsNotExist(err) { err = fmt.Errorf("upload directory does not exist: %s", store.Path) } - return "", err + return nil, err } defer file.Close() + upload := &fileUpload{ + info: info, + infoPath: store.infoPath(id), + binPath: store.binPath(id), + } + // writeInfo creates the file by itself if necessary - err = store.writeInfo(id, info) - return + err = upload.writeInfo() + if err != nil { + return nil, err + } + + return upload, nil } -func (store FileStore) WriteChunk(id string, offset int64, src io.Reader) (int64, error) { - file, err := os.OpenFile(store.binPath(id), os.O_WRONLY|os.O_APPEND, defaultFilePerm) +func (store FileStore) GetUpload(id string) (handler.Upload, error) { + info := handler.FileInfo{} + data, err := ioutil.ReadFile(store.infoPath(id)) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + binPath := store.binPath(id) + infoPath := store.infoPath(id) + stat, err := os.Stat(binPath) + if err != nil { + return nil, err + } + + info.Offset = stat.Size() + + return &fileUpload{ + info: info, + binPath: binPath, + infoPath: infoPath, + }, nil +} + +func (store FileStore) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload { + return upload.(*fileUpload) +} + +func (store FileStore) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload { + return upload.(*fileUpload) +} + +// binPath returns the path to the file storing the binary data. +func (store FileStore) binPath(id string) string { + return filepath.Join(store.Path, id) +} + +// infoPath returns the path to the .info file storing the file's info. +func (store FileStore) infoPath(id string) string { + return filepath.Join(store.Path, id+".info") +} + +type fileUpload struct { + // info stores the current information about the upload + info handler.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string +} + +func (upload *fileUpload) GetInfo() (handler.FileInfo, error) { + return upload.info, nil +} + +func (upload *fileUpload) WriteChunk(offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) if err != nil { return 0, err } @@ -99,39 +155,20 @@ func (store FileStore) WriteChunk(id string, offset int64, src io.Reader) (int64 err = nil } + upload.info.Offset += n + return n, err } -func (store FileStore) GetInfo(id string) (handler.FileInfo, error) { - info := handler.FileInfo{} - data, err := ioutil.ReadFile(store.infoPath(id)) - if err != nil { - return info, err - } - if err := json.Unmarshal(data, &info); err != nil { - return info, err - } - - binPath := store.binPath(id) - stat, err := os.Stat(binPath) - if err != nil { - return info, err - } - - info.Offset = stat.Size() - - return info, nil +func (upload *fileUpload) GetReader() (io.Reader, error) { + return os.Open(upload.binPath) } -func (store FileStore) GetReader(id string) (io.Reader, error) { - return os.Open(store.binPath(id)) -} - -func (store FileStore) Terminate(id string) error { - if err := os.Remove(store.infoPath(id)); err != nil { +func (upload *fileUpload) Terminate() error { + if err := os.Remove(upload.infoPath); err != nil { return err } - if err := os.Remove(store.binPath(id)); err != nil { + if err := os.Remove(upload.binPath); err != nil { return err } return nil @@ -145,7 +182,7 @@ func (store FileStore) ConcatUploads(dest string, uploads []string) (err error) defer file.Close() for _, id := range uploads { - src, err := store.GetReader(id) + src, err := os.Open(store.binPath(id)) if err != nil { return err } @@ -158,76 +195,21 @@ func (store FileStore) ConcatUploads(dest string, uploads []string) (err error) return } -func (store FileStore) DeclareLength(id string, length int64) error { - info, err := store.GetInfo(id) - if err != nil { - return err - } - info.Size = length - info.SizeIsDeferred = false - return store.writeInfo(id, info) -} - -func (store FileStore) LockUpload(id string) error { - lock, err := store.newLock(id) - if err != nil { - return err - } - - err = lock.TryLock() - if err == lockfile.ErrBusy { - return handler.ErrFileLocked - } - - return err -} - -func (store FileStore) UnlockUpload(id string) error { - lock, err := store.newLock(id) - if err != nil { - return err - } - - err = lock.Unlock() - - // A "no such file or directory" will be returned if no lockfile was found. - // Since this means that the file has never been locked, we drop the error - // and continue as if nothing happened. - if os.IsNotExist(err) { - err = nil - } - - return err -} - -// newLock contructs a new Lockfile instance. -func (store FileStore) newLock(id string) (lockfile.Lockfile, error) { - path, err := filepath.Abs(filepath.Join(store.Path, id+".lock")) - if err != nil { - return lockfile.Lockfile(""), err - } - - // We use Lockfile directly instead of lockfile.New to bypass the unnecessary - // check whether the provided path is absolute since we just resolved it - // on our own. - return lockfile.Lockfile(path), nil -} - -// binPath returns the path to the file storing the binary data. -func (store FileStore) binPath(id string) string { - return filepath.Join(store.Path, id) -} - -// infoPath returns the path to the .info file storing the file's info. -func (store FileStore) infoPath(id string) string { - return filepath.Join(store.Path, id+".info") +func (upload *fileUpload) DeclareLength(length int64) error { + upload.info.Size = length + upload.info.SizeIsDeferred = false + return upload.writeInfo() } // writeInfo updates the entire information. Everything will be overwritten. -func (store FileStore) writeInfo(id string, info handler.FileInfo) error { - data, err := json.Marshal(info) +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) if err != nil { return err } - return ioutil.WriteFile(store.infoPath(id), data, defaultFilePerm) + return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm) +} + +func (upload *fileUpload) FinishUpload() error { + return nil } diff --git a/pkg/filestore/filestore_test.go b/pkg/filestore/filestore_test.go index fbf0fb0..10a0dd9 100644 --- a/pkg/filestore/filestore_test.go +++ b/pkg/filestore/filestore_test.go @@ -9,15 +9,12 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/tus/tusd/pkg/handler" ) // Test interface implementation of Filestore var _ handler.DataStore = FileStore{} -var _ handler.GetReaderDataStore = FileStore{} var _ handler.TerminaterDataStore = FileStore{} -var _ handler.LockerDataStore = FileStore{} var _ handler.ConcaterDataStore = FileStore{} var _ handler.LengthDeferrerDataStore = FileStore{} @@ -30,38 +27,38 @@ func TestFilestore(t *testing.T) { store := FileStore{tmp} // Create new upload - id, err := store.NewUpload(handler.FileInfo{ + upload, err := store.NewUpload(handler.FileInfo{ Size: 42, MetaData: map[string]string{ "hello": "world", }, }) a.NoError(err) - a.NotEqual("", id) + a.NotEqual(nil, upload) // Check info without writing - info, err := store.GetInfo(id) + info, err := upload.GetInfo() a.NoError(err) a.EqualValues(42, info.Size) a.EqualValues(0, info.Offset) a.Equal(handler.MetaData{"hello": "world"}, info.MetaData) a.Equal(2, len(info.Storage)) a.Equal("filestore", info.Storage["Type"]) - a.Equal(filepath.Join(tmp, id), info.Storage["Path"]) + a.Equal(filepath.Join(tmp, info.ID), info.Storage["Path"]) // Write data to upload - bytesWritten, err := store.WriteChunk(id, 0, strings.NewReader("hello world")) + bytesWritten, err := upload.WriteChunk(0, strings.NewReader("hello world")) a.NoError(err) a.EqualValues(len("hello world"), bytesWritten) // Check new offset - info, err = store.GetInfo(id) + info, err = upload.GetInfo() a.NoError(err) a.EqualValues(42, info.Size) a.EqualValues(11, info.Offset) // Read content - reader, err := store.GetReader(id) + reader, err := upload.GetReader() a.NoError(err) content, err := ioutil.ReadAll(reader) @@ -70,10 +67,11 @@ func TestFilestore(t *testing.T) { reader.(io.Closer).Close() // Terminate upload - a.NoError(store.Terminate(id)) + a.NoError(store.AsTerminatableUpload(upload).Terminate()) // Test if upload is deleted - _, err = store.GetInfo(id) + upload, err = store.GetUpload(info.ID) + a.Equal(nil, upload) a.True(os.IsNotExist(err)) } @@ -82,24 +80,10 @@ func TestMissingPath(t *testing.T) { store := FileStore{"./path-that-does-not-exist"} - id, err := store.NewUpload(handler.FileInfo{}) + upload, err := store.NewUpload(handler.FileInfo{}) a.Error(err) - a.Equal(err.Error(), "upload directory does not exist: ./path-that-does-not-exist") - a.Equal(id, "") -} - -func TestFileLocker(t *testing.T) { - a := assert.New(t) - - dir, err := ioutil.TempDir("", "tusd-file-locker") - a.NoError(err) - - var locker handler.LockerDataStore - locker = FileStore{dir} - - a.NoError(locker.LockUpload("one")) - a.Equal(handler.ErrFileLocked, locker.LockUpload("one")) - a.NoError(locker.UnlockUpload("one")) + a.Equal("upload directory does not exist: ./path-that-does-not-exist", err.Error()) + a.Equal(nil, upload) } func TestConcatUploads(t *testing.T) { @@ -111,9 +95,13 @@ func TestConcatUploads(t *testing.T) { store := FileStore{tmp} // Create new upload to hold concatenated upload - finId, err := store.NewUpload(handler.FileInfo{Size: 9}) + finUpload, err := store.NewUpload(handler.FileInfo{Size: 9}) a.NoError(err) - a.NotEqual("", finId) + a.NotEqual(nil, finUpload) + + finInfo, err := finUpload.GetInfo() + a.NoError(err) + finId := finInfo.ID // Create three uploads for concatenating ids := make([]string, 3) @@ -123,27 +111,33 @@ func TestConcatUploads(t *testing.T) { "ghi", } for i := 0; i < 3; i++ { - id, err := store.NewUpload(handler.FileInfo{Size: 3}) + upload, err := store.NewUpload(handler.FileInfo{Size: 3}) a.NoError(err) - n, err := store.WriteChunk(id, 0, strings.NewReader(contents[i])) + n, err := upload.WriteChunk(0, strings.NewReader(contents[i])) a.NoError(err) a.EqualValues(3, n) - ids[i] = id + info, err := upload.GetInfo() + a.NoError(err) + + ids[i] = info.ID } err = store.ConcatUploads(finId, ids) a.NoError(err) // Check offset - info, err := store.GetInfo(finId) + finUpload, err = store.GetUpload(finId) + a.NoError(err) + + info, err := finUpload.GetInfo() a.NoError(err) a.EqualValues(9, info.Size) a.EqualValues(9, info.Offset) // Read content - reader, err := store.GetReader(finId) + reader, err := finUpload.GetReader() a.NoError(err) content, err := ioutil.ReadAll(reader) @@ -160,19 +154,23 @@ func TestDeclareLength(t *testing.T) { store := FileStore{tmp} - originalInfo := handler.FileInfo{Size: 0, SizeIsDeferred: true} - id, err := store.NewUpload(originalInfo) + upload, err := store.NewUpload(handler.FileInfo{ + Size: 0, + SizeIsDeferred: true, + }) + a.NoError(err) + a.NotEqual(nil, upload) + + info, err := upload.GetInfo() + a.NoError(err) + a.EqualValues(0, info.Size) + a.Equal(true, info.SizeIsDeferred) + + err = store.AsLengthDeclarableUpload(upload).DeclareLength(100) a.NoError(err) - info, err := store.GetInfo(id) - a.Equal(info.Size, originalInfo.Size) - a.Equal(info.SizeIsDeferred, originalInfo.SizeIsDeferred) - - size := int64(100) - err = store.DeclareLength(id, size) + updatedInfo, err := upload.GetInfo() a.NoError(err) - - updatedInfo, err := store.GetInfo(id) - a.Equal(updatedInfo.Size, size) - a.False(updatedInfo.SizeIsDeferred) + a.EqualValues(100, updatedInfo.Size) + a.Equal(false, updatedInfo.SizeIsDeferred) }