From 82bc04cb36c4085d4c41566cfc36b3a9f40e907b Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Mon, 23 Apr 2018 17:07:51 -0400 Subject: [PATCH 01/23] Add SizeIsDeferred to FileInfo --- datastore.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datastore.go b/datastore.go index 588199a..b4c2f00 100644 --- a/datastore.go +++ b/datastore.go @@ -10,6 +10,8 @@ type FileInfo struct { ID string // Total file size in bytes specified in the NewUpload call Size int64 + // Indicates whether the total file size is deferred until later + SizeIsDeferred bool // Offset in bytes (zero-based) Offset int64 MetaData MetaData From 1affbbdbe4fddf2add7546a07a4431fe1bd1a915 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Mon, 23 Apr 2018 17:09:14 -0400 Subject: [PATCH 02/23] Add error constants for deferred upload length --- unrouted_handler.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/unrouted_handler.go b/unrouted_handler.go index c547fd2..f35adae 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -47,19 +47,21 @@ func NewHTTPError(err error, statusCode int) HTTPError { } var ( - ErrUnsupportedVersion = NewHTTPError(errors.New("unsupported version"), http.StatusPreconditionFailed) - ErrMaxSizeExceeded = NewHTTPError(errors.New("maximum size exceeded"), http.StatusRequestEntityTooLarge) - ErrInvalidContentType = NewHTTPError(errors.New("missing or invalid Content-Type header"), http.StatusBadRequest) - ErrInvalidUploadLength = NewHTTPError(errors.New("missing or invalid Upload-Length header"), http.StatusBadRequest) - ErrInvalidOffset = NewHTTPError(errors.New("missing or invalid Upload-Offset header"), http.StatusBadRequest) - ErrNotFound = NewHTTPError(errors.New("upload not found"), http.StatusNotFound) - ErrFileLocked = NewHTTPError(errors.New("file currently locked"), 423) // Locked (WebDAV) (RFC 4918) - ErrMismatchOffset = NewHTTPError(errors.New("mismatched offset"), http.StatusConflict) - ErrSizeExceeded = NewHTTPError(errors.New("resource's size exceeded"), http.StatusRequestEntityTooLarge) - ErrNotImplemented = NewHTTPError(errors.New("feature not implemented"), http.StatusNotImplemented) - ErrUploadNotFinished = NewHTTPError(errors.New("one of the partial uploads is not finished"), http.StatusBadRequest) - ErrInvalidConcat = NewHTTPError(errors.New("invalid Upload-Concat header"), http.StatusBadRequest) - ErrModifyFinal = NewHTTPError(errors.New("modifying a final upload is not allowed"), http.StatusForbidden) + ErrUnsupportedVersion = NewHTTPError(errors.New("unsupported version"), http.StatusPreconditionFailed) + ErrMaxSizeExceeded = NewHTTPError(errors.New("maximum size exceeded"), http.StatusRequestEntityTooLarge) + ErrInvalidContentType = NewHTTPError(errors.New("missing or invalid Content-Type header"), http.StatusBadRequest) + ErrInvalidUploadLength = NewHTTPError(errors.New("missing or invalid Upload-Length header"), http.StatusBadRequest) + ErrInvalidOffset = NewHTTPError(errors.New("missing or invalid Upload-Offset header"), http.StatusBadRequest) + ErrNotFound = NewHTTPError(errors.New("upload not found"), http.StatusNotFound) + ErrFileLocked = NewHTTPError(errors.New("file currently locked"), 423) // Locked (WebDAV) (RFC 4918) + ErrMismatchOffset = NewHTTPError(errors.New("mismatched offset"), http.StatusConflict) + ErrSizeExceeded = NewHTTPError(errors.New("resource's size exceeded"), http.StatusRequestEntityTooLarge) + ErrNotImplemented = NewHTTPError(errors.New("feature not implemented"), http.StatusNotImplemented) + ErrUploadNotFinished = NewHTTPError(errors.New("one of the partial uploads is not finished"), http.StatusBadRequest) + ErrInvalidConcat = NewHTTPError(errors.New("invalid Upload-Concat header"), http.StatusBadRequest) + ErrModifyFinal = NewHTTPError(errors.New("modifying a final upload is not allowed"), http.StatusForbidden) + ErrUploadLengthAndUploadDeferLength = NewHTTPError(errors.New("provided both Upload-Length and Upload-Defer-Length"), http.StatusBadRequest) + ErrInvalidUploadDeferLength = NewHTTPError(errors.New("invalid Upload-Defer-Length header"), http.StatusBadRequest) ) // UnroutedHandler exposes methods to handle requests as part of the tus protocol, From 1ab523164399dd06f174cc23bf0ed4fb2c99cc02 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Mon, 23 Apr 2018 17:10:23 -0400 Subject: [PATCH 03/23] Add basic Upload-Defer-Length header handling --- s3store/s3store_test.go | 4 ++-- unrouted_handler.go | 42 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/s3store/s3store_test.go b/s3store/s3store_test.go index f9729e0..ecd8f7f 100644 --- a/s3store/s3store_test.go +++ b/s3store/s3store_test.go @@ -52,8 +52,8 @@ func TestNewUpload(t *testing.T) { s3obj.EXPECT().PutObject(&s3.PutObjectInput{ Bucket: aws.String("bucket"), Key: aws.String("uploadId.info"), - Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)), - ContentLength: aws.Int64(int64(148)), + Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)), + ContentLength: aws.Int64(int64(171)), }), ) diff --git a/unrouted_handler.go b/unrouted_handler.go index f35adae..8376ca5 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -15,6 +15,8 @@ import ( "time" ) +const UploadLengthDeferred = "1" + var ( reExtractFileID = regexp.MustCompile(`([^/]+)\/?$`) reForwardedHost = regexp.MustCompile(`host=([^,]+)`) @@ -243,6 +245,7 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) // uploads the size is sum of all sizes of these files (no need for // Upload-Length header) var size int64 + var sizeIsDeferred bool if isFinal { // A final upload must not contain a chunk within the creation request if containsChunk { @@ -256,9 +259,11 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) return } } else { - size, err = strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) - if err != nil || size < 0 { - handler.sendError(w, r, ErrInvalidUploadLength) + uploadLengthHeader := r.Header.Get("Upload-Length") + uploadDeferLengthHeader := r.Header.Get("Upload-Defer-Length") + size, sizeIsDeferred, err = validateNewUploadLengthHeaders(uploadLengthHeader, uploadDeferLengthHeader) + if err != nil { + handler.sendError(w, r, err) return } } @@ -274,6 +279,7 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) info := FileInfo{ Size: size, + SizeIsDeferred: sizeIsDeferred, MetaData: meta, IsPartial: isPartial, IsFinal: isFinal, @@ -382,8 +388,13 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request) w.Header().Set("Upload-Metadata", SerializeMetadataHeader(info.MetaData)) } + if info.SizeIsDeferred { + w.Header().Set("Upload-Defer-Length", UploadLengthDeferred) + } else { + w.Header().Set("Upload-Length", strconv.FormatInt(info.Size, 10)) + } + w.Header().Set("Cache-Control", "no-store") - w.Header().Set("Upload-Length", strconv.FormatInt(info.Size, 10)) w.Header().Set("Upload-Offset", strconv.FormatInt(info.Offset, 10)) handler.sendResp(w, r, http.StatusOK) } @@ -849,6 +860,29 @@ func (handler *UnroutedHandler) sizeOfUploads(ids []string) (size int64, err err return } +// Verify that the Upload-Length and Upload-Defer-Length headers are acceptable for creating a +// new upload +func validateNewUploadLengthHeaders(uploadLengthHeader string, uploadDeferLengthHeader string) (uploadLength int64, uploadLengthDeferred bool, err error) { + haveBothLengthHeaders := uploadLengthHeader != "" && uploadDeferLengthHeader != "" + haveInvalidDeferHeader := uploadDeferLengthHeader != "" && uploadDeferLengthHeader != UploadLengthDeferred + lengthIsDeferred := uploadDeferLengthHeader == UploadLengthDeferred + + if haveBothLengthHeaders { + err = ErrUploadLengthAndUploadDeferLength + } else if haveInvalidDeferHeader { + err = ErrInvalidUploadDeferLength + } else if lengthIsDeferred { + uploadLengthDeferred = true + } else { + uploadLength, err = strconv.ParseInt(uploadLengthHeader, 10, 64) + if err != nil || uploadLength < 0 { + err = ErrInvalidUploadLength + } + } + + return +} + // ParseMetadataHeader parses the Upload-Metadata header as defined in the // File Creation extension. // e.g. Upload-Metadata: name bHVucmpzLnBuZw==,type aW1hZ2UvcG5n From c94e0cd0b6c9694ba3df5948a86f85303bd0226e Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Mon, 23 Apr 2018 17:13:35 -0400 Subject: [PATCH 04/23] Add creation-defer-length to supported extensions --- concat_test.go | 2 +- options_test.go | 2 +- terminate_test.go | 2 +- unrouted_handler.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/concat_test.go b/concat_test.go index e223c42..9b127ac 100644 --- a/concat_test.go +++ b/concat_test.go @@ -25,7 +25,7 @@ func TestConcat(t *testing.T) { Method: "OPTIONS", Code: http.StatusOK, ResHeader: map[string]string{ - "Tus-Extension": "creation,creation-with-upload,concatenation", + "Tus-Extension": "creation,creation-with-upload,creation-defer-length,concatenation", }, }).Run(handler, t) }) diff --git a/options_test.go b/options_test.go index 2194035..81b5170 100644 --- a/options_test.go +++ b/options_test.go @@ -20,7 +20,7 @@ func TestOptions(t *testing.T) { (&httpTest{ Method: "OPTIONS", ResHeader: map[string]string{ - "Tus-Extension": "creation,creation-with-upload", + "Tus-Extension": "creation,creation-with-upload,creation-defer-length", "Tus-Version": "1.0.0", "Tus-Resumable": "1.0.0", "Tus-Max-Size": "400", diff --git a/terminate_test.go b/terminate_test.go index 098d72c..046141e 100644 --- a/terminate_test.go +++ b/terminate_test.go @@ -24,7 +24,7 @@ func TestTerminate(t *testing.T) { Method: "OPTIONS", Code: http.StatusOK, ResHeader: map[string]string{ - "Tus-Extension": "creation,creation-with-upload,termination", + "Tus-Extension": "creation,creation-with-upload,creation-defer-length,termination", }, }).Run(handler, t) }) diff --git a/unrouted_handler.go b/unrouted_handler.go index 8376ca5..3769bdf 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -119,7 +119,7 @@ func NewUnroutedHandler(config Config) (*UnroutedHandler, error) { } // Only promote extesions using the Tus-Extension header which are implemented - extensions := "creation,creation-with-upload" + extensions := "creation,creation-with-upload,creation-defer-length" if config.StoreComposer.UsesTerminater { extensions += ",termination" } From 64c02a9de196153a0fc132a0a16739a83efbfb65 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Mon, 23 Apr 2018 17:16:20 -0400 Subject: [PATCH 05/23] Update the upload length in FileInfo when it's sent --- unrouted_handler.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/unrouted_handler.go b/unrouted_handler.go index 3769bdf..675519b 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -456,6 +456,17 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request return } + if info.SizeIsDeferred && r.Header.Get("Upload-Length") != "" { + uploadLength, err := strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) + if err != nil || uploadLength < 0 { + handler.sendError(w, r, ErrInvalidOffset) + return + } + + info.Size = uploadLength + info.SizeIsDeferred = false + } + if err := handler.writeChunk(id, info, w, r); err != nil { handler.sendError(w, r, err) return From 5252c98cd70f19662b4ac7e9a67bb482774bb764 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Mon, 23 Apr 2018 17:16:28 -0400 Subject: [PATCH 06/23] Avoid incorrectly returning errors when upload length is deferred --- s3store/s3store.go | 10 ++++++---- unrouted_handler.go | 8 ++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/s3store/s3store.go b/s3store/s3store.go index 3d0ad98..00224a6 100644 --- a/s3store/s3store.go +++ b/s3store/s3store.go @@ -271,12 +271,14 @@ func (store S3Store) WriteChunk(id string, offset int64, src io.Reader) (int64, return bytesUploaded, nil } - if (size - offset) <= optimalPartSize { - if (size - offset) != n { + if !info.SizeIsDeferred { + if (size - offset) <= optimalPartSize { + if (size - offset) != n { + return bytesUploaded, nil + } + } else if n < optimalPartSize { return bytesUploaded, nil } - } else if n < optimalPartSize { - return bytesUploaded, nil } // Seek to the beginning of the file diff --git a/unrouted_handler.go b/unrouted_handler.go index 675519b..cf60681 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -333,7 +333,7 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) handler.sendError(w, r, err) return } - } else if size == 0 { + } else if !sizeIsDeferred && size == 0 { // Directly finish the upload if the upload is empty (i.e. has a size of 0). // This statement is in an else-if block to avoid causing duplicate calls // to finishUploadIfComplete if an upload is empty and contains a chunk. @@ -450,7 +450,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request } // Do not proxy the call to the data store if the upload is already completed - if info.Offset == info.Size { + if !info.SizeIsDeferred && info.Offset == info.Size { w.Header().Set("Upload-Offset", strconv.FormatInt(offset, 10)) handler.sendResp(w, r, http.StatusNoContent) return @@ -484,7 +484,7 @@ func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.Resp offset := info.Offset // Test if this upload fits into the file's size - if offset+length > info.Size { + if !info.SizeIsDeferred && offset+length > info.Size { return ErrSizeExceeded } @@ -531,7 +531,7 @@ func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.Resp // function and send the necessary message on the CompleteUpload channel. func (handler *UnroutedHandler) finishUploadIfComplete(info FileInfo) error { // If the upload is completed, ... - if info.Offset == info.Size { + if !info.SizeIsDeferred && info.Offset == info.Size { // ... allow custom mechanism to finish and cleanup the upload if handler.composer.UsesFinisher { if err := handler.composer.Finisher.FinishUpload(info.ID); err != nil { From acaaa10ff7fa19a1d27c5c79d62a5b1478103b0a Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sat, 28 Apr 2018 15:39:40 -0400 Subject: [PATCH 07/23] Add POST tests --- post_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/post_test.go b/post_test.go index 639308d..53f88af 100644 --- a/post_test.go +++ b/post_test.go @@ -123,6 +123,54 @@ func TestPost(t *testing.T) { }).Run(handler, t) }) + SubTest(t, "UploadLengthAndUploadDeferLengthFail", func(t *testing.T, store *MockFullDataStore) { + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + (&httpTest{ + Method: "POST", + URL: "", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Upload-Length": "10", + "Upload-Defer-Length": "1", + }, + Code: http.StatusBadRequest, + }).Run(handler, t) + }) + + SubTest(t, "NeitherUploadLengthNorUploadDeferLengthFail", func(t *testing.T, store *MockFullDataStore) { + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + (&httpTest{ + Method: "POST", + URL: "", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + }, + Code: http.StatusBadRequest, + }).Run(handler, t) + }) + + SubTest(t, "InvalidUploadDeferLengthFail", func(t *testing.T, store *MockFullDataStore) { + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + (&httpTest{ + Method: "POST", + URL: "", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Upload-Defer-Length": "bad", + }, + Code: http.StatusBadRequest, + }).Run(handler, t) + }) + SubTest(t, "ForwardHeaders", func(t *testing.T, store *MockFullDataStore) { SubTest(t, "IgnoreXForwarded", func(t *testing.T, store *MockFullDataStore) { store.EXPECT().NewUpload(FileInfo{ From 77a8455a675566369e349c0801c3eb6b0ff3d635 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sat, 28 Apr 2018 15:39:46 -0400 Subject: [PATCH 08/23] Add HEAD tests --- head_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/head_test.go b/head_test.go index 15b4099..021507b 100644 --- a/head_test.go +++ b/head_test.go @@ -81,4 +81,52 @@ func TestHead(t *testing.T) { t.Errorf("Expected empty body for failed HEAD request") } }) + + SubTest(t, "DeferLengthHeader", func(t *testing.T, store *MockFullDataStore) { + store.EXPECT().GetInfo("yes").Return(FileInfo{ + SizeIsDeferred: true, + Size: 0, + }, nil) + + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + (&httpTest{ + Method: "HEAD", + URL: "yes", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + }, + Code: http.StatusOK, + ResHeader: map[string]string{ + "Upload-Defer-Length": "1", + }, + }).Run(handler, t) + }) + + SubTest(t, "NoDeferLengthHeader", func(t *testing.T, store *MockFullDataStore) { + gomock.InOrder( + store.EXPECT().GetInfo("yes").Return(FileInfo{ + SizeIsDeferred: false, + Size: 10, + }, nil), + ) + + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + (&httpTest{ + Method: "HEAD", + URL: "yes", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + }, + Code: http.StatusOK, + ResHeader: map[string]string{ + "Upload-Defer-Length": "", + }, + }).Run(handler, t) + }) } From 55f99cb34abfad0036e6a2c5b2e01bd76ebfec99 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sat, 5 May 2018 11:37:56 -0400 Subject: [PATCH 09/23] Add LengthDeferrerDataStore interface --- datastore.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/datastore.go b/datastore.go index b4c2f00..c6e4249 100644 --- a/datastore.go +++ b/datastore.go @@ -114,3 +114,11 @@ type ConcaterDataStore interface { // must be respected during concatenation. ConcatUploads(destination string, partialUploads []string) error } + +// LengthDeferrerDataStore is the interface that must be implemented if the +// creation-defer-length extension should be enabled. The extension enables a +// client to upload files when their total size is not yet known. Instead, the +// client must send the total size as soon as it becomes known. +type LengthDeferrerDataStore interface { + DeclareLength(id string, length int64) error +} From d40c50f80a7e2dfb67daa7078893ae298299af97 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sun, 13 May 2018 10:27:38 -0400 Subject: [PATCH 10/23] Add length deferrer support to composer --- composer.go | 36 ++++++++++++++++++++++++++---------- composer.mgo | 4 ++++ 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/composer.go b/composer.go index f15b38a..9ec8aa2 100644 --- a/composer.go +++ b/composer.go @@ -6,16 +6,18 @@ package tusd type StoreComposer struct { Core DataStore - UsesTerminater bool - Terminater TerminaterDataStore - UsesFinisher bool - Finisher FinisherDataStore - UsesLocker bool - Locker LockerDataStore - UsesGetReader bool - GetReader GetReaderDataStore - UsesConcater bool - Concater ConcaterDataStore + UsesTerminater bool + Terminater TerminaterDataStore + UsesFinisher bool + Finisher FinisherDataStore + UsesLocker bool + Locker LockerDataStore + UsesGetReader bool + GetReader GetReaderDataStore + UsesConcater bool + Concater ConcaterDataStore + UsesLengthDeferrer bool + LengthDeferrer LengthDeferrerDataStore } // NewStoreComposer creates a new and empty store composer. @@ -45,6 +47,9 @@ func newStoreComposerFromDataStore(store DataStore) *StoreComposer { if mod, ok := store.(ConcaterDataStore); ok { composer.UseConcater(mod) } + if mod, ok := store.(LengthDeferrerDataStore); ok { + composer.UseLengthDeferrer(mod) + } return composer } @@ -90,6 +95,12 @@ func (store *StoreComposer) Capabilities() string { } else { str += "✗" } + str += ` LengthDeferrer: ` + if store.UsesLengthDeferrer { + str += "✓" + } else { + str += "✗" + } return str } @@ -120,3 +131,8 @@ func (store *StoreComposer) UseConcater(ext ConcaterDataStore) { store.UsesConcater = ext != nil store.Concater = ext } + +func (store *StoreComposer) UseLengthDeferrer(ext LengthDeferrerDataStore) { + store.UsesLengthDeferrer = ext != nil + store.LengthDeferrer = ext +} diff --git a/composer.mgo b/composer.mgo index 32db7ea..247cdb8 100644 --- a/composer.mgo +++ b/composer.mgo @@ -31,6 +31,7 @@ type StoreComposer struct { USE_FIELD(Locker) USE_FIELD(GetReader) USE_FIELD(Concater) + USE_FIELD(LengthDeferrer) } // NewStoreComposer creates a new and empty store composer. @@ -50,6 +51,7 @@ func newStoreComposerFromDataStore(store DataStore) *StoreComposer { USE_FROM(Locker) USE_FROM(GetReader) USE_FROM(Concater) + USE_FROM(LengthDeferrer) return composer } @@ -70,6 +72,7 @@ func (store *StoreComposer) Capabilities() string { USE_CAP(Locker) USE_CAP(GetReader) USE_CAP(Concater) + USE_CAP(LengthDeferrer) return str } @@ -85,3 +88,4 @@ USE_FUNC(Finisher) USE_FUNC(Locker) USE_FUNC(GetReader) USE_FUNC(Concater) +USE_FUNC(LengthDeferrer) From 5bf2133b3c493c9210b94040ab15c66261129d6a Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sun, 13 May 2018 10:28:02 -0400 Subject: [PATCH 11/23] Add length deferrer support to FileStore --- filestore/filestore.go | 11 +++++++++++ filestore/filestore_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/filestore/filestore.go b/filestore/filestore.go index 23a1618..4944dbd 100644 --- a/filestore/filestore.go +++ b/filestore/filestore.go @@ -55,6 +55,7 @@ func (store FileStore) UseIn(composer *tusd.StoreComposer) { composer.UseTerminater(store) composer.UseLocker(store) composer.UseConcater(store) + composer.UseLengthDeferrer(store) } func (store FileStore) NewUpload(info tusd.FileInfo) (id string, err error) { @@ -142,6 +143,16 @@ func (store FileStore) ConcatUploads(dest string, uploads []string) (err error) return } +func (store FileStore) DeclareLength(id string, length int64) error { + info, err := store.GetInfo(id) + if err != nil { + return err + } + info.Size = length + info.SizeIsDeferred = false + return store.writeInfo(id, info) +} + func (store FileStore) LockUpload(id string) error { lock, err := store.newLock(id) if err != nil { diff --git a/filestore/filestore_test.go b/filestore/filestore_test.go index 69c99c8..5f4c27d 100644 --- a/filestore/filestore_test.go +++ b/filestore/filestore_test.go @@ -18,6 +18,7 @@ var _ tusd.GetReaderDataStore = FileStore{} var _ tusd.TerminaterDataStore = FileStore{} var _ tusd.LockerDataStore = FileStore{} var _ tusd.ConcaterDataStore = FileStore{} +var _ tusd.LengthDeferrerDataStore = FileStore{} func TestFilestore(t *testing.T) { a := assert.New(t) @@ -146,3 +147,28 @@ func TestConcatUploads(t *testing.T) { a.Equal("abcdefghi", string(content)) reader.(io.Closer).Close() } + +func TestDeclareLength(t *testing.T) { + a := assert.New(t) + + tmp, err := ioutil.TempDir("", "tusd-filestore-declare-length-") + a.NoError(err) + + store := FileStore{tmp} + + originalInfo := tusd.FileInfo{Size: 0, SizeIsDeferred: true} + id, err := store.NewUpload(originalInfo) + a.NoError(err) + + info, err := store.GetInfo(id) + a.Equal(info.Size, originalInfo.Size) + a.Equal(info.SizeIsDeferred, originalInfo.SizeIsDeferred) + + size := int64(100) + err = store.DeclareLength(id, size) + a.NoError(err) + + updatedInfo, err := store.GetInfo(id) + a.Equal(updatedInfo.Size, size) + a.False(updatedInfo.SizeIsDeferred) +} From cb96d06350c9550c4db805c66b3ace0680ab293c Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sun, 13 May 2018 10:28:13 -0400 Subject: [PATCH 12/23] Add length deferrer support to FullDataStore --- utils_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/utils_test.go b/utils_test.go index 93d96ce..550e87e 100644 --- a/utils_test.go +++ b/utils_test.go @@ -27,6 +27,7 @@ type FullDataStore interface { tusd.ConcaterDataStore tusd.GetReaderDataStore tusd.FinisherDataStore + tusd.LengthDeferrerDataStore } type Locker interface { From 326986cc65f68d7a0f141512aedffbed80ff829c Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sat, 5 May 2018 15:18:47 -0400 Subject: [PATCH 13/23] Regenerate handler mock --- handler_mock_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/handler_mock_test.go b/handler_mock_test.go index ef6f8c6..53c256a 100644 --- a/handler_mock_test.go +++ b/handler_mock_test.go @@ -4,10 +4,9 @@ package tusd_test import ( - io "io" - gomock "github.com/golang/mock/gomock" tusd "github.com/tus/tusd" + io "io" ) // Mock of FullDataStore interface @@ -105,6 +104,16 @@ func (_mr *_MockFullDataStoreRecorder) FinishUpload(arg0 interface{}) *gomock.Ca return _mr.mock.ctrl.RecordCall(_mr.mock, "FinishUpload", arg0) } +func (_m *MockFullDataStore) DeclareLength(id string, length int64) error { + ret := _m.ctrl.Call(_m, "DeclareLength", id, length) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockFullDataStoreRecorder) DeclareLength(arg0, arg1 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "DeclareLength", arg0, arg1) +} + // Mock of Locker interface type MockLocker struct { ctrl *gomock.Controller From c6d5ad08c00d75e7f06b57232abf7ba68335506b Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sat, 5 May 2018 15:20:26 -0400 Subject: [PATCH 14/23] When it's requested, verify length deferrer support in the datastore --- unrouted_handler.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/unrouted_handler.go b/unrouted_handler.go index cf60681..481f360 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -261,7 +261,7 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) } else { uploadLengthHeader := r.Header.Get("Upload-Length") uploadDeferLengthHeader := r.Header.Get("Upload-Defer-Length") - size, sizeIsDeferred, err = validateNewUploadLengthHeaders(uploadLengthHeader, uploadDeferLengthHeader) + size, sizeIsDeferred, err = handler.validateNewUploadLengthHeaders(uploadLengthHeader, uploadDeferLengthHeader) if err != nil { handler.sendError(w, r, err) return @@ -873,12 +873,14 @@ func (handler *UnroutedHandler) sizeOfUploads(ids []string) (size int64, err err // Verify that the Upload-Length and Upload-Defer-Length headers are acceptable for creating a // new upload -func validateNewUploadLengthHeaders(uploadLengthHeader string, uploadDeferLengthHeader string) (uploadLength int64, uploadLengthDeferred bool, err error) { +func (handler *UnroutedHandler) validateNewUploadLengthHeaders(uploadLengthHeader string, uploadDeferLengthHeader string) (uploadLength int64, uploadLengthDeferred bool, err error) { haveBothLengthHeaders := uploadLengthHeader != "" && uploadDeferLengthHeader != "" haveInvalidDeferHeader := uploadDeferLengthHeader != "" && uploadDeferLengthHeader != UploadLengthDeferred lengthIsDeferred := uploadDeferLengthHeader == UploadLengthDeferred - if haveBothLengthHeaders { + if lengthIsDeferred && !handler.composer.UsesLengthDeferrer { + err = ErrNotImplemented + } else if haveBothLengthHeaders { err = ErrUploadLengthAndUploadDeferLength } else if haveInvalidDeferHeader { err = ErrInvalidUploadDeferLength From 7bd1cd6f9dee154ffeb5de3ce3067e035610fa29 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sun, 13 May 2018 09:52:27 -0400 Subject: [PATCH 15/23] Integrate DeclareLength into handler's PatchFile --- unrouted_handler.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/unrouted_handler.go b/unrouted_handler.go index 481f360..3c0275b 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -456,7 +456,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request return } - if info.SizeIsDeferred && r.Header.Get("Upload-Length") != "" { + if handler.composer.UsesLengthDeferrer && info.SizeIsDeferred && r.Header.Get("Upload-Length") != "" { uploadLength, err := strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) if err != nil || uploadLength < 0 { handler.sendError(w, r, ErrInvalidOffset) @@ -465,6 +465,10 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request info.Size = uploadLength info.SizeIsDeferred = false + if err := handler.composer.LengthDeferrer.DeclareLength(id, info.Size); err != nil { + handler.sendError(w, r, err) + return + } } if err := handler.writeChunk(id, info, w, r); err != nil { From 09768639477f1b56f7f67b54cf1e195c7ca56330 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sun, 13 May 2018 09:53:35 -0400 Subject: [PATCH 16/23] Add DeclareLength tests for PATCH requests --- patch_test.go | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/patch_test.go b/patch_test.go index c62d4b1..9fa2fab 100644 --- a/patch_test.go +++ b/patch_test.go @@ -257,6 +257,129 @@ func TestPatch(t *testing.T) { }).Run(handler, t) }) + SubTest(t, "DeclareLengthOnFinalChunk", func(t *testing.T, store *MockFullDataStore) { + gomock.InOrder( + store.EXPECT().GetInfo("yes").Return(FileInfo{ + ID: "yes", + Offset: 5, + Size: 0, + SizeIsDeferred: true, + }, nil), + store.EXPECT().DeclareLength("yes", int64(20)), + store.EXPECT().WriteChunk("yes", int64(5), NewReaderMatcher("hellothisismore")).Return(int64(15), nil), + store.EXPECT().FinishUpload("yes"), + ) + + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + body := strings.NewReader("hellothisismore") + + (&httpTest{ + Method: "PATCH", + URL: "yes", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Content-Type": "application/offset+octet-stream", + "Upload-Offset": "5", + "Upload-Length": "20", + }, + ReqBody: body, + Code: http.StatusNoContent, + ResHeader: map[string]string{ + "Upload-Offset": "20", + }, + }).Run(handler, t) + }) + + SubTest(t, "DeclareLengthAfterFinalChunk", func(t *testing.T, store *MockFullDataStore) { + gomock.InOrder( + store.EXPECT().GetInfo("yes").Return(FileInfo{ + ID: "yes", + Offset: 20, + Size: 0, + SizeIsDeferred: true, + }, nil), + store.EXPECT().DeclareLength("yes", int64(20)), + store.EXPECT().FinishUpload("yes"), + ) + + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + (&httpTest{ + Method: "PATCH", + URL: "yes", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Content-Type": "application/offset+octet-stream", + "Upload-Offset": "20", + "Upload-Length": "20", + }, + ReqBody: nil, + Code: http.StatusNoContent, + ResHeader: map[string]string{}, + }).Run(handler, t) + }) + + SubTest(t, "DeclareLengthOnNonFinalChunk", func(t *testing.T, store *MockFullDataStore) { + gomock.InOrder( + store.EXPECT().GetInfo("yes").Return(FileInfo{ + ID: "yes", + Offset: 5, + Size: 0, + SizeIsDeferred: true, + }, nil), + store.EXPECT().DeclareLength("yes", int64(20)), + store.EXPECT().WriteChunk("yes", int64(5), NewReaderMatcher("hello")).Return(int64(5), nil), + store.EXPECT().GetInfo("yes").Return(FileInfo{ + ID: "yes", + Offset: 10, + Size: 20, + SizeIsDeferred: false, + }, nil), + store.EXPECT().WriteChunk("yes", int64(10), NewReaderMatcher("thisismore")).Return(int64(10), nil), + store.EXPECT().FinishUpload("yes"), + ) + + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + (&httpTest{ + Method: "PATCH", + URL: "yes", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Content-Type": "application/offset+octet-stream", + "Upload-Offset": "5", + "Upload-Length": "20", + }, + ReqBody: strings.NewReader("hello"), + Code: http.StatusNoContent, + ResHeader: map[string]string{ + "Upload-Offset": "10", + }, + }).Run(handler, t) + + (&httpTest{ + Method: "PATCH", + URL: "yes", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Content-Type": "application/offset+octet-stream", + "Upload-Offset": "10", + }, + ReqBody: strings.NewReader("thisismore"), + Code: http.StatusNoContent, + ResHeader: map[string]string{ + "Upload-Offset": "20", + }, + }).Run(handler, t) + }) + SubTest(t, "Locker", func(t *testing.T, store *MockFullDataStore) { ctrl := gomock.NewController(t) defer ctrl.Finish() From d545b6518bced4d08fee5c21fda0dfafc33cee0a Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sun, 3 Jun 2018 12:15:50 -0400 Subject: [PATCH 17/23] Conditionally include creation-defer-length if the composer supports it --- unrouted_handler.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unrouted_handler.go b/unrouted_handler.go index 3c0275b..e8a52e2 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -119,13 +119,16 @@ func NewUnroutedHandler(config Config) (*UnroutedHandler, error) { } // Only promote extesions using the Tus-Extension header which are implemented - extensions := "creation,creation-with-upload,creation-defer-length" + extensions := "creation,creation-with-upload" if config.StoreComposer.UsesTerminater { extensions += ",termination" } if config.StoreComposer.UsesConcater { extensions += ",concatenation" } + if config.StoreComposer.UsesLengthDeferrer { + extensions += ",creation-defer-length" + } handler := &UnroutedHandler{ config: config, From 22fdd3935bb0ed4aaf62c689dc0df94e74955a51 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sun, 3 Jun 2018 12:16:51 -0400 Subject: [PATCH 18/23] Return upload length error if it's too small or too large --- unrouted_handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unrouted_handler.go b/unrouted_handler.go index e8a52e2..2b9411a 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -461,8 +461,8 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request if handler.composer.UsesLengthDeferrer && info.SizeIsDeferred && r.Header.Get("Upload-Length") != "" { uploadLength, err := strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) - if err != nil || uploadLength < 0 { - handler.sendError(w, r, ErrInvalidOffset) + if err != nil || uploadLength < 0 || uploadLength < info.Offset || uploadLength > handler.config.MaxSize { + handler.sendError(w, r, ErrInvalidUploadLength) return } From a366b00e3e251fb5b2dd786391cfd81642df5261 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sun, 3 Jun 2018 12:24:51 -0400 Subject: [PATCH 19/23] Return appropriate errors if upload-length is misused in PATCH requests --- unrouted_handler.go | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/unrouted_handler.go b/unrouted_handler.go index 2b9411a..dd5acab 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -459,18 +459,26 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request return } - if handler.composer.UsesLengthDeferrer && info.SizeIsDeferred && r.Header.Get("Upload-Length") != "" { - uploadLength, err := strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) - if err != nil || uploadLength < 0 || uploadLength < info.Offset || uploadLength > handler.config.MaxSize { - handler.sendError(w, r, ErrInvalidUploadLength) - return - } + if r.Header.Get("Upload-Length") != "" { + if handler.composer.UsesLengthDeferrer { + if info.SizeIsDeferred { + uploadLength, err := strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) + if err != nil || uploadLength < 0 || uploadLength < info.Offset || uploadLength > handler.config.MaxSize { + handler.sendError(w, r, ErrInvalidUploadLength) + return + } - info.Size = uploadLength - info.SizeIsDeferred = false - if err := handler.composer.LengthDeferrer.DeclareLength(id, info.Size); err != nil { - handler.sendError(w, r, err) - return + info.Size = uploadLength + info.SizeIsDeferred = false + if err := handler.composer.LengthDeferrer.DeclareLength(id, info.Size); err != nil { + handler.sendError(w, r, err) + return + } + } else { + handler.sendError(w, r, ErrInvalidUploadLength) + } + } else { + handler.sendError(w, r, ErrNotImplemented) } } From 8c3ceebcfcaf622ecbbb5f6434cfba62585a886b Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sun, 3 Jun 2018 12:37:45 -0400 Subject: [PATCH 20/23] If size is deferred, then constrain how much of the request body to read --- unrouted_handler.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/unrouted_handler.go b/unrouted_handler.go index dd5acab..da07812 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -5,6 +5,7 @@ import ( "errors" "io" "log" + "math" "net" "net/http" "os" @@ -504,6 +505,18 @@ func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.Resp } maxSize := info.Size - offset + // If the upload's length is deferred and the PATCH request does not contain the Content-Length + // header (which is allowed if 'Transfer-Encoding: chunked' is used), we still need to set limits for + // the body size. + if info.SizeIsDeferred { + if handler.config.MaxSize > 0 { + // Ensure that the upload does not exceed the maximum upload size + maxSize = handler.config.MaxSize - offset + } else { + // If no upload limit is given, we allow arbitrary sizes + maxSize = math.MaxInt64 + } + } if length > 0 { maxSize = length } From a74c46816dd4d87c32a31b5cddcf79201ca6f981 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sun, 3 Jun 2018 12:54:21 -0400 Subject: [PATCH 21/23] Remove creation-defer-length declaration from tests that don't use it --- concat_test.go | 2 +- options_test.go | 2 +- terminate_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/concat_test.go b/concat_test.go index 9b127ac..e223c42 100644 --- a/concat_test.go +++ b/concat_test.go @@ -25,7 +25,7 @@ func TestConcat(t *testing.T) { Method: "OPTIONS", Code: http.StatusOK, ResHeader: map[string]string{ - "Tus-Extension": "creation,creation-with-upload,creation-defer-length,concatenation", + "Tus-Extension": "creation,creation-with-upload,concatenation", }, }).Run(handler, t) }) diff --git a/options_test.go b/options_test.go index 81b5170..2194035 100644 --- a/options_test.go +++ b/options_test.go @@ -20,7 +20,7 @@ func TestOptions(t *testing.T) { (&httpTest{ Method: "OPTIONS", ResHeader: map[string]string{ - "Tus-Extension": "creation,creation-with-upload,creation-defer-length", + "Tus-Extension": "creation,creation-with-upload", "Tus-Version": "1.0.0", "Tus-Resumable": "1.0.0", "Tus-Max-Size": "400", diff --git a/terminate_test.go b/terminate_test.go index 046141e..098d72c 100644 --- a/terminate_test.go +++ b/terminate_test.go @@ -24,7 +24,7 @@ func TestTerminate(t *testing.T) { Method: "OPTIONS", Code: http.StatusOK, ResHeader: map[string]string{ - "Tus-Extension": "creation,creation-with-upload,creation-defer-length,termination", + "Tus-Extension": "creation,creation-with-upload,termination", }, }).Run(handler, t) }) From 750d4d5eb62d221c2d4ae90386950d0315e1228e Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sun, 3 Jun 2018 12:54:40 -0400 Subject: [PATCH 22/23] Specify MaxSize in PATCH tests that refer to it --- patch_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/patch_test.go b/patch_test.go index 9fa2fab..a2f85c0 100644 --- a/patch_test.go +++ b/patch_test.go @@ -272,6 +272,7 @@ func TestPatch(t *testing.T) { handler, _ := NewHandler(Config{ DataStore: store, + MaxSize: 20, }) body := strings.NewReader("hellothisismore") @@ -307,6 +308,7 @@ func TestPatch(t *testing.T) { handler, _ := NewHandler(Config{ DataStore: store, + MaxSize: 20, }) (&httpTest{ @@ -346,6 +348,7 @@ func TestPatch(t *testing.T) { handler, _ := NewHandler(Config{ DataStore: store, + MaxSize: 20, }) (&httpTest{ From c605926ff80d28c7ab29343b63e77d5bf38949f7 Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Sun, 3 Jun 2018 13:07:07 -0400 Subject: [PATCH 23/23] If a partial upload has a deferred size, then it's not finished --- unrouted_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unrouted_handler.go b/unrouted_handler.go index da07812..4cc4e23 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -888,7 +888,7 @@ func (handler *UnroutedHandler) sizeOfUploads(ids []string) (size int64, err err return size, err } - if info.Offset != info.Size { + if info.SizeIsDeferred || info.Offset != info.Size { err = ErrUploadNotFinished return size, err }