diff --git a/composer.go b/composer.go index f15b38a..9ec8aa2 100644 --- a/composer.go +++ b/composer.go @@ -6,16 +6,18 @@ package tusd type StoreComposer struct { Core DataStore - UsesTerminater bool - Terminater TerminaterDataStore - UsesFinisher bool - Finisher FinisherDataStore - UsesLocker bool - Locker LockerDataStore - UsesGetReader bool - GetReader GetReaderDataStore - UsesConcater bool - Concater ConcaterDataStore + UsesTerminater bool + Terminater TerminaterDataStore + UsesFinisher bool + Finisher FinisherDataStore + UsesLocker bool + Locker LockerDataStore + UsesGetReader bool + GetReader GetReaderDataStore + UsesConcater bool + Concater ConcaterDataStore + UsesLengthDeferrer bool + LengthDeferrer LengthDeferrerDataStore } // NewStoreComposer creates a new and empty store composer. @@ -45,6 +47,9 @@ func newStoreComposerFromDataStore(store DataStore) *StoreComposer { if mod, ok := store.(ConcaterDataStore); ok { composer.UseConcater(mod) } + if mod, ok := store.(LengthDeferrerDataStore); ok { + composer.UseLengthDeferrer(mod) + } return composer } @@ -90,6 +95,12 @@ func (store *StoreComposer) Capabilities() string { } else { str += "✗" } + str += ` LengthDeferrer: ` + if store.UsesLengthDeferrer { + str += "✓" + } else { + str += "✗" + } return str } @@ -120,3 +131,8 @@ func (store *StoreComposer) UseConcater(ext ConcaterDataStore) { store.UsesConcater = ext != nil store.Concater = ext } + +func (store *StoreComposer) UseLengthDeferrer(ext LengthDeferrerDataStore) { + store.UsesLengthDeferrer = ext != nil + store.LengthDeferrer = ext +} diff --git a/composer.mgo b/composer.mgo index 32db7ea..247cdb8 100644 --- a/composer.mgo +++ b/composer.mgo @@ -31,6 +31,7 @@ type StoreComposer struct { USE_FIELD(Locker) USE_FIELD(GetReader) USE_FIELD(Concater) + USE_FIELD(LengthDeferrer) } // NewStoreComposer creates a new and empty store composer. @@ -50,6 +51,7 @@ func newStoreComposerFromDataStore(store DataStore) *StoreComposer { USE_FROM(Locker) USE_FROM(GetReader) USE_FROM(Concater) + USE_FROM(LengthDeferrer) return composer } @@ -70,6 +72,7 @@ func (store *StoreComposer) Capabilities() string { USE_CAP(Locker) USE_CAP(GetReader) USE_CAP(Concater) + USE_CAP(LengthDeferrer) return str } @@ -85,3 +88,4 @@ USE_FUNC(Finisher) USE_FUNC(Locker) USE_FUNC(GetReader) USE_FUNC(Concater) +USE_FUNC(LengthDeferrer) diff --git a/datastore.go b/datastore.go index 588199a..c6e4249 100644 --- a/datastore.go +++ b/datastore.go @@ -10,6 +10,8 @@ type FileInfo struct { ID string // Total file size in bytes specified in the NewUpload call Size int64 + // Indicates whether the total file size is deferred until later + SizeIsDeferred bool // Offset in bytes (zero-based) Offset int64 MetaData MetaData @@ -112,3 +114,11 @@ type ConcaterDataStore interface { // must be respected during concatenation. ConcatUploads(destination string, partialUploads []string) error } + +// LengthDeferrerDataStore is the interface that must be implemented if the +// creation-defer-length extension should be enabled. The extension enables a +// client to upload files when their total size is not yet known. Instead, the +// client must send the total size as soon as it becomes known. +type LengthDeferrerDataStore interface { + DeclareLength(id string, length int64) error +} diff --git a/filestore/filestore.go b/filestore/filestore.go index 23a1618..4944dbd 100644 --- a/filestore/filestore.go +++ b/filestore/filestore.go @@ -55,6 +55,7 @@ func (store FileStore) UseIn(composer *tusd.StoreComposer) { composer.UseTerminater(store) composer.UseLocker(store) composer.UseConcater(store) + composer.UseLengthDeferrer(store) } func (store FileStore) NewUpload(info tusd.FileInfo) (id string, err error) { @@ -142,6 +143,16 @@ func (store FileStore) ConcatUploads(dest string, uploads []string) (err error) return } +func (store FileStore) DeclareLength(id string, length int64) error { + info, err := store.GetInfo(id) + if err != nil { + return err + } + info.Size = length + info.SizeIsDeferred = false + return store.writeInfo(id, info) +} + func (store FileStore) LockUpload(id string) error { lock, err := store.newLock(id) if err != nil { diff --git a/filestore/filestore_test.go b/filestore/filestore_test.go index 69c99c8..5f4c27d 100644 --- a/filestore/filestore_test.go +++ b/filestore/filestore_test.go @@ -18,6 +18,7 @@ var _ tusd.GetReaderDataStore = FileStore{} var _ tusd.TerminaterDataStore = FileStore{} var _ tusd.LockerDataStore = FileStore{} var _ tusd.ConcaterDataStore = FileStore{} +var _ tusd.LengthDeferrerDataStore = FileStore{} func TestFilestore(t *testing.T) { a := assert.New(t) @@ -146,3 +147,28 @@ func TestConcatUploads(t *testing.T) { a.Equal("abcdefghi", string(content)) reader.(io.Closer).Close() } + +func TestDeclareLength(t *testing.T) { + a := assert.New(t) + + tmp, err := ioutil.TempDir("", "tusd-filestore-declare-length-") + a.NoError(err) + + store := FileStore{tmp} + + originalInfo := tusd.FileInfo{Size: 0, SizeIsDeferred: true} + id, err := store.NewUpload(originalInfo) + a.NoError(err) + + info, err := store.GetInfo(id) + a.Equal(info.Size, originalInfo.Size) + a.Equal(info.SizeIsDeferred, originalInfo.SizeIsDeferred) + + size := int64(100) + err = store.DeclareLength(id, size) + a.NoError(err) + + updatedInfo, err := store.GetInfo(id) + a.Equal(updatedInfo.Size, size) + a.False(updatedInfo.SizeIsDeferred) +} diff --git a/handler_mock_test.go b/handler_mock_test.go index ef6f8c6..53c256a 100644 --- a/handler_mock_test.go +++ b/handler_mock_test.go @@ -4,10 +4,9 @@ package tusd_test import ( - io "io" - gomock "github.com/golang/mock/gomock" tusd "github.com/tus/tusd" + io "io" ) // Mock of FullDataStore interface @@ -105,6 +104,16 @@ func (_mr *_MockFullDataStoreRecorder) FinishUpload(arg0 interface{}) *gomock.Ca return _mr.mock.ctrl.RecordCall(_mr.mock, "FinishUpload", arg0) } +func (_m *MockFullDataStore) DeclareLength(id string, length int64) error { + ret := _m.ctrl.Call(_m, "DeclareLength", id, length) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockFullDataStoreRecorder) DeclareLength(arg0, arg1 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "DeclareLength", arg0, arg1) +} + // Mock of Locker interface type MockLocker struct { ctrl *gomock.Controller diff --git a/head_test.go b/head_test.go index 15b4099..021507b 100644 --- a/head_test.go +++ b/head_test.go @@ -81,4 +81,52 @@ func TestHead(t *testing.T) { t.Errorf("Expected empty body for failed HEAD request") } }) + + SubTest(t, "DeferLengthHeader", func(t *testing.T, store *MockFullDataStore) { + store.EXPECT().GetInfo("yes").Return(FileInfo{ + SizeIsDeferred: true, + Size: 0, + }, nil) + + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + (&httpTest{ + Method: "HEAD", + URL: "yes", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + }, + Code: http.StatusOK, + ResHeader: map[string]string{ + "Upload-Defer-Length": "1", + }, + }).Run(handler, t) + }) + + SubTest(t, "NoDeferLengthHeader", func(t *testing.T, store *MockFullDataStore) { + gomock.InOrder( + store.EXPECT().GetInfo("yes").Return(FileInfo{ + SizeIsDeferred: false, + Size: 10, + }, nil), + ) + + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + (&httpTest{ + Method: "HEAD", + URL: "yes", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + }, + Code: http.StatusOK, + ResHeader: map[string]string{ + "Upload-Defer-Length": "", + }, + }).Run(handler, t) + }) } diff --git a/patch_test.go b/patch_test.go index c62d4b1..a2f85c0 100644 --- a/patch_test.go +++ b/patch_test.go @@ -257,6 +257,132 @@ func TestPatch(t *testing.T) { }).Run(handler, t) }) + SubTest(t, "DeclareLengthOnFinalChunk", func(t *testing.T, store *MockFullDataStore) { + gomock.InOrder( + store.EXPECT().GetInfo("yes").Return(FileInfo{ + ID: "yes", + Offset: 5, + Size: 0, + SizeIsDeferred: true, + }, nil), + store.EXPECT().DeclareLength("yes", int64(20)), + store.EXPECT().WriteChunk("yes", int64(5), NewReaderMatcher("hellothisismore")).Return(int64(15), nil), + store.EXPECT().FinishUpload("yes"), + ) + + handler, _ := NewHandler(Config{ + DataStore: store, + MaxSize: 20, + }) + + body := strings.NewReader("hellothisismore") + + (&httpTest{ + Method: "PATCH", + URL: "yes", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Content-Type": "application/offset+octet-stream", + "Upload-Offset": "5", + "Upload-Length": "20", + }, + ReqBody: body, + Code: http.StatusNoContent, + ResHeader: map[string]string{ + "Upload-Offset": "20", + }, + }).Run(handler, t) + }) + + SubTest(t, "DeclareLengthAfterFinalChunk", func(t *testing.T, store *MockFullDataStore) { + gomock.InOrder( + store.EXPECT().GetInfo("yes").Return(FileInfo{ + ID: "yes", + Offset: 20, + Size: 0, + SizeIsDeferred: true, + }, nil), + store.EXPECT().DeclareLength("yes", int64(20)), + store.EXPECT().FinishUpload("yes"), + ) + + handler, _ := NewHandler(Config{ + DataStore: store, + MaxSize: 20, + }) + + (&httpTest{ + Method: "PATCH", + URL: "yes", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Content-Type": "application/offset+octet-stream", + "Upload-Offset": "20", + "Upload-Length": "20", + }, + ReqBody: nil, + Code: http.StatusNoContent, + ResHeader: map[string]string{}, + }).Run(handler, t) + }) + + SubTest(t, "DeclareLengthOnNonFinalChunk", func(t *testing.T, store *MockFullDataStore) { + gomock.InOrder( + store.EXPECT().GetInfo("yes").Return(FileInfo{ + ID: "yes", + Offset: 5, + Size: 0, + SizeIsDeferred: true, + }, nil), + store.EXPECT().DeclareLength("yes", int64(20)), + store.EXPECT().WriteChunk("yes", int64(5), NewReaderMatcher("hello")).Return(int64(5), nil), + store.EXPECT().GetInfo("yes").Return(FileInfo{ + ID: "yes", + Offset: 10, + Size: 20, + SizeIsDeferred: false, + }, nil), + store.EXPECT().WriteChunk("yes", int64(10), NewReaderMatcher("thisismore")).Return(int64(10), nil), + store.EXPECT().FinishUpload("yes"), + ) + + handler, _ := NewHandler(Config{ + DataStore: store, + MaxSize: 20, + }) + + (&httpTest{ + Method: "PATCH", + URL: "yes", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Content-Type": "application/offset+octet-stream", + "Upload-Offset": "5", + "Upload-Length": "20", + }, + ReqBody: strings.NewReader("hello"), + Code: http.StatusNoContent, + ResHeader: map[string]string{ + "Upload-Offset": "10", + }, + }).Run(handler, t) + + (&httpTest{ + Method: "PATCH", + URL: "yes", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Content-Type": "application/offset+octet-stream", + "Upload-Offset": "10", + }, + ReqBody: strings.NewReader("thisismore"), + Code: http.StatusNoContent, + ResHeader: map[string]string{ + "Upload-Offset": "20", + }, + }).Run(handler, t) + }) + SubTest(t, "Locker", func(t *testing.T, store *MockFullDataStore) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/post_test.go b/post_test.go index 639308d..53f88af 100644 --- a/post_test.go +++ b/post_test.go @@ -123,6 +123,54 @@ func TestPost(t *testing.T) { }).Run(handler, t) }) + SubTest(t, "UploadLengthAndUploadDeferLengthFail", func(t *testing.T, store *MockFullDataStore) { + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + (&httpTest{ + Method: "POST", + URL: "", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Upload-Length": "10", + "Upload-Defer-Length": "1", + }, + Code: http.StatusBadRequest, + }).Run(handler, t) + }) + + SubTest(t, "NeitherUploadLengthNorUploadDeferLengthFail", func(t *testing.T, store *MockFullDataStore) { + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + (&httpTest{ + Method: "POST", + URL: "", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + }, + Code: http.StatusBadRequest, + }).Run(handler, t) + }) + + SubTest(t, "InvalidUploadDeferLengthFail", func(t *testing.T, store *MockFullDataStore) { + handler, _ := NewHandler(Config{ + DataStore: store, + }) + + (&httpTest{ + Method: "POST", + URL: "", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Upload-Defer-Length": "bad", + }, + Code: http.StatusBadRequest, + }).Run(handler, t) + }) + SubTest(t, "ForwardHeaders", func(t *testing.T, store *MockFullDataStore) { SubTest(t, "IgnoreXForwarded", func(t *testing.T, store *MockFullDataStore) { store.EXPECT().NewUpload(FileInfo{ diff --git a/s3store/s3store.go b/s3store/s3store.go index 3d0ad98..00224a6 100644 --- a/s3store/s3store.go +++ b/s3store/s3store.go @@ -271,12 +271,14 @@ func (store S3Store) WriteChunk(id string, offset int64, src io.Reader) (int64, return bytesUploaded, nil } - if (size - offset) <= optimalPartSize { - if (size - offset) != n { + if !info.SizeIsDeferred { + if (size - offset) <= optimalPartSize { + if (size - offset) != n { + return bytesUploaded, nil + } + } else if n < optimalPartSize { return bytesUploaded, nil } - } else if n < optimalPartSize { - return bytesUploaded, nil } // Seek to the beginning of the file diff --git a/s3store/s3store_test.go b/s3store/s3store_test.go index f9729e0..ecd8f7f 100644 --- a/s3store/s3store_test.go +++ b/s3store/s3store_test.go @@ -52,8 +52,8 @@ func TestNewUpload(t *testing.T) { s3obj.EXPECT().PutObject(&s3.PutObjectInput{ Bucket: aws.String("bucket"), Key: aws.String("uploadId.info"), - Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)), - ContentLength: aws.Int64(int64(148)), + Body: bytes.NewReader([]byte(`{"ID":"uploadId+multipartId","Size":500,"SizeIsDeferred":false,"Offset":0,"MetaData":{"bar":"menü","foo":"hello"},"IsPartial":false,"IsFinal":false,"PartialUploads":null}`)), + ContentLength: aws.Int64(int64(171)), }), ) diff --git a/unrouted_handler.go b/unrouted_handler.go index c547fd2..4cc4e23 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -5,6 +5,7 @@ import ( "errors" "io" "log" + "math" "net" "net/http" "os" @@ -15,6 +16,8 @@ import ( "time" ) +const UploadLengthDeferred = "1" + var ( reExtractFileID = regexp.MustCompile(`([^/]+)\/?$`) reForwardedHost = regexp.MustCompile(`host=([^,]+)`) @@ -47,19 +50,21 @@ func NewHTTPError(err error, statusCode int) HTTPError { } var ( - ErrUnsupportedVersion = NewHTTPError(errors.New("unsupported version"), http.StatusPreconditionFailed) - ErrMaxSizeExceeded = NewHTTPError(errors.New("maximum size exceeded"), http.StatusRequestEntityTooLarge) - ErrInvalidContentType = NewHTTPError(errors.New("missing or invalid Content-Type header"), http.StatusBadRequest) - ErrInvalidUploadLength = NewHTTPError(errors.New("missing or invalid Upload-Length header"), http.StatusBadRequest) - ErrInvalidOffset = NewHTTPError(errors.New("missing or invalid Upload-Offset header"), http.StatusBadRequest) - ErrNotFound = NewHTTPError(errors.New("upload not found"), http.StatusNotFound) - ErrFileLocked = NewHTTPError(errors.New("file currently locked"), 423) // Locked (WebDAV) (RFC 4918) - ErrMismatchOffset = NewHTTPError(errors.New("mismatched offset"), http.StatusConflict) - ErrSizeExceeded = NewHTTPError(errors.New("resource's size exceeded"), http.StatusRequestEntityTooLarge) - ErrNotImplemented = NewHTTPError(errors.New("feature not implemented"), http.StatusNotImplemented) - ErrUploadNotFinished = NewHTTPError(errors.New("one of the partial uploads is not finished"), http.StatusBadRequest) - ErrInvalidConcat = NewHTTPError(errors.New("invalid Upload-Concat header"), http.StatusBadRequest) - ErrModifyFinal = NewHTTPError(errors.New("modifying a final upload is not allowed"), http.StatusForbidden) + ErrUnsupportedVersion = NewHTTPError(errors.New("unsupported version"), http.StatusPreconditionFailed) + ErrMaxSizeExceeded = NewHTTPError(errors.New("maximum size exceeded"), http.StatusRequestEntityTooLarge) + ErrInvalidContentType = NewHTTPError(errors.New("missing or invalid Content-Type header"), http.StatusBadRequest) + ErrInvalidUploadLength = NewHTTPError(errors.New("missing or invalid Upload-Length header"), http.StatusBadRequest) + ErrInvalidOffset = NewHTTPError(errors.New("missing or invalid Upload-Offset header"), http.StatusBadRequest) + ErrNotFound = NewHTTPError(errors.New("upload not found"), http.StatusNotFound) + ErrFileLocked = NewHTTPError(errors.New("file currently locked"), 423) // Locked (WebDAV) (RFC 4918) + ErrMismatchOffset = NewHTTPError(errors.New("mismatched offset"), http.StatusConflict) + ErrSizeExceeded = NewHTTPError(errors.New("resource's size exceeded"), http.StatusRequestEntityTooLarge) + ErrNotImplemented = NewHTTPError(errors.New("feature not implemented"), http.StatusNotImplemented) + ErrUploadNotFinished = NewHTTPError(errors.New("one of the partial uploads is not finished"), http.StatusBadRequest) + ErrInvalidConcat = NewHTTPError(errors.New("invalid Upload-Concat header"), http.StatusBadRequest) + ErrModifyFinal = NewHTTPError(errors.New("modifying a final upload is not allowed"), http.StatusForbidden) + ErrUploadLengthAndUploadDeferLength = NewHTTPError(errors.New("provided both Upload-Length and Upload-Defer-Length"), http.StatusBadRequest) + ErrInvalidUploadDeferLength = NewHTTPError(errors.New("invalid Upload-Defer-Length header"), http.StatusBadRequest) ) // UnroutedHandler exposes methods to handle requests as part of the tus protocol, @@ -122,6 +127,9 @@ func NewUnroutedHandler(config Config) (*UnroutedHandler, error) { if config.StoreComposer.UsesConcater { extensions += ",concatenation" } + if config.StoreComposer.UsesLengthDeferrer { + extensions += ",creation-defer-length" + } handler := &UnroutedHandler{ config: config, @@ -241,6 +249,7 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) // uploads the size is sum of all sizes of these files (no need for // Upload-Length header) var size int64 + var sizeIsDeferred bool if isFinal { // A final upload must not contain a chunk within the creation request if containsChunk { @@ -254,9 +263,11 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) return } } else { - size, err = strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) - if err != nil || size < 0 { - handler.sendError(w, r, ErrInvalidUploadLength) + uploadLengthHeader := r.Header.Get("Upload-Length") + uploadDeferLengthHeader := r.Header.Get("Upload-Defer-Length") + size, sizeIsDeferred, err = handler.validateNewUploadLengthHeaders(uploadLengthHeader, uploadDeferLengthHeader) + if err != nil { + handler.sendError(w, r, err) return } } @@ -272,6 +283,7 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) info := FileInfo{ Size: size, + SizeIsDeferred: sizeIsDeferred, MetaData: meta, IsPartial: isPartial, IsFinal: isFinal, @@ -325,7 +337,7 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) handler.sendError(w, r, err) return } - } else if size == 0 { + } else if !sizeIsDeferred && size == 0 { // Directly finish the upload if the upload is empty (i.e. has a size of 0). // This statement is in an else-if block to avoid causing duplicate calls // to finishUploadIfComplete if an upload is empty and contains a chunk. @@ -380,8 +392,13 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request) w.Header().Set("Upload-Metadata", SerializeMetadataHeader(info.MetaData)) } + if info.SizeIsDeferred { + w.Header().Set("Upload-Defer-Length", UploadLengthDeferred) + } else { + w.Header().Set("Upload-Length", strconv.FormatInt(info.Size, 10)) + } + w.Header().Set("Cache-Control", "no-store") - w.Header().Set("Upload-Length", strconv.FormatInt(info.Size, 10)) w.Header().Set("Upload-Offset", strconv.FormatInt(info.Offset, 10)) handler.sendResp(w, r, http.StatusOK) } @@ -437,12 +454,35 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request } // Do not proxy the call to the data store if the upload is already completed - if info.Offset == info.Size { + if !info.SizeIsDeferred && info.Offset == info.Size { w.Header().Set("Upload-Offset", strconv.FormatInt(offset, 10)) handler.sendResp(w, r, http.StatusNoContent) return } + if r.Header.Get("Upload-Length") != "" { + if handler.composer.UsesLengthDeferrer { + if info.SizeIsDeferred { + uploadLength, err := strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) + if err != nil || uploadLength < 0 || uploadLength < info.Offset || uploadLength > handler.config.MaxSize { + handler.sendError(w, r, ErrInvalidUploadLength) + return + } + + info.Size = uploadLength + info.SizeIsDeferred = false + if err := handler.composer.LengthDeferrer.DeclareLength(id, info.Size); err != nil { + handler.sendError(w, r, err) + return + } + } else { + handler.sendError(w, r, ErrInvalidUploadLength) + } + } else { + handler.sendError(w, r, ErrNotImplemented) + } + } + if err := handler.writeChunk(id, info, w, r); err != nil { handler.sendError(w, r, err) return @@ -460,11 +500,23 @@ func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.Resp offset := info.Offset // Test if this upload fits into the file's size - if offset+length > info.Size { + if !info.SizeIsDeferred && offset+length > info.Size { return ErrSizeExceeded } maxSize := info.Size - offset + // If the upload's length is deferred and the PATCH request does not contain the Content-Length + // header (which is allowed if 'Transfer-Encoding: chunked' is used), we still need to set limits for + // the body size. + if info.SizeIsDeferred { + if handler.config.MaxSize > 0 { + // Ensure that the upload does not exceed the maximum upload size + maxSize = handler.config.MaxSize - offset + } else { + // If no upload limit is given, we allow arbitrary sizes + maxSize = math.MaxInt64 + } + } if length > 0 { maxSize = length } @@ -507,7 +559,7 @@ func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.Resp // function and send the necessary message on the CompleteUpload channel. func (handler *UnroutedHandler) finishUploadIfComplete(info FileInfo) error { // If the upload is completed, ... - if info.Offset == info.Size { + if !info.SizeIsDeferred && info.Offset == info.Size { // ... allow custom mechanism to finish and cleanup the upload if handler.composer.UsesFinisher { if err := handler.composer.Finisher.FinishUpload(info.ID); err != nil { @@ -836,7 +888,7 @@ func (handler *UnroutedHandler) sizeOfUploads(ids []string) (size int64, err err return size, err } - if info.Offset != info.Size { + if info.SizeIsDeferred || info.Offset != info.Size { err = ErrUploadNotFinished return size, err } @@ -847,6 +899,31 @@ func (handler *UnroutedHandler) sizeOfUploads(ids []string) (size int64, err err return } +// Verify that the Upload-Length and Upload-Defer-Length headers are acceptable for creating a +// new upload +func (handler *UnroutedHandler) validateNewUploadLengthHeaders(uploadLengthHeader string, uploadDeferLengthHeader string) (uploadLength int64, uploadLengthDeferred bool, err error) { + haveBothLengthHeaders := uploadLengthHeader != "" && uploadDeferLengthHeader != "" + haveInvalidDeferHeader := uploadDeferLengthHeader != "" && uploadDeferLengthHeader != UploadLengthDeferred + lengthIsDeferred := uploadDeferLengthHeader == UploadLengthDeferred + + if lengthIsDeferred && !handler.composer.UsesLengthDeferrer { + err = ErrNotImplemented + } else if haveBothLengthHeaders { + err = ErrUploadLengthAndUploadDeferLength + } else if haveInvalidDeferHeader { + err = ErrInvalidUploadDeferLength + } else if lengthIsDeferred { + uploadLengthDeferred = true + } else { + uploadLength, err = strconv.ParseInt(uploadLengthHeader, 10, 64) + if err != nil || uploadLength < 0 { + err = ErrInvalidUploadLength + } + } + + return +} + // ParseMetadataHeader parses the Upload-Metadata header as defined in the // File Creation extension. // e.g. Upload-Metadata: name bHVucmpzLnBuZw==,type aW1hZ2UvcG5n diff --git a/utils_test.go b/utils_test.go index 93d96ce..550e87e 100644 --- a/utils_test.go +++ b/utils_test.go @@ -27,6 +27,7 @@ type FullDataStore interface { tusd.ConcaterDataStore tusd.GetReaderDataStore tusd.FinisherDataStore + tusd.LengthDeferrerDataStore } type Locker interface {