diff --git a/.gitmodules b/.gitmodules index d23d6a1..fa4617e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,9 +4,6 @@ [submodule "vendor/github.com/nightlyone/lockfile"] path = vendor/github.com/nightlyone/lockfile url = https://github.com/nightlyone/lockfile -[submodule "vendor/github.com/aws/aws-sdk-go"] - path = vendor/github.com/aws/aws-sdk-go - url = https://github.com/aws/aws-sdk-go [submodule "vendor/github.com/go-ini/ini"] path = vendor/github.com/go-ini/ini url = https://github.com/go-ini/ini @@ -25,9 +22,6 @@ [submodule "vendor/github.com/pmezard/go-difflib"] path = vendor/github.com/pmezard/go-difflib url = https://github.com/pmezard/go-difflib -[submodule "vendor/github.com/hashicorp/consul"] - path = vendor/github.com/hashicorp/consul - url = https://github.com/hashicorp/consul [submodule "vendor/github.com/hashicorp/go-cleanhttp"] path = vendor/github.com/hashicorp/go-cleanhttp url = https://github.com/hashicorp/go-cleanhttp diff --git a/.travis.yml b/.travis.yml index c8c9239..74747ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ go: - 1.4 - 1.5 - 1.6 +- 1.7 - tip sudo: required cache: @@ -22,6 +23,7 @@ matrix: install: - export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \)) - rsync -r ./vendor/ $GOPATH/src +- go get $PACKAGES script: - go test $PACKAGES before_deploy: diff --git a/cmd/tusd/cli/composer.go b/cmd/tusd/cli/composer.go index 492433b..b68d090 100644 --- a/cmd/tusd/cli/composer.go +++ b/cmd/tusd/cli/composer.go @@ -25,7 +25,7 @@ func CreateComposer() { dir := Flags.UploadDir stdout.Printf("Using '%s' as directory storage.\n", dir) - if err := os.MkdirAll(dir, os.FileMode(0775)); err != nil { + if err := os.MkdirAll(dir, os.FileMode(0774)); err != nil { stderr.Fatalf("Unable to ensure directory exists: %s", err) } diff --git a/cmd/tusd/cli/flags.go b/cmd/tusd/cli/flags.go index 20c8af2..f8f4dcf 100644 --- a/cmd/tusd/cli/flags.go +++ b/cmd/tusd/cli/flags.go @@ -17,6 +17,7 @@ var Flags struct { HooksDir string ShowVersion bool ExposeMetrics bool + MetricsPath string BehindProxy bool HooksInstalled bool @@ -34,6 +35,7 @@ func ParseFlags() { flag.StringVar(&Flags.HooksDir, "hooks-dir", "", "Directory to search for available hooks scripts") flag.BoolVar(&Flags.ShowVersion, "version", false, "Print tusd version information") flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage") + flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible") flag.BoolVar(&Flags.BehindProxy, "behind-proxy", false, "Respect X-Forwarded-* and similar headers which may be set by proxies") flag.Parse() diff --git a/cmd/tusd/cli/greeting.go b/cmd/tusd/cli/greeting.go index 47f8d31..22f178b 100644 --- a/cmd/tusd/cli/greeting.go +++ b/cmd/tusd/cli/greeting.go @@ -12,19 +12,23 @@ func PrepareGreeting() { `Welcome to tusd =============== -Congratulations for setting up tusd! You are now part of the chosen elite and -able to experience the feeling of resumable uploads! We hope you are as excited -as we are (a lot)! +Congratulations on setting up tusd! Thanks for joining our cause, you have taken +the first step towards making the future of resumable uploading a reality! We +hope you are as excited about this as we are! -However, there is something you should be aware of: While you got tusd -running (you did an awesome job!), this is the root directory of the server -and tus requests are only accepted at the %s route. +While you did an awesome job on getting tusd running, this is just the welcome +message, so let's talk about the places that really matter: -So don't waste time, head over there and experience the future! +- %s - send your tus uploads to this endpoint +- %s - gather statistics to keep tusd running smoothly +- https://github.com/tus/tusd/issues - report your bugs here + +So quit lollygagging, send over your files and experience the future! Version = %s GitCommit = %s -BuildDate = %s`, Flags.Basepath, VersionName, GitCommit, BuildDate) +BuildDate = %s +`, Flags.Basepath, Flags.MetricsPath, VersionName, GitCommit, BuildDate) } func DisplayGreeting(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/tusd/cli/metrics.go b/cmd/tusd/cli/metrics.go index 2894b0e..921d67a 100644 --- a/cmd/tusd/cli/metrics.go +++ b/cmd/tusd/cli/metrics.go @@ -18,5 +18,6 @@ func SetupMetrics(handler *tusd.Handler) { prometheus.MustRegister(MetricsOpenConnections) prometheus.MustRegister(prometheuscollector.New(handler.Metrics)) - http.Handle("/metrics", prometheus.Handler()) + stdout.Printf("Using %s as the metrics path.\n", Flags.MetricsPath) + http.Handle(Flags.MetricsPath, prometheus.Handler()) } diff --git a/cmd/tusd/cli/serve.go b/cmd/tusd/cli/serve.go index 53cad7e..37d6317 100644 --- a/cmd/tusd/cli/serve.go +++ b/cmd/tusd/cli/serve.go @@ -27,7 +27,6 @@ func Serve() { stdout.Printf("Using %s as address to listen.\n", address) stdout.Printf("Using %s as the base path.\n", basepath) - stdout.Printf(Composer.Capabilities()) SetupPostHooks(handler) @@ -35,6 +34,8 @@ func Serve() { SetupMetrics(handler) } + stdout.Printf(Composer.Capabilities()) + // Do not display the greeting if the tusd handler will be mounted at the root // path. Else this would cause a "multiple registrations for /" panic. if basepath != "/" { diff --git a/concat_test.go b/concat_test.go index 4f522fb..0a1a108 100644 --- a/concat_test.go +++ b/concat_test.go @@ -46,7 +46,7 @@ func TestConcatPartial(t *testing.T) { Method: "OPTIONS", URL: "", ResHeader: map[string]string{ - "Tus-Extension": "creation,concatenation", + "Tus-Extension": "creation,creation-with-upload,concatenation", }, Code: http.StatusOK, }).Run(handler, t) @@ -69,7 +69,7 @@ func TestConcatPartial(t *testing.T) { ReqHeader: map[string]string{ "Tus-Resumable": "1.0.0", }, - Code: http.StatusNoContent, + Code: http.StatusOK, ResHeader: map[string]string{ "Upload-Concat": "partial", }, @@ -165,7 +165,7 @@ func TestConcatFinal(t *testing.T) { ReqHeader: map[string]string{ "Tus-Resumable": "1.0.0", }, - Code: http.StatusNoContent, + Code: http.StatusOK, ResHeader: map[string]string{ "Upload-Concat": "final; http://tus.io/files/a http://tus.io/files/b", "Upload-Length": "10", diff --git a/filestore/filestore.go b/filestore/filestore.go index 5dfe299..b676633 100644 --- a/filestore/filestore.go +++ b/filestore/filestore.go @@ -28,7 +28,7 @@ import ( "github.com/nightlyone/lockfile" ) -var defaultFilePerm = os.FileMode(0775) +var defaultFilePerm = os.FileMode(0664) // See the tusd.DataStore interface for documentation about the different // methods. diff --git a/head_test.go b/head_test.go index 9b63b59..90c5554 100644 --- a/head_test.go +++ b/head_test.go @@ -40,7 +40,7 @@ func TestHead(t *testing.T) { ReqHeader: map[string]string{ "Tus-Resumable": "1.0.0", }, - Code: http.StatusNoContent, + Code: http.StatusOK, ResHeader: map[string]string{ "Upload-Offset": "11", "Upload-Length": "44", diff --git a/options_test.go b/options_test.go index 206fdf2..2d89ddf 100644 --- a/options_test.go +++ b/options_test.go @@ -20,7 +20,7 @@ func TestOptions(t *testing.T) { Method: "OPTIONS", Code: http.StatusOK, ResHeader: map[string]string{ - "Tus-Extension": "creation", + "Tus-Extension": "creation,creation-with-upload", "Tus-Version": "1.0.0", "Tus-Resumable": "1.0.0", "Tus-Max-Size": "400", diff --git a/post_test.go b/post_test.go index 5f41ddd..ea91f21 100644 --- a/post_test.go +++ b/post_test.go @@ -1,44 +1,57 @@ package tusd_test import ( + "bytes" + "io" + "io/ioutil" "net/http" + "strings" "testing" + "github.com/stretchr/testify/assert" + . "github.com/tus/tusd" ) type postStore struct { - t *testing.T + t *assert.Assertions zeroStore } func (s postStore) NewUpload(info FileInfo) (string, error) { - if info.Size != 300 { - s.t.Errorf("Expected size to be 300 (got %v)", info.Size) - } + s.t.Equal(int64(300), info.Size) metaData := info.MetaData - if len(metaData) != 2 { - s.t.Errorf("Expected two elements in metadata") - } - - if v := metaData["foo"]; v != "hello" { - s.t.Errorf("Expected foo element to be 'hello' but got %s", v) - } - - if v := metaData["bar"]; v != "world" { - s.t.Errorf("Expected bar element to be 'world' but got %s", v) - } + s.t.Equal(2, len(metaData)) + s.t.Equal("hello", metaData["foo"]) + s.t.Equal("world", metaData["bar"]) return "foo", nil } +func (s postStore) WriteChunk(id string, offset int64, src io.Reader) (int64, error) { + s.t.Equal(int64(0), offset) + + data, err := ioutil.ReadAll(src) + s.t.Nil(err) + s.t.Equal("hello", string(data)) + + return 5, nil +} + +func (s postStore) ConcatUploads(id string, uploads []string) error { + s.t.True(false, "concatenation should not be attempted") + return nil +} + func TestPost(t *testing.T) { + a := assert.New(t) + handler, _ := NewHandler(Config{ MaxSize: 400, BasePath: "files", DataStore: postStore{ - t: t, + t: a, }, }) @@ -87,7 +100,7 @@ func TestPost(t *testing.T) { MaxSize: 400, BasePath: "files", DataStore: postStore{ - t: t, + t: a, }, RespectForwardedHeaders: true, }) @@ -141,3 +154,70 @@ func TestPost(t *testing.T) { }, }).Run(handler, t) } + +func TestPostWithUpload(t *testing.T) { + a := assert.New(t) + + handler, _ := NewHandler(Config{ + MaxSize: 400, + BasePath: "files", + DataStore: postStore{ + t: a, + }, + }) + + (&httpTest{ + Name: "Successful request", + Method: "POST", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Upload-Length": "300", + "Content-Type": "application/offset+octet-stream", + "Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=", + }, + ReqBody: strings.NewReader("hello"), + Code: http.StatusCreated, + ResHeader: map[string]string{ + "Location": "http://tus.io/files/foo", + "Upload-Offset": "5", + }, + }).Run(handler, t) + + (&httpTest{ + Name: "Exceeding upload size", + Method: "POST", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Upload-Length": "300", + "Content-Type": "application/offset+octet-stream", + "Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=", + }, + ReqBody: bytes.NewReader(make([]byte, 400)), + Code: http.StatusRequestEntityTooLarge, + }).Run(handler, t) + + (&httpTest{ + Name: "Incorrect content type", + Method: "POST", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Content-Type": "application/false", + }, + ReqBody: strings.NewReader("hello"), + Code: http.StatusBadRequest, + }).Run(handler, t) + + (&httpTest{ + Name: "Upload and final concatenation", + Method: "POST", + ReqHeader: map[string]string{ + "Tus-Resumable": "1.0.0", + "Upload-Length": "300", + "Content-Type": "application/offset+octet-stream", + "Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=", + "Upload-Concat": "final; http://tus.io/files/a http://tus.io/files/b", + }, + ReqBody: strings.NewReader("hello"), + Code: http.StatusForbidden, + }).Run(handler, t) +} diff --git a/s3store/s3store_mock_test.go b/s3store/s3store_mock_test.go index cc2ed9c..2f6db55 100644 --- a/s3store/s3store_mock_test.go +++ b/s3store/s3store_mock_test.go @@ -907,6 +907,16 @@ func (_mr *_MockS3APIRecorder) ListObjectsV2(arg0 interface{}) *gomock.Call { return _mr.mock.ctrl.RecordCall(_mr.mock, "ListObjectsV2", arg0) } +func (_m *MockS3API) ListObjectsV2Pages(_param0 *s3.ListObjectsV2Input, _param1 func(*s3.ListObjectsV2Output, bool) bool) error { + ret := _m.ctrl.Call(_m, "ListObjectsV2Pages", _param0, _param1) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockS3APIRecorder) ListObjectsV2Pages(arg0, arg1 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "ListObjectsV2Pages", arg0, arg1) +} + func (_m *MockS3API) ListObjectsV2Request(_param0 *s3.ListObjectsV2Input) (*request.Request, *s3.ListObjectsV2Output) { ret := _m.ctrl.Call(_m, "ListObjectsV2Request", _param0) ret0, _ := ret[0].(*request.Request) diff --git a/terminate_test.go b/terminate_test.go index ba15164..55b6293 100644 --- a/terminate_test.go +++ b/terminate_test.go @@ -44,7 +44,7 @@ func TestTerminate(t *testing.T) { Method: "OPTIONS", URL: "", ResHeader: map[string]string{ - "Tus-Extension": "creation,termination", + "Tus-Extension": "creation,creation-with-upload,termination", }, Code: http.StatusOK, }).Run(handler, t) diff --git a/unrouted_handler.go b/unrouted_handler.go index 4610760..0876e5f 100644 --- a/unrouted_handler.go +++ b/unrouted_handler.go @@ -89,7 +89,7 @@ func NewUnroutedHandler(config Config) (*UnroutedHandler, error) { } // Only promote extesions using the Tus-Extension header which are implemented - extensions := "creation" + extensions := "creation,creation-with-upload" if config.StoreComposer.UsesTerminater { extensions += ",termination" } @@ -191,6 +191,16 @@ func (handler *UnroutedHandler) Middleware(h http.Handler) http.Handler { // PostFile creates a new file upload using the datastore after validating the // length and parsing the metadata. func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) { + // Check for presence of application/offset+octet-stream + containsChunk := false + if contentType := r.Header.Get("Content-Type"); contentType != "" { + if contentType != "application/offset+octet-stream" { + handler.sendError(w, r, ErrInvalidContentType) + return + } + containsChunk = true + } + // Only use the proper Upload-Concat header if the concatenation extension // is even supported by the data store. var concatHeader string @@ -210,6 +220,12 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) // Upload-Length header) var size int64 if isFinal { + // A final upload must not contain a chunk within the creation request + if containsChunk { + handler.sendError(w, r, ErrModifyFinal) + return + } + size, err = handler.sizeOfUploads(partialUploads) if err != nil { handler.sendError(w, r, err) @@ -246,6 +262,13 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) return } + // Add the Location header directly after creating the new resource to even + // include it in cases of failure when an error is returned + url := handler.absFileURL(r, id) + w.Header().Set("Location", url) + + go handler.Metrics.incUploadsCreated() + if isFinal { if err := handler.composer.Concater.ConcatUploads(id, partialUploads); err != nil { handler.sendError(w, r, err) @@ -257,15 +280,26 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) info.ID = id handler.CompleteUploads <- info } - - go handler.Metrics.incUploadsFinished() } - url := handler.absFileURL(r, id) - w.Header().Set("Location", url) - w.WriteHeader(http.StatusCreated) + if containsChunk { + if handler.composer.UsesLocker { + locker := handler.composer.Locker + if err := locker.LockUpload(id); err != nil { + handler.sendError(w, r, err) + return + } - go handler.Metrics.incUploadsCreated() + defer locker.UnlockUpload(id) + } + + if err := handler.writeChunk(id, info, w, r); err != nil { + handler.sendError(w, r, err) + return + } + } + + w.WriteHeader(http.StatusCreated) } // HeadFile returns the length and offset for the HEAD request @@ -313,7 +347,7 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request) w.Header().Set("Cache-Control", "no-store") w.Header().Set("Upload-Length", strconv.FormatInt(info.Size, 10)) w.Header().Set("Upload-Offset", strconv.FormatInt(info.Offset, 10)) - w.WriteHeader(http.StatusNoContent) + w.WriteHeader(http.StatusOK) } // PatchFile adds a chunk to an upload. Only allowed enough space is left. @@ -372,13 +406,23 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request return } + if err := handler.writeChunk(id, info, w, r); err != nil { + handler.sendError(w, r, err) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +// PatchFile adds a chunk to an upload. Only allowed enough space is left. +func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.ResponseWriter, r *http.Request) error { // Get Content-Length if possible length := r.ContentLength + offset := info.Offset // Test if this upload fits into the file's size if offset+length > info.Size { - handler.sendError(w, r, ErrSizeExceeded) - return + return ErrSizeExceeded } maxSize := info.Size - offset @@ -386,13 +430,18 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request maxSize = length } - // Limit the data read from the request's body to the allowed maxiumum - reader := io.LimitReader(r.Body, maxSize) + var bytesWritten int64 + // Prevent a nil pointer derefernce when accessing the body which may not be + // available in the case of a malicious request. + if r.Body != nil { + // Limit the data read from the request's body to the allowed maxiumum + reader := io.LimitReader(r.Body, maxSize) - bytesWritten, err := handler.composer.Core.WriteChunk(id, offset, reader) - if err != nil { - handler.sendError(w, r, err) - return + var err error + bytesWritten, err = handler.composer.Core.WriteChunk(id, offset, reader) + if err != nil { + return err + } } // Send new offset to client @@ -405,8 +454,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request // ... allow custom mechanism to finish and cleanup the upload if handler.composer.UsesFinisher { if err := handler.composer.Finisher.FinishUpload(id); err != nil { - handler.sendError(w, r, err) - return + return err } } @@ -419,7 +467,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request go handler.Metrics.incUploadsFinished() } - w.WriteHeader(http.StatusNoContent) + return nil } // GetFile handles requests to download a file using a GET request. This is not diff --git a/vendor/github.com/aws/aws-sdk-go b/vendor/github.com/aws/aws-sdk-go deleted file mode 160000 index c76e891..0000000 --- a/vendor/github.com/aws/aws-sdk-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c76e8918e8f08490e3bb154178a84a0b2bdf8d6e diff --git a/vendor/github.com/hashicorp/consul b/vendor/github.com/hashicorp/consul deleted file mode 160000 index f6fef66..0000000 --- a/vendor/github.com/hashicorp/consul +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f6fef66e1bf17be4f3c9855fbec6de802ca6bd7d