Merge branch 'master' of github.com:tus/tusd

.gitmodules

@@ -4,9 +4,6 @@
 [submodule "vendor/github.com/nightlyone/lockfile"]
 	path = vendor/github.com/nightlyone/lockfile
 	url = https://github.com/nightlyone/lockfile
-[submodule "vendor/github.com/aws/aws-sdk-go"]
-	path = vendor/github.com/aws/aws-sdk-go
-	url = https://github.com/aws/aws-sdk-go
 [submodule "vendor/github.com/go-ini/ini"]
 	path = vendor/github.com/go-ini/ini
 	url = https://github.com/go-ini/ini
@@ -25,9 +22,6 @@
 [submodule "vendor/github.com/pmezard/go-difflib"]
 	path = vendor/github.com/pmezard/go-difflib
 	url = https://github.com/pmezard/go-difflib
-[submodule "vendor/github.com/hashicorp/consul"]
-	path = vendor/github.com/hashicorp/consul
-	url = https://github.com/hashicorp/consul
 [submodule "vendor/github.com/hashicorp/go-cleanhttp"]
 	path = vendor/github.com/hashicorp/go-cleanhttp
 	url = https://github.com/hashicorp/go-cleanhttp

.travis.yml

@@ -4,6 +4,7 @@ go:
 - 1.4
 - 1.5
 - 1.6
+- 1.7
 - tip
 sudo: required
 cache:
@@ -22,6 +23,7 @@ matrix:
 install:
 - export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \))
 - rsync -r ./vendor/ $GOPATH/src
+- go get $PACKAGES
 script:
 - go test $PACKAGES
 before_deploy:

cmd/tusd/cli/composer.go

@@ -25,7 +25,7 @@ func CreateComposer() {
 		dir := Flags.UploadDir
 
 		stdout.Printf("Using '%s' as directory storage.\n", dir)
-		if err := os.MkdirAll(dir, os.FileMode(0775)); err != nil {
+		if err := os.MkdirAll(dir, os.FileMode(0774)); err != nil {
 			stderr.Fatalf("Unable to ensure directory exists: %s", err)
 		}
 

cmd/tusd/cli/flags.go

@@ -17,6 +17,7 @@ var Flags struct {
 	HooksDir      string
 	ShowVersion   bool
 	ExposeMetrics bool
+	MetricsPath   string
 	BehindProxy   bool
 
 	HooksInstalled bool
@@ -34,6 +35,7 @@ func ParseFlags() {
 	flag.StringVar(&Flags.HooksDir, "hooks-dir", "", "Directory to search for available hooks scripts")
 	flag.BoolVar(&Flags.ShowVersion, "version", false, "Print tusd version information")
 	flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage")
+	flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible")
 	flag.BoolVar(&Flags.BehindProxy, "behind-proxy", false, "Respect X-Forwarded-* and similar headers which may be set by proxies")
 
 	flag.Parse()

cmd/tusd/cli/greeting.go

@@ -12,19 +12,23 @@ func PrepareGreeting() {
 		`Welcome to tusd
 ===============
 
-Congratulations for setting up tusd! You are now part of the chosen elite and
-able to experience the feeling of resumable uploads! We hope you are as excited
-as we are (a lot)!
+Congratulations on setting up tusd! Thanks for joining our cause, you have taken
+the first step towards making the future of resumable uploading a reality! We
+hope you are as excited about this as we are!
 
-However, there is something you should be aware of: While you got tusd
-running (you did an awesome job!), this is the root directory of the server
-and tus requests are only accepted at the %s route.
+While you did an awesome job on getting tusd running, this is just the welcome
+message, so let's talk about the places that really matter:
 
-So don't waste time, head over there and experience the future!
+- %s - send your tus uploads to this endpoint
+- %s - gather statistics to keep tusd running smoothly
+- https://github.com/tus/tusd/issues - report your bugs here
+
+So quit lollygagging, send over your files and experience the future!
 
 Version = %s
 GitCommit = %s
-BuildDate = %s`, Flags.Basepath, VersionName, GitCommit, BuildDate)
+BuildDate = %s
+`, Flags.Basepath, Flags.MetricsPath, VersionName, GitCommit, BuildDate)
 }
 
 func DisplayGreeting(w http.ResponseWriter, r *http.Request) {

cmd/tusd/cli/metrics.go

@@ -18,5 +18,6 @@ func SetupMetrics(handler *tusd.Handler) {
 	prometheus.MustRegister(MetricsOpenConnections)
 	prometheus.MustRegister(prometheuscollector.New(handler.Metrics))
 
-	http.Handle("/metrics", prometheus.Handler())
+	stdout.Printf("Using %s as the metrics path.\n", Flags.MetricsPath)
+	http.Handle(Flags.MetricsPath, prometheus.Handler())
 }

cmd/tusd/cli/serve.go

@@ -27,7 +27,6 @@ func Serve() {
 
 	stdout.Printf("Using %s as address to listen.\n", address)
 	stdout.Printf("Using %s as the base path.\n", basepath)
-	stdout.Printf(Composer.Capabilities())
 
 	SetupPostHooks(handler)
 
@@ -35,6 +34,8 @@ func Serve() {
 		SetupMetrics(handler)
 	}
 
+	stdout.Printf(Composer.Capabilities())
+
 	// Do not display the greeting if the tusd handler will be mounted at the root
 	// path. Else this would cause a "multiple registrations for /" panic.
 	if basepath != "/" {

concat_test.go

@@ -46,7 +46,7 @@ func TestConcatPartial(t *testing.T) {
 		Method: "OPTIONS",
 		URL:    "",
 		ResHeader: map[string]string{
-			"Tus-Extension": "creation,concatenation",
+			"Tus-Extension": "creation,creation-with-upload,concatenation",
 		},
 		Code: http.StatusOK,
 	}).Run(handler, t)
@@ -69,7 +69,7 @@ func TestConcatPartial(t *testing.T) {
 		ReqHeader: map[string]string{
 			"Tus-Resumable": "1.0.0",
 		},
-		Code: http.StatusNoContent,
+		Code: http.StatusOK,
 		ResHeader: map[string]string{
 			"Upload-Concat": "partial",
 		},
@@ -165,7 +165,7 @@ func TestConcatFinal(t *testing.T) {
 		ReqHeader: map[string]string{
 			"Tus-Resumable": "1.0.0",
 		},
-		Code: http.StatusNoContent,
+		Code: http.StatusOK,
 		ResHeader: map[string]string{
 			"Upload-Concat": "final; http://tus.io/files/a http://tus.io/files/b",
 			"Upload-Length": "10",

filestore/filestore.go

@@ -28,7 +28,7 @@ import (
 	"github.com/nightlyone/lockfile"
 )
 
-var defaultFilePerm = os.FileMode(0775)
+var defaultFilePerm = os.FileMode(0664)
 
 // See the tusd.DataStore interface for documentation about the different
 // methods.

head_test.go

@@ -40,7 +40,7 @@ func TestHead(t *testing.T) {
 		ReqHeader: map[string]string{
 			"Tus-Resumable": "1.0.0",
 		},
-		Code: http.StatusNoContent,
+		Code: http.StatusOK,
 		ResHeader: map[string]string{
 			"Upload-Offset": "11",
 			"Upload-Length": "44",

options_test.go

@@ -20,7 +20,7 @@ func TestOptions(t *testing.T) {
 		Method: "OPTIONS",
 		Code:   http.StatusOK,
 		ResHeader: map[string]string{
-			"Tus-Extension": "creation",
+			"Tus-Extension": "creation,creation-with-upload",
 			"Tus-Version":   "1.0.0",
 			"Tus-Resumable": "1.0.0",
 			"Tus-Max-Size":  "400",

post_test.go

@@ -1,44 +1,57 @@
 package tusd_test
 
 import (
+	"bytes"
+	"io"
+	"io/ioutil"
 	"net/http"
+	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	. "github.com/tus/tusd"
 )
 
 type postStore struct {
-	t *testing.T
+	t *assert.Assertions
 	zeroStore
 }
 
 func (s postStore) NewUpload(info FileInfo) (string, error) {
-	if info.Size != 300 {
-		s.t.Errorf("Expected size to be 300 (got %v)", info.Size)
-	}
+	s.t.Equal(int64(300), info.Size)
 
 	metaData := info.MetaData
-	if len(metaData) != 2 {
-		s.t.Errorf("Expected two elements in metadata")
-	}
-
-	if v := metaData["foo"]; v != "hello" {
-		s.t.Errorf("Expected foo element to be 'hello' but got %s", v)
-	}
-
-	if v := metaData["bar"]; v != "world" {
-		s.t.Errorf("Expected bar element to be 'world' but got %s", v)
-	}
+	s.t.Equal(2, len(metaData))
+	s.t.Equal("hello", metaData["foo"])
+	s.t.Equal("world", metaData["bar"])
 
 	return "foo", nil
 }
 
+func (s postStore) WriteChunk(id string, offset int64, src io.Reader) (int64, error) {
+	s.t.Equal(int64(0), offset)
+
+	data, err := ioutil.ReadAll(src)
+	s.t.Nil(err)
+	s.t.Equal("hello", string(data))
+
+	return 5, nil
+}
+
+func (s postStore) ConcatUploads(id string, uploads []string) error {
+	s.t.True(false, "concatenation should not be attempted")
+	return nil
+}
+
 func TestPost(t *testing.T) {
+	a := assert.New(t)
+
 	handler, _ := NewHandler(Config{
 		MaxSize:  400,
 		BasePath: "files",
 		DataStore: postStore{
-			t: t,
+			t: a,
 		},
 	})
 
@@ -87,7 +100,7 @@ func TestPost(t *testing.T) {
 		MaxSize:  400,
 		BasePath: "files",
 		DataStore: postStore{
-			t: t,
+			t: a,
 		},
 		RespectForwardedHeaders: true,
 	})
@@ -141,3 +154,70 @@ func TestPost(t *testing.T) {
 		},
 	}).Run(handler, t)
 }
+
+func TestPostWithUpload(t *testing.T) {
+	a := assert.New(t)
+
+	handler, _ := NewHandler(Config{
+		MaxSize:  400,
+		BasePath: "files",
+		DataStore: postStore{
+			t: a,
+		},
+	})
+
+	(&httpTest{
+		Name:   "Successful request",
+		Method: "POST",
+		ReqHeader: map[string]string{
+			"Tus-Resumable":   "1.0.0",
+			"Upload-Length":   "300",
+			"Content-Type":    "application/offset+octet-stream",
+			"Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=",
+		},
+		ReqBody: strings.NewReader("hello"),
+		Code:    http.StatusCreated,
+		ResHeader: map[string]string{
+			"Location":      "http://tus.io/files/foo",
+			"Upload-Offset": "5",
+		},
+	}).Run(handler, t)
+
+	(&httpTest{
+		Name:   "Exceeding upload size",
+		Method: "POST",
+		ReqHeader: map[string]string{
+			"Tus-Resumable":   "1.0.0",
+			"Upload-Length":   "300",
+			"Content-Type":    "application/offset+octet-stream",
+			"Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=",
+		},
+		ReqBody: bytes.NewReader(make([]byte, 400)),
+		Code:    http.StatusRequestEntityTooLarge,
+	}).Run(handler, t)
+
+	(&httpTest{
+		Name:   "Incorrect content type",
+		Method: "POST",
+		ReqHeader: map[string]string{
+			"Tus-Resumable": "1.0.0",
+			"Content-Type":  "application/false",
+		},
+		ReqBody: strings.NewReader("hello"),
+		Code:    http.StatusBadRequest,
+	}).Run(handler, t)
+
+	(&httpTest{
+		Name:   "Upload and final concatenation",
+		Method: "POST",
+		ReqHeader: map[string]string{
+			"Tus-Resumable":   "1.0.0",
+			"Upload-Length":   "300",
+			"Content-Type":    "application/offset+octet-stream",
+			"Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=",
+			"Upload-Concat":   "final; http://tus.io/files/a http://tus.io/files/b",
+		},
+		ReqBody: strings.NewReader("hello"),
+		Code:    http.StatusForbidden,
+	}).Run(handler, t)
+}

s3store/s3store_mock_test.go

@@ -907,6 +907,16 @@ func (_mr *_MockS3APIRecorder) ListObjectsV2(arg0 interface{}) *gomock.Call {
 	return _mr.mock.ctrl.RecordCall(_mr.mock, "ListObjectsV2", arg0)
 }
 
+func (_m *MockS3API) ListObjectsV2Pages(_param0 *s3.ListObjectsV2Input, _param1 func(*s3.ListObjectsV2Output, bool) bool) error {
+	ret := _m.ctrl.Call(_m, "ListObjectsV2Pages", _param0, _param1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+func (_mr *_MockS3APIRecorder) ListObjectsV2Pages(arg0, arg1 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "ListObjectsV2Pages", arg0, arg1)
+}
+
 func (_m *MockS3API) ListObjectsV2Request(_param0 *s3.ListObjectsV2Input) (*request.Request, *s3.ListObjectsV2Output) {
 	ret := _m.ctrl.Call(_m, "ListObjectsV2Request", _param0)
 	ret0, _ := ret[0].(*request.Request)

terminate_test.go

@@ -44,7 +44,7 @@ func TestTerminate(t *testing.T) {
 		Method: "OPTIONS",
 		URL:    "",
 		ResHeader: map[string]string{
-			"Tus-Extension": "creation,termination",
+			"Tus-Extension": "creation,creation-with-upload,termination",
 		},
 		Code: http.StatusOK,
 	}).Run(handler, t)

unrouted_handler.go

@@ -89,7 +89,7 @@ func NewUnroutedHandler(config Config) (*UnroutedHandler, error) {
 	}
 
 	// Only promote extesions using the Tus-Extension header which are implemented
-	extensions := "creation"
+	extensions := "creation,creation-with-upload"
 	if config.StoreComposer.UsesTerminater {
 		extensions += ",termination"
 	}
@@ -191,6 +191,16 @@ func (handler *UnroutedHandler) Middleware(h http.Handler) http.Handler {
 // PostFile creates a new file upload using the datastore after validating the
 // length and parsing the metadata.
 func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) {
+	// Check for presence of application/offset+octet-stream
+	containsChunk := false
+	if contentType := r.Header.Get("Content-Type"); contentType != "" {
+		if contentType != "application/offset+octet-stream" {
+			handler.sendError(w, r, ErrInvalidContentType)
+			return
+		}
+		containsChunk = true
+	}
+
 	// Only use the proper Upload-Concat header if the concatenation extension
 	// is even supported by the data store.
 	var concatHeader string
@@ -210,6 +220,12 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
 	// Upload-Length header)
 	var size int64
 	if isFinal {
+		// A final upload must not contain a chunk within the creation request
+		if containsChunk {
+			handler.sendError(w, r, ErrModifyFinal)
+			return
+		}
+
 		size, err = handler.sizeOfUploads(partialUploads)
 		if err != nil {
 			handler.sendError(w, r, err)
@@ -246,6 +262,13 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	// Add the Location header directly after creating the new resource to even
+	// include it in cases of failure when an error is returned
+	url := handler.absFileURL(r, id)
+	w.Header().Set("Location", url)
+
+	go handler.Metrics.incUploadsCreated()
+
 	if isFinal {
 		if err := handler.composer.Concater.ConcatUploads(id, partialUploads); err != nil {
 			handler.sendError(w, r, err)
@@ -257,15 +280,26 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
 			info.ID = id
 			handler.CompleteUploads <- info
 		}
-
-		go handler.Metrics.incUploadsFinished()
 	}
 
-	url := handler.absFileURL(r, id)
-	w.Header().Set("Location", url)
-	w.WriteHeader(http.StatusCreated)
+	if containsChunk {
+		if handler.composer.UsesLocker {
+			locker := handler.composer.Locker
+			if err := locker.LockUpload(id); err != nil {
+				handler.sendError(w, r, err)
+				return
+			}
 
-	go handler.Metrics.incUploadsCreated()
+			defer locker.UnlockUpload(id)
+		}
+
+		if err := handler.writeChunk(id, info, w, r); err != nil {
+			handler.sendError(w, r, err)
+			return
+		}
+	}
+
+	w.WriteHeader(http.StatusCreated)
 }
 
 // HeadFile returns the length and offset for the HEAD request
@@ -313,7 +347,7 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request)
 	w.Header().Set("Cache-Control", "no-store")
 	w.Header().Set("Upload-Length", strconv.FormatInt(info.Size, 10))
 	w.Header().Set("Upload-Offset", strconv.FormatInt(info.Offset, 10))
-	w.WriteHeader(http.StatusNoContent)
+	w.WriteHeader(http.StatusOK)
 }
 
 // PatchFile adds a chunk to an upload. Only allowed enough space is left.
@@ -372,13 +406,23 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
 		return
 	}
 
+	if err := handler.writeChunk(id, info, w, r); err != nil {
+		handler.sendError(w, r, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// PatchFile adds a chunk to an upload. Only allowed enough space is left.
+func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.ResponseWriter, r *http.Request) error {
 	// Get Content-Length if possible
 	length := r.ContentLength
+	offset := info.Offset
 
 	// Test if this upload fits into the file's size
 	if offset+length > info.Size {
-		handler.sendError(w, r, ErrSizeExceeded)
-		return
+		return ErrSizeExceeded
 	}
 
 	maxSize := info.Size - offset
@@ -386,13 +430,18 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
 		maxSize = length
 	}
 
-	// Limit the data read from the request's body to the allowed maxiumum
-	reader := io.LimitReader(r.Body, maxSize)
+	var bytesWritten int64
+	// Prevent a nil pointer derefernce when accessing the body which may not be
+	// available in the case of a malicious request.
+	if r.Body != nil {
+		// Limit the data read from the request's body to the allowed maxiumum
+		reader := io.LimitReader(r.Body, maxSize)
 
-	bytesWritten, err := handler.composer.Core.WriteChunk(id, offset, reader)
-	if err != nil {
-		handler.sendError(w, r, err)
-		return
+		var err error
+		bytesWritten, err = handler.composer.Core.WriteChunk(id, offset, reader)
+		if err != nil {
+			return err
+		}
 	}
 
 	// Send new offset to client
@@ -405,8 +454,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
 		// ... allow custom mechanism to finish and cleanup the upload
 		if handler.composer.UsesFinisher {
 			if err := handler.composer.Finisher.FinishUpload(id); err != nil {
-				handler.sendError(w, r, err)
-				return
+				return err
 			}
 		}
 
@@ -419,7 +467,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
 		go handler.Metrics.incUploadsFinished()
 	}
 
-	w.WriteHeader(http.StatusNoContent)
+	return nil
 }
 
 // GetFile handles requests to download a file using a GET request. This is not

vendor/github.com/aws/aws-sdk-go

@@ -1 +0,0 @@
-Subproject commit c76e8918e8f08490e3bb154178a84a0b2bdf8d6e

vendor/github.com/hashicorp/consul

@@ -1 +0,0 @@
-Subproject commit f6fef66e1bf17be4f3c9855fbec6de802ca6bd7d