Merge branch 'master' of github.com:tus/tusd

This commit is contained in:
Kevin van Zonneveld 2016-09-15 10:18:25 +02:00
commit bea4d10ae6
17 changed files with 202 additions and 62 deletions

6
.gitmodules vendored
View File

@ -4,9 +4,6 @@
[submodule "vendor/github.com/nightlyone/lockfile"] [submodule "vendor/github.com/nightlyone/lockfile"]
path = vendor/github.com/nightlyone/lockfile path = vendor/github.com/nightlyone/lockfile
url = https://github.com/nightlyone/lockfile url = https://github.com/nightlyone/lockfile
[submodule "vendor/github.com/aws/aws-sdk-go"]
path = vendor/github.com/aws/aws-sdk-go
url = https://github.com/aws/aws-sdk-go
[submodule "vendor/github.com/go-ini/ini"] [submodule "vendor/github.com/go-ini/ini"]
path = vendor/github.com/go-ini/ini path = vendor/github.com/go-ini/ini
url = https://github.com/go-ini/ini url = https://github.com/go-ini/ini
@ -25,9 +22,6 @@
[submodule "vendor/github.com/pmezard/go-difflib"] [submodule "vendor/github.com/pmezard/go-difflib"]
path = vendor/github.com/pmezard/go-difflib path = vendor/github.com/pmezard/go-difflib
url = https://github.com/pmezard/go-difflib url = https://github.com/pmezard/go-difflib
[submodule "vendor/github.com/hashicorp/consul"]
path = vendor/github.com/hashicorp/consul
url = https://github.com/hashicorp/consul
[submodule "vendor/github.com/hashicorp/go-cleanhttp"] [submodule "vendor/github.com/hashicorp/go-cleanhttp"]
path = vendor/github.com/hashicorp/go-cleanhttp path = vendor/github.com/hashicorp/go-cleanhttp
url = https://github.com/hashicorp/go-cleanhttp url = https://github.com/hashicorp/go-cleanhttp

View File

@ -4,6 +4,7 @@ go:
- 1.4 - 1.4
- 1.5 - 1.5
- 1.6 - 1.6
- 1.7
- tip - tip
sudo: required sudo: required
cache: cache:
@ -22,6 +23,7 @@ matrix:
install: install:
- export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \)) - export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \))
- rsync -r ./vendor/ $GOPATH/src - rsync -r ./vendor/ $GOPATH/src
- go get $PACKAGES
script: script:
- go test $PACKAGES - go test $PACKAGES
before_deploy: before_deploy:

View File

@ -25,7 +25,7 @@ func CreateComposer() {
dir := Flags.UploadDir dir := Flags.UploadDir
stdout.Printf("Using '%s' as directory storage.\n", dir) stdout.Printf("Using '%s' as directory storage.\n", dir)
if err := os.MkdirAll(dir, os.FileMode(0775)); err != nil { if err := os.MkdirAll(dir, os.FileMode(0774)); err != nil {
stderr.Fatalf("Unable to ensure directory exists: %s", err) stderr.Fatalf("Unable to ensure directory exists: %s", err)
} }

View File

@ -17,6 +17,7 @@ var Flags struct {
HooksDir string HooksDir string
ShowVersion bool ShowVersion bool
ExposeMetrics bool ExposeMetrics bool
MetricsPath string
BehindProxy bool BehindProxy bool
HooksInstalled bool HooksInstalled bool
@ -34,6 +35,7 @@ func ParseFlags() {
flag.StringVar(&Flags.HooksDir, "hooks-dir", "", "Directory to search for available hooks scripts") flag.StringVar(&Flags.HooksDir, "hooks-dir", "", "Directory to search for available hooks scripts")
flag.BoolVar(&Flags.ShowVersion, "version", false, "Print tusd version information") flag.BoolVar(&Flags.ShowVersion, "version", false, "Print tusd version information")
flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage") flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage")
flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible")
flag.BoolVar(&Flags.BehindProxy, "behind-proxy", false, "Respect X-Forwarded-* and similar headers which may be set by proxies") flag.BoolVar(&Flags.BehindProxy, "behind-proxy", false, "Respect X-Forwarded-* and similar headers which may be set by proxies")
flag.Parse() flag.Parse()

View File

@ -12,19 +12,23 @@ func PrepareGreeting() {
`Welcome to tusd `Welcome to tusd
=============== ===============
Congratulations for setting up tusd! You are now part of the chosen elite and Congratulations on setting up tusd! Thanks for joining our cause, you have taken
able to experience the feeling of resumable uploads! We hope you are as excited the first step towards making the future of resumable uploading a reality! We
as we are (a lot)! hope you are as excited about this as we are!
However, there is something you should be aware of: While you got tusd While you did an awesome job on getting tusd running, this is just the welcome
running (you did an awesome job!), this is the root directory of the server message, so let's talk about the places that really matter:
and tus requests are only accepted at the %s route.
So don't waste time, head over there and experience the future! - %s - send your tus uploads to this endpoint
- %s - gather statistics to keep tusd running smoothly
- https://github.com/tus/tusd/issues - report your bugs here
So quit lollygagging, send over your files and experience the future!
Version = %s Version = %s
GitCommit = %s GitCommit = %s
BuildDate = %s`, Flags.Basepath, VersionName, GitCommit, BuildDate) BuildDate = %s
`, Flags.Basepath, Flags.MetricsPath, VersionName, GitCommit, BuildDate)
} }
func DisplayGreeting(w http.ResponseWriter, r *http.Request) { func DisplayGreeting(w http.ResponseWriter, r *http.Request) {

View File

@ -18,5 +18,6 @@ func SetupMetrics(handler *tusd.Handler) {
prometheus.MustRegister(MetricsOpenConnections) prometheus.MustRegister(MetricsOpenConnections)
prometheus.MustRegister(prometheuscollector.New(handler.Metrics)) prometheus.MustRegister(prometheuscollector.New(handler.Metrics))
http.Handle("/metrics", prometheus.Handler()) stdout.Printf("Using %s as the metrics path.\n", Flags.MetricsPath)
http.Handle(Flags.MetricsPath, prometheus.Handler())
} }

View File

@ -27,7 +27,6 @@ func Serve() {
stdout.Printf("Using %s as address to listen.\n", address) stdout.Printf("Using %s as address to listen.\n", address)
stdout.Printf("Using %s as the base path.\n", basepath) stdout.Printf("Using %s as the base path.\n", basepath)
stdout.Printf(Composer.Capabilities())
SetupPostHooks(handler) SetupPostHooks(handler)
@ -35,6 +34,8 @@ func Serve() {
SetupMetrics(handler) SetupMetrics(handler)
} }
stdout.Printf(Composer.Capabilities())
// Do not display the greeting if the tusd handler will be mounted at the root // Do not display the greeting if the tusd handler will be mounted at the root
// path. Else this would cause a "multiple registrations for /" panic. // path. Else this would cause a "multiple registrations for /" panic.
if basepath != "/" { if basepath != "/" {

View File

@ -46,7 +46,7 @@ func TestConcatPartial(t *testing.T) {
Method: "OPTIONS", Method: "OPTIONS",
URL: "", URL: "",
ResHeader: map[string]string{ ResHeader: map[string]string{
"Tus-Extension": "creation,concatenation", "Tus-Extension": "creation,creation-with-upload,concatenation",
}, },
Code: http.StatusOK, Code: http.StatusOK,
}).Run(handler, t) }).Run(handler, t)
@ -69,7 +69,7 @@ func TestConcatPartial(t *testing.T) {
ReqHeader: map[string]string{ ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0", "Tus-Resumable": "1.0.0",
}, },
Code: http.StatusNoContent, Code: http.StatusOK,
ResHeader: map[string]string{ ResHeader: map[string]string{
"Upload-Concat": "partial", "Upload-Concat": "partial",
}, },
@ -165,7 +165,7 @@ func TestConcatFinal(t *testing.T) {
ReqHeader: map[string]string{ ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0", "Tus-Resumable": "1.0.0",
}, },
Code: http.StatusNoContent, Code: http.StatusOK,
ResHeader: map[string]string{ ResHeader: map[string]string{
"Upload-Concat": "final; http://tus.io/files/a http://tus.io/files/b", "Upload-Concat": "final; http://tus.io/files/a http://tus.io/files/b",
"Upload-Length": "10", "Upload-Length": "10",

View File

@ -28,7 +28,7 @@ import (
"github.com/nightlyone/lockfile" "github.com/nightlyone/lockfile"
) )
var defaultFilePerm = os.FileMode(0775) var defaultFilePerm = os.FileMode(0664)
// See the tusd.DataStore interface for documentation about the different // See the tusd.DataStore interface for documentation about the different
// methods. // methods.

View File

@ -40,7 +40,7 @@ func TestHead(t *testing.T) {
ReqHeader: map[string]string{ ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0", "Tus-Resumable": "1.0.0",
}, },
Code: http.StatusNoContent, Code: http.StatusOK,
ResHeader: map[string]string{ ResHeader: map[string]string{
"Upload-Offset": "11", "Upload-Offset": "11",
"Upload-Length": "44", "Upload-Length": "44",

View File

@ -20,7 +20,7 @@ func TestOptions(t *testing.T) {
Method: "OPTIONS", Method: "OPTIONS",
Code: http.StatusOK, Code: http.StatusOK,
ResHeader: map[string]string{ ResHeader: map[string]string{
"Tus-Extension": "creation", "Tus-Extension": "creation,creation-with-upload",
"Tus-Version": "1.0.0", "Tus-Version": "1.0.0",
"Tus-Resumable": "1.0.0", "Tus-Resumable": "1.0.0",
"Tus-Max-Size": "400", "Tus-Max-Size": "400",

View File

@ -1,44 +1,57 @@
package tusd_test package tusd_test
import ( import (
"bytes"
"io"
"io/ioutil"
"net/http" "net/http"
"strings"
"testing" "testing"
"github.com/stretchr/testify/assert"
. "github.com/tus/tusd" . "github.com/tus/tusd"
) )
type postStore struct { type postStore struct {
t *testing.T t *assert.Assertions
zeroStore zeroStore
} }
func (s postStore) NewUpload(info FileInfo) (string, error) { func (s postStore) NewUpload(info FileInfo) (string, error) {
if info.Size != 300 { s.t.Equal(int64(300), info.Size)
s.t.Errorf("Expected size to be 300 (got %v)", info.Size)
}
metaData := info.MetaData metaData := info.MetaData
if len(metaData) != 2 { s.t.Equal(2, len(metaData))
s.t.Errorf("Expected two elements in metadata") s.t.Equal("hello", metaData["foo"])
} s.t.Equal("world", metaData["bar"])
if v := metaData["foo"]; v != "hello" {
s.t.Errorf("Expected foo element to be 'hello' but got %s", v)
}
if v := metaData["bar"]; v != "world" {
s.t.Errorf("Expected bar element to be 'world' but got %s", v)
}
return "foo", nil return "foo", nil
} }
func (s postStore) WriteChunk(id string, offset int64, src io.Reader) (int64, error) {
s.t.Equal(int64(0), offset)
data, err := ioutil.ReadAll(src)
s.t.Nil(err)
s.t.Equal("hello", string(data))
return 5, nil
}
func (s postStore) ConcatUploads(id string, uploads []string) error {
s.t.True(false, "concatenation should not be attempted")
return nil
}
func TestPost(t *testing.T) { func TestPost(t *testing.T) {
a := assert.New(t)
handler, _ := NewHandler(Config{ handler, _ := NewHandler(Config{
MaxSize: 400, MaxSize: 400,
BasePath: "files", BasePath: "files",
DataStore: postStore{ DataStore: postStore{
t: t, t: a,
}, },
}) })
@ -87,7 +100,7 @@ func TestPost(t *testing.T) {
MaxSize: 400, MaxSize: 400,
BasePath: "files", BasePath: "files",
DataStore: postStore{ DataStore: postStore{
t: t, t: a,
}, },
RespectForwardedHeaders: true, RespectForwardedHeaders: true,
}) })
@ -141,3 +154,70 @@ func TestPost(t *testing.T) {
}, },
}).Run(handler, t) }).Run(handler, t)
} }
func TestPostWithUpload(t *testing.T) {
a := assert.New(t)
handler, _ := NewHandler(Config{
MaxSize: 400,
BasePath: "files",
DataStore: postStore{
t: a,
},
})
(&httpTest{
Name: "Successful request",
Method: "POST",
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
"Upload-Length": "300",
"Content-Type": "application/offset+octet-stream",
"Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=",
},
ReqBody: strings.NewReader("hello"),
Code: http.StatusCreated,
ResHeader: map[string]string{
"Location": "http://tus.io/files/foo",
"Upload-Offset": "5",
},
}).Run(handler, t)
(&httpTest{
Name: "Exceeding upload size",
Method: "POST",
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
"Upload-Length": "300",
"Content-Type": "application/offset+octet-stream",
"Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=",
},
ReqBody: bytes.NewReader(make([]byte, 400)),
Code: http.StatusRequestEntityTooLarge,
}).Run(handler, t)
(&httpTest{
Name: "Incorrect content type",
Method: "POST",
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
"Content-Type": "application/false",
},
ReqBody: strings.NewReader("hello"),
Code: http.StatusBadRequest,
}).Run(handler, t)
(&httpTest{
Name: "Upload and final concatenation",
Method: "POST",
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
"Upload-Length": "300",
"Content-Type": "application/offset+octet-stream",
"Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=",
"Upload-Concat": "final; http://tus.io/files/a http://tus.io/files/b",
},
ReqBody: strings.NewReader("hello"),
Code: http.StatusForbidden,
}).Run(handler, t)
}

View File

@ -907,6 +907,16 @@ func (_mr *_MockS3APIRecorder) ListObjectsV2(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "ListObjectsV2", arg0) return _mr.mock.ctrl.RecordCall(_mr.mock, "ListObjectsV2", arg0)
} }
func (_m *MockS3API) ListObjectsV2Pages(_param0 *s3.ListObjectsV2Input, _param1 func(*s3.ListObjectsV2Output, bool) bool) error {
ret := _m.ctrl.Call(_m, "ListObjectsV2Pages", _param0, _param1)
ret0, _ := ret[0].(error)
return ret0
}
func (_mr *_MockS3APIRecorder) ListObjectsV2Pages(arg0, arg1 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "ListObjectsV2Pages", arg0, arg1)
}
func (_m *MockS3API) ListObjectsV2Request(_param0 *s3.ListObjectsV2Input) (*request.Request, *s3.ListObjectsV2Output) { func (_m *MockS3API) ListObjectsV2Request(_param0 *s3.ListObjectsV2Input) (*request.Request, *s3.ListObjectsV2Output) {
ret := _m.ctrl.Call(_m, "ListObjectsV2Request", _param0) ret := _m.ctrl.Call(_m, "ListObjectsV2Request", _param0)
ret0, _ := ret[0].(*request.Request) ret0, _ := ret[0].(*request.Request)

View File

@ -44,7 +44,7 @@ func TestTerminate(t *testing.T) {
Method: "OPTIONS", Method: "OPTIONS",
URL: "", URL: "",
ResHeader: map[string]string{ ResHeader: map[string]string{
"Tus-Extension": "creation,termination", "Tus-Extension": "creation,creation-with-upload,termination",
}, },
Code: http.StatusOK, Code: http.StatusOK,
}).Run(handler, t) }).Run(handler, t)

View File

@ -89,7 +89,7 @@ func NewUnroutedHandler(config Config) (*UnroutedHandler, error) {
} }
// Only promote extesions using the Tus-Extension header which are implemented // Only promote extesions using the Tus-Extension header which are implemented
extensions := "creation" extensions := "creation,creation-with-upload"
if config.StoreComposer.UsesTerminater { if config.StoreComposer.UsesTerminater {
extensions += ",termination" extensions += ",termination"
} }
@ -191,6 +191,16 @@ func (handler *UnroutedHandler) Middleware(h http.Handler) http.Handler {
// PostFile creates a new file upload using the datastore after validating the // PostFile creates a new file upload using the datastore after validating the
// length and parsing the metadata. // length and parsing the metadata.
func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) { func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) {
// Check for presence of application/offset+octet-stream
containsChunk := false
if contentType := r.Header.Get("Content-Type"); contentType != "" {
if contentType != "application/offset+octet-stream" {
handler.sendError(w, r, ErrInvalidContentType)
return
}
containsChunk = true
}
// Only use the proper Upload-Concat header if the concatenation extension // Only use the proper Upload-Concat header if the concatenation extension
// is even supported by the data store. // is even supported by the data store.
var concatHeader string var concatHeader string
@ -210,6 +220,12 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
// Upload-Length header) // Upload-Length header)
var size int64 var size int64
if isFinal { if isFinal {
// A final upload must not contain a chunk within the creation request
if containsChunk {
handler.sendError(w, r, ErrModifyFinal)
return
}
size, err = handler.sizeOfUploads(partialUploads) size, err = handler.sizeOfUploads(partialUploads)
if err != nil { if err != nil {
handler.sendError(w, r, err) handler.sendError(w, r, err)
@ -246,6 +262,13 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
return return
} }
// Add the Location header directly after creating the new resource to even
// include it in cases of failure when an error is returned
url := handler.absFileURL(r, id)
w.Header().Set("Location", url)
go handler.Metrics.incUploadsCreated()
if isFinal { if isFinal {
if err := handler.composer.Concater.ConcatUploads(id, partialUploads); err != nil { if err := handler.composer.Concater.ConcatUploads(id, partialUploads); err != nil {
handler.sendError(w, r, err) handler.sendError(w, r, err)
@ -257,15 +280,26 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
info.ID = id info.ID = id
handler.CompleteUploads <- info handler.CompleteUploads <- info
} }
go handler.Metrics.incUploadsFinished()
} }
url := handler.absFileURL(r, id) if containsChunk {
w.Header().Set("Location", url) if handler.composer.UsesLocker {
w.WriteHeader(http.StatusCreated) locker := handler.composer.Locker
if err := locker.LockUpload(id); err != nil {
handler.sendError(w, r, err)
return
}
go handler.Metrics.incUploadsCreated() defer locker.UnlockUpload(id)
}
if err := handler.writeChunk(id, info, w, r); err != nil {
handler.sendError(w, r, err)
return
}
}
w.WriteHeader(http.StatusCreated)
} }
// HeadFile returns the length and offset for the HEAD request // HeadFile returns the length and offset for the HEAD request
@ -313,7 +347,7 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request)
w.Header().Set("Cache-Control", "no-store") w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Upload-Length", strconv.FormatInt(info.Size, 10)) w.Header().Set("Upload-Length", strconv.FormatInt(info.Size, 10))
w.Header().Set("Upload-Offset", strconv.FormatInt(info.Offset, 10)) w.Header().Set("Upload-Offset", strconv.FormatInt(info.Offset, 10))
w.WriteHeader(http.StatusNoContent) w.WriteHeader(http.StatusOK)
} }
// PatchFile adds a chunk to an upload. Only allowed enough space is left. // PatchFile adds a chunk to an upload. Only allowed enough space is left.
@ -372,13 +406,23 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
return return
} }
if err := handler.writeChunk(id, info, w, r); err != nil {
handler.sendError(w, r, err)
return
}
w.WriteHeader(http.StatusNoContent)
}
// PatchFile adds a chunk to an upload. Only allowed enough space is left.
func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.ResponseWriter, r *http.Request) error {
// Get Content-Length if possible // Get Content-Length if possible
length := r.ContentLength length := r.ContentLength
offset := info.Offset
// Test if this upload fits into the file's size // Test if this upload fits into the file's size
if offset+length > info.Size { if offset+length > info.Size {
handler.sendError(w, r, ErrSizeExceeded) return ErrSizeExceeded
return
} }
maxSize := info.Size - offset maxSize := info.Size - offset
@ -386,13 +430,18 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
maxSize = length maxSize = length
} }
// Limit the data read from the request's body to the allowed maxiumum var bytesWritten int64
reader := io.LimitReader(r.Body, maxSize) // Prevent a nil pointer derefernce when accessing the body which may not be
// available in the case of a malicious request.
if r.Body != nil {
// Limit the data read from the request's body to the allowed maxiumum
reader := io.LimitReader(r.Body, maxSize)
bytesWritten, err := handler.composer.Core.WriteChunk(id, offset, reader) var err error
if err != nil { bytesWritten, err = handler.composer.Core.WriteChunk(id, offset, reader)
handler.sendError(w, r, err) if err != nil {
return return err
}
} }
// Send new offset to client // Send new offset to client
@ -405,8 +454,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
// ... allow custom mechanism to finish and cleanup the upload // ... allow custom mechanism to finish and cleanup the upload
if handler.composer.UsesFinisher { if handler.composer.UsesFinisher {
if err := handler.composer.Finisher.FinishUpload(id); err != nil { if err := handler.composer.Finisher.FinishUpload(id); err != nil {
handler.sendError(w, r, err) return err
return
} }
} }
@ -419,7 +467,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
go handler.Metrics.incUploadsFinished() go handler.Metrics.incUploadsFinished()
} }
w.WriteHeader(http.StatusNoContent) return nil
} }
// GetFile handles requests to download a file using a GET request. This is not // GetFile handles requests to download a file using a GET request. This is not

1
vendor/github.com/aws/aws-sdk-go generated vendored

@ -1 +0,0 @@
Subproject commit c76e8918e8f08490e3bb154178a84a0b2bdf8d6e

1
vendor/github.com/hashicorp/consul generated vendored

@ -1 +0,0 @@
Subproject commit f6fef66e1bf17be4f3c9855fbec6de802ca6bd7d