Merge branch 'master' of github.com:tus/tusd

This commit is contained in:
Kevin van Zonneveld 2016-09-15 10:18:25 +02:00
commit bea4d10ae6
17 changed files with 202 additions and 62 deletions

6
.gitmodules vendored
View File

@ -4,9 +4,6 @@
[submodule "vendor/github.com/nightlyone/lockfile"]
path = vendor/github.com/nightlyone/lockfile
url = https://github.com/nightlyone/lockfile
[submodule "vendor/github.com/aws/aws-sdk-go"]
path = vendor/github.com/aws/aws-sdk-go
url = https://github.com/aws/aws-sdk-go
[submodule "vendor/github.com/go-ini/ini"]
path = vendor/github.com/go-ini/ini
url = https://github.com/go-ini/ini
@ -25,9 +22,6 @@
[submodule "vendor/github.com/pmezard/go-difflib"]
path = vendor/github.com/pmezard/go-difflib
url = https://github.com/pmezard/go-difflib
[submodule "vendor/github.com/hashicorp/consul"]
path = vendor/github.com/hashicorp/consul
url = https://github.com/hashicorp/consul
[submodule "vendor/github.com/hashicorp/go-cleanhttp"]
path = vendor/github.com/hashicorp/go-cleanhttp
url = https://github.com/hashicorp/go-cleanhttp

View File

@ -4,6 +4,7 @@ go:
- 1.4
- 1.5
- 1.6
- 1.7
- tip
sudo: required
cache:
@ -22,6 +23,7 @@ matrix:
install:
- export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \))
- rsync -r ./vendor/ $GOPATH/src
- go get $PACKAGES
script:
- go test $PACKAGES
before_deploy:

View File

@ -25,7 +25,7 @@ func CreateComposer() {
dir := Flags.UploadDir
stdout.Printf("Using '%s' as directory storage.\n", dir)
if err := os.MkdirAll(dir, os.FileMode(0775)); err != nil {
if err := os.MkdirAll(dir, os.FileMode(0774)); err != nil {
stderr.Fatalf("Unable to ensure directory exists: %s", err)
}

View File

@ -17,6 +17,7 @@ var Flags struct {
HooksDir string
ShowVersion bool
ExposeMetrics bool
MetricsPath string
BehindProxy bool
HooksInstalled bool
@ -34,6 +35,7 @@ func ParseFlags() {
flag.StringVar(&Flags.HooksDir, "hooks-dir", "", "Directory to search for available hooks scripts")
flag.BoolVar(&Flags.ShowVersion, "version", false, "Print tusd version information")
flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage")
flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible")
flag.BoolVar(&Flags.BehindProxy, "behind-proxy", false, "Respect X-Forwarded-* and similar headers which may be set by proxies")
flag.Parse()

View File

@ -12,19 +12,23 @@ func PrepareGreeting() {
`Welcome to tusd
===============
Congratulations for setting up tusd! You are now part of the chosen elite and
able to experience the feeling of resumable uploads! We hope you are as excited
as we are (a lot)!
Congratulations on setting up tusd! Thanks for joining our cause, you have taken
the first step towards making the future of resumable uploading a reality! We
hope you are as excited about this as we are!
However, there is something you should be aware of: While you got tusd
running (you did an awesome job!), this is the root directory of the server
and tus requests are only accepted at the %s route.
While you did an awesome job on getting tusd running, this is just the welcome
message, so let's talk about the places that really matter:
So don't waste time, head over there and experience the future!
- %s - send your tus uploads to this endpoint
- %s - gather statistics to keep tusd running smoothly
- https://github.com/tus/tusd/issues - report your bugs here
So quit lollygagging, send over your files and experience the future!
Version = %s
GitCommit = %s
BuildDate = %s`, Flags.Basepath, VersionName, GitCommit, BuildDate)
BuildDate = %s
`, Flags.Basepath, Flags.MetricsPath, VersionName, GitCommit, BuildDate)
}
func DisplayGreeting(w http.ResponseWriter, r *http.Request) {

View File

@ -18,5 +18,6 @@ func SetupMetrics(handler *tusd.Handler) {
prometheus.MustRegister(MetricsOpenConnections)
prometheus.MustRegister(prometheuscollector.New(handler.Metrics))
http.Handle("/metrics", prometheus.Handler())
stdout.Printf("Using %s as the metrics path.\n", Flags.MetricsPath)
http.Handle(Flags.MetricsPath, prometheus.Handler())
}

View File

@ -27,7 +27,6 @@ func Serve() {
stdout.Printf("Using %s as address to listen.\n", address)
stdout.Printf("Using %s as the base path.\n", basepath)
stdout.Printf(Composer.Capabilities())
SetupPostHooks(handler)
@ -35,6 +34,8 @@ func Serve() {
SetupMetrics(handler)
}
stdout.Printf(Composer.Capabilities())
// Do not display the greeting if the tusd handler will be mounted at the root
// path. Else this would cause a "multiple registrations for /" panic.
if basepath != "/" {

View File

@ -46,7 +46,7 @@ func TestConcatPartial(t *testing.T) {
Method: "OPTIONS",
URL: "",
ResHeader: map[string]string{
"Tus-Extension": "creation,concatenation",
"Tus-Extension": "creation,creation-with-upload,concatenation",
},
Code: http.StatusOK,
}).Run(handler, t)
@ -69,7 +69,7 @@ func TestConcatPartial(t *testing.T) {
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
},
Code: http.StatusNoContent,
Code: http.StatusOK,
ResHeader: map[string]string{
"Upload-Concat": "partial",
},
@ -165,7 +165,7 @@ func TestConcatFinal(t *testing.T) {
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
},
Code: http.StatusNoContent,
Code: http.StatusOK,
ResHeader: map[string]string{
"Upload-Concat": "final; http://tus.io/files/a http://tus.io/files/b",
"Upload-Length": "10",

View File

@ -28,7 +28,7 @@ import (
"github.com/nightlyone/lockfile"
)
var defaultFilePerm = os.FileMode(0775)
var defaultFilePerm = os.FileMode(0664)
// See the tusd.DataStore interface for documentation about the different
// methods.

View File

@ -40,7 +40,7 @@ func TestHead(t *testing.T) {
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
},
Code: http.StatusNoContent,
Code: http.StatusOK,
ResHeader: map[string]string{
"Upload-Offset": "11",
"Upload-Length": "44",

View File

@ -20,7 +20,7 @@ func TestOptions(t *testing.T) {
Method: "OPTIONS",
Code: http.StatusOK,
ResHeader: map[string]string{
"Tus-Extension": "creation",
"Tus-Extension": "creation,creation-with-upload",
"Tus-Version": "1.0.0",
"Tus-Resumable": "1.0.0",
"Tus-Max-Size": "400",

View File

@ -1,44 +1,57 @@
package tusd_test
import (
"bytes"
"io"
"io/ioutil"
"net/http"
"strings"
"testing"
"github.com/stretchr/testify/assert"
. "github.com/tus/tusd"
)
type postStore struct {
t *testing.T
t *assert.Assertions
zeroStore
}
func (s postStore) NewUpload(info FileInfo) (string, error) {
if info.Size != 300 {
s.t.Errorf("Expected size to be 300 (got %v)", info.Size)
}
s.t.Equal(int64(300), info.Size)
metaData := info.MetaData
if len(metaData) != 2 {
s.t.Errorf("Expected two elements in metadata")
}
if v := metaData["foo"]; v != "hello" {
s.t.Errorf("Expected foo element to be 'hello' but got %s", v)
}
if v := metaData["bar"]; v != "world" {
s.t.Errorf("Expected bar element to be 'world' but got %s", v)
}
s.t.Equal(2, len(metaData))
s.t.Equal("hello", metaData["foo"])
s.t.Equal("world", metaData["bar"])
return "foo", nil
}
func (s postStore) WriteChunk(id string, offset int64, src io.Reader) (int64, error) {
s.t.Equal(int64(0), offset)
data, err := ioutil.ReadAll(src)
s.t.Nil(err)
s.t.Equal("hello", string(data))
return 5, nil
}
func (s postStore) ConcatUploads(id string, uploads []string) error {
s.t.True(false, "concatenation should not be attempted")
return nil
}
func TestPost(t *testing.T) {
a := assert.New(t)
handler, _ := NewHandler(Config{
MaxSize: 400,
BasePath: "files",
DataStore: postStore{
t: t,
t: a,
},
})
@ -87,7 +100,7 @@ func TestPost(t *testing.T) {
MaxSize: 400,
BasePath: "files",
DataStore: postStore{
t: t,
t: a,
},
RespectForwardedHeaders: true,
})
@ -141,3 +154,70 @@ func TestPost(t *testing.T) {
},
}).Run(handler, t)
}
func TestPostWithUpload(t *testing.T) {
a := assert.New(t)
handler, _ := NewHandler(Config{
MaxSize: 400,
BasePath: "files",
DataStore: postStore{
t: a,
},
})
(&httpTest{
Name: "Successful request",
Method: "POST",
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
"Upload-Length": "300",
"Content-Type": "application/offset+octet-stream",
"Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=",
},
ReqBody: strings.NewReader("hello"),
Code: http.StatusCreated,
ResHeader: map[string]string{
"Location": "http://tus.io/files/foo",
"Upload-Offset": "5",
},
}).Run(handler, t)
(&httpTest{
Name: "Exceeding upload size",
Method: "POST",
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
"Upload-Length": "300",
"Content-Type": "application/offset+octet-stream",
"Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=",
},
ReqBody: bytes.NewReader(make([]byte, 400)),
Code: http.StatusRequestEntityTooLarge,
}).Run(handler, t)
(&httpTest{
Name: "Incorrect content type",
Method: "POST",
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
"Content-Type": "application/false",
},
ReqBody: strings.NewReader("hello"),
Code: http.StatusBadRequest,
}).Run(handler, t)
(&httpTest{
Name: "Upload and final concatenation",
Method: "POST",
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
"Upload-Length": "300",
"Content-Type": "application/offset+octet-stream",
"Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=",
"Upload-Concat": "final; http://tus.io/files/a http://tus.io/files/b",
},
ReqBody: strings.NewReader("hello"),
Code: http.StatusForbidden,
}).Run(handler, t)
}

View File

@ -907,6 +907,16 @@ func (_mr *_MockS3APIRecorder) ListObjectsV2(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "ListObjectsV2", arg0)
}
func (_m *MockS3API) ListObjectsV2Pages(_param0 *s3.ListObjectsV2Input, _param1 func(*s3.ListObjectsV2Output, bool) bool) error {
ret := _m.ctrl.Call(_m, "ListObjectsV2Pages", _param0, _param1)
ret0, _ := ret[0].(error)
return ret0
}
func (_mr *_MockS3APIRecorder) ListObjectsV2Pages(arg0, arg1 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "ListObjectsV2Pages", arg0, arg1)
}
func (_m *MockS3API) ListObjectsV2Request(_param0 *s3.ListObjectsV2Input) (*request.Request, *s3.ListObjectsV2Output) {
ret := _m.ctrl.Call(_m, "ListObjectsV2Request", _param0)
ret0, _ := ret[0].(*request.Request)

View File

@ -44,7 +44,7 @@ func TestTerminate(t *testing.T) {
Method: "OPTIONS",
URL: "",
ResHeader: map[string]string{
"Tus-Extension": "creation,termination",
"Tus-Extension": "creation,creation-with-upload,termination",
},
Code: http.StatusOK,
}).Run(handler, t)

View File

@ -89,7 +89,7 @@ func NewUnroutedHandler(config Config) (*UnroutedHandler, error) {
}
// Only promote extesions using the Tus-Extension header which are implemented
extensions := "creation"
extensions := "creation,creation-with-upload"
if config.StoreComposer.UsesTerminater {
extensions += ",termination"
}
@ -191,6 +191,16 @@ func (handler *UnroutedHandler) Middleware(h http.Handler) http.Handler {
// PostFile creates a new file upload using the datastore after validating the
// length and parsing the metadata.
func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) {
// Check for presence of application/offset+octet-stream
containsChunk := false
if contentType := r.Header.Get("Content-Type"); contentType != "" {
if contentType != "application/offset+octet-stream" {
handler.sendError(w, r, ErrInvalidContentType)
return
}
containsChunk = true
}
// Only use the proper Upload-Concat header if the concatenation extension
// is even supported by the data store.
var concatHeader string
@ -210,6 +220,12 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
// Upload-Length header)
var size int64
if isFinal {
// A final upload must not contain a chunk within the creation request
if containsChunk {
handler.sendError(w, r, ErrModifyFinal)
return
}
size, err = handler.sizeOfUploads(partialUploads)
if err != nil {
handler.sendError(w, r, err)
@ -246,6 +262,13 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
return
}
// Add the Location header directly after creating the new resource to even
// include it in cases of failure when an error is returned
url := handler.absFileURL(r, id)
w.Header().Set("Location", url)
go handler.Metrics.incUploadsCreated()
if isFinal {
if err := handler.composer.Concater.ConcatUploads(id, partialUploads); err != nil {
handler.sendError(w, r, err)
@ -257,15 +280,26 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
info.ID = id
handler.CompleteUploads <- info
}
go handler.Metrics.incUploadsFinished()
}
url := handler.absFileURL(r, id)
w.Header().Set("Location", url)
w.WriteHeader(http.StatusCreated)
if containsChunk {
if handler.composer.UsesLocker {
locker := handler.composer.Locker
if err := locker.LockUpload(id); err != nil {
handler.sendError(w, r, err)
return
}
go handler.Metrics.incUploadsCreated()
defer locker.UnlockUpload(id)
}
if err := handler.writeChunk(id, info, w, r); err != nil {
handler.sendError(w, r, err)
return
}
}
w.WriteHeader(http.StatusCreated)
}
// HeadFile returns the length and offset for the HEAD request
@ -313,7 +347,7 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request)
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Upload-Length", strconv.FormatInt(info.Size, 10))
w.Header().Set("Upload-Offset", strconv.FormatInt(info.Offset, 10))
w.WriteHeader(http.StatusNoContent)
w.WriteHeader(http.StatusOK)
}
// PatchFile adds a chunk to an upload. Only allowed enough space is left.
@ -372,13 +406,23 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
return
}
if err := handler.writeChunk(id, info, w, r); err != nil {
handler.sendError(w, r, err)
return
}
w.WriteHeader(http.StatusNoContent)
}
// PatchFile adds a chunk to an upload. Only allowed enough space is left.
func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.ResponseWriter, r *http.Request) error {
// Get Content-Length if possible
length := r.ContentLength
offset := info.Offset
// Test if this upload fits into the file's size
if offset+length > info.Size {
handler.sendError(w, r, ErrSizeExceeded)
return
return ErrSizeExceeded
}
maxSize := info.Size - offset
@ -386,13 +430,18 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
maxSize = length
}
var bytesWritten int64
// Prevent a nil pointer derefernce when accessing the body which may not be
// available in the case of a malicious request.
if r.Body != nil {
// Limit the data read from the request's body to the allowed maxiumum
reader := io.LimitReader(r.Body, maxSize)
bytesWritten, err := handler.composer.Core.WriteChunk(id, offset, reader)
var err error
bytesWritten, err = handler.composer.Core.WriteChunk(id, offset, reader)
if err != nil {
handler.sendError(w, r, err)
return
return err
}
}
// Send new offset to client
@ -405,8 +454,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
// ... allow custom mechanism to finish and cleanup the upload
if handler.composer.UsesFinisher {
if err := handler.composer.Finisher.FinishUpload(id); err != nil {
handler.sendError(w, r, err)
return
return err
}
}
@ -419,7 +467,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
go handler.Metrics.incUploadsFinished()
}
w.WriteHeader(http.StatusNoContent)
return nil
}
// GetFile handles requests to download a file using a GET request. This is not

1
vendor/github.com/aws/aws-sdk-go generated vendored

@ -1 +0,0 @@
Subproject commit c76e8918e8f08490e3bb154178a84a0b2bdf8d6e

1
vendor/github.com/hashicorp/consul generated vendored

@ -1 +0,0 @@
Subproject commit f6fef66e1bf17be4f3c9855fbec6de802ca6bd7d