Merge branch 'master' of github.com:tus/tusd
This commit is contained in:
commit
bea4d10ae6
|
@ -4,9 +4,6 @@
|
|||
[submodule "vendor/github.com/nightlyone/lockfile"]
|
||||
path = vendor/github.com/nightlyone/lockfile
|
||||
url = https://github.com/nightlyone/lockfile
|
||||
[submodule "vendor/github.com/aws/aws-sdk-go"]
|
||||
path = vendor/github.com/aws/aws-sdk-go
|
||||
url = https://github.com/aws/aws-sdk-go
|
||||
[submodule "vendor/github.com/go-ini/ini"]
|
||||
path = vendor/github.com/go-ini/ini
|
||||
url = https://github.com/go-ini/ini
|
||||
|
@ -25,9 +22,6 @@
|
|||
[submodule "vendor/github.com/pmezard/go-difflib"]
|
||||
path = vendor/github.com/pmezard/go-difflib
|
||||
url = https://github.com/pmezard/go-difflib
|
||||
[submodule "vendor/github.com/hashicorp/consul"]
|
||||
path = vendor/github.com/hashicorp/consul
|
||||
url = https://github.com/hashicorp/consul
|
||||
[submodule "vendor/github.com/hashicorp/go-cleanhttp"]
|
||||
path = vendor/github.com/hashicorp/go-cleanhttp
|
||||
url = https://github.com/hashicorp/go-cleanhttp
|
||||
|
|
|
@ -4,6 +4,7 @@ go:
|
|||
- 1.4
|
||||
- 1.5
|
||||
- 1.6
|
||||
- 1.7
|
||||
- tip
|
||||
sudo: required
|
||||
cache:
|
||||
|
@ -22,6 +23,7 @@ matrix:
|
|||
install:
|
||||
- export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \))
|
||||
- rsync -r ./vendor/ $GOPATH/src
|
||||
- go get $PACKAGES
|
||||
script:
|
||||
- go test $PACKAGES
|
||||
before_deploy:
|
||||
|
|
|
@ -25,7 +25,7 @@ func CreateComposer() {
|
|||
dir := Flags.UploadDir
|
||||
|
||||
stdout.Printf("Using '%s' as directory storage.\n", dir)
|
||||
if err := os.MkdirAll(dir, os.FileMode(0775)); err != nil {
|
||||
if err := os.MkdirAll(dir, os.FileMode(0774)); err != nil {
|
||||
stderr.Fatalf("Unable to ensure directory exists: %s", err)
|
||||
}
|
||||
|
||||
|
|
|
@ -17,6 +17,7 @@ var Flags struct {
|
|||
HooksDir string
|
||||
ShowVersion bool
|
||||
ExposeMetrics bool
|
||||
MetricsPath string
|
||||
BehindProxy bool
|
||||
|
||||
HooksInstalled bool
|
||||
|
@ -34,6 +35,7 @@ func ParseFlags() {
|
|||
flag.StringVar(&Flags.HooksDir, "hooks-dir", "", "Directory to search for available hooks scripts")
|
||||
flag.BoolVar(&Flags.ShowVersion, "version", false, "Print tusd version information")
|
||||
flag.BoolVar(&Flags.ExposeMetrics, "expose-metrics", true, "Expose metrics about tusd usage")
|
||||
flag.StringVar(&Flags.MetricsPath, "metrics-path", "/metrics", "Path under which the metrics endpoint will be accessible")
|
||||
flag.BoolVar(&Flags.BehindProxy, "behind-proxy", false, "Respect X-Forwarded-* and similar headers which may be set by proxies")
|
||||
|
||||
flag.Parse()
|
||||
|
|
|
@ -12,19 +12,23 @@ func PrepareGreeting() {
|
|||
`Welcome to tusd
|
||||
===============
|
||||
|
||||
Congratulations for setting up tusd! You are now part of the chosen elite and
|
||||
able to experience the feeling of resumable uploads! We hope you are as excited
|
||||
as we are (a lot)!
|
||||
Congratulations on setting up tusd! Thanks for joining our cause, you have taken
|
||||
the first step towards making the future of resumable uploading a reality! We
|
||||
hope you are as excited about this as we are!
|
||||
|
||||
However, there is something you should be aware of: While you got tusd
|
||||
running (you did an awesome job!), this is the root directory of the server
|
||||
and tus requests are only accepted at the %s route.
|
||||
While you did an awesome job on getting tusd running, this is just the welcome
|
||||
message, so let's talk about the places that really matter:
|
||||
|
||||
So don't waste time, head over there and experience the future!
|
||||
- %s - send your tus uploads to this endpoint
|
||||
- %s - gather statistics to keep tusd running smoothly
|
||||
- https://github.com/tus/tusd/issues - report your bugs here
|
||||
|
||||
So quit lollygagging, send over your files and experience the future!
|
||||
|
||||
Version = %s
|
||||
GitCommit = %s
|
||||
BuildDate = %s`, Flags.Basepath, VersionName, GitCommit, BuildDate)
|
||||
BuildDate = %s
|
||||
`, Flags.Basepath, Flags.MetricsPath, VersionName, GitCommit, BuildDate)
|
||||
}
|
||||
|
||||
func DisplayGreeting(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
|
@ -18,5 +18,6 @@ func SetupMetrics(handler *tusd.Handler) {
|
|||
prometheus.MustRegister(MetricsOpenConnections)
|
||||
prometheus.MustRegister(prometheuscollector.New(handler.Metrics))
|
||||
|
||||
http.Handle("/metrics", prometheus.Handler())
|
||||
stdout.Printf("Using %s as the metrics path.\n", Flags.MetricsPath)
|
||||
http.Handle(Flags.MetricsPath, prometheus.Handler())
|
||||
}
|
||||
|
|
|
@ -27,7 +27,6 @@ func Serve() {
|
|||
|
||||
stdout.Printf("Using %s as address to listen.\n", address)
|
||||
stdout.Printf("Using %s as the base path.\n", basepath)
|
||||
stdout.Printf(Composer.Capabilities())
|
||||
|
||||
SetupPostHooks(handler)
|
||||
|
||||
|
@ -35,6 +34,8 @@ func Serve() {
|
|||
SetupMetrics(handler)
|
||||
}
|
||||
|
||||
stdout.Printf(Composer.Capabilities())
|
||||
|
||||
// Do not display the greeting if the tusd handler will be mounted at the root
|
||||
// path. Else this would cause a "multiple registrations for /" panic.
|
||||
if basepath != "/" {
|
||||
|
|
|
@ -46,7 +46,7 @@ func TestConcatPartial(t *testing.T) {
|
|||
Method: "OPTIONS",
|
||||
URL: "",
|
||||
ResHeader: map[string]string{
|
||||
"Tus-Extension": "creation,concatenation",
|
||||
"Tus-Extension": "creation,creation-with-upload,concatenation",
|
||||
},
|
||||
Code: http.StatusOK,
|
||||
}).Run(handler, t)
|
||||
|
@ -69,7 +69,7 @@ func TestConcatPartial(t *testing.T) {
|
|||
ReqHeader: map[string]string{
|
||||
"Tus-Resumable": "1.0.0",
|
||||
},
|
||||
Code: http.StatusNoContent,
|
||||
Code: http.StatusOK,
|
||||
ResHeader: map[string]string{
|
||||
"Upload-Concat": "partial",
|
||||
},
|
||||
|
@ -165,7 +165,7 @@ func TestConcatFinal(t *testing.T) {
|
|||
ReqHeader: map[string]string{
|
||||
"Tus-Resumable": "1.0.0",
|
||||
},
|
||||
Code: http.StatusNoContent,
|
||||
Code: http.StatusOK,
|
||||
ResHeader: map[string]string{
|
||||
"Upload-Concat": "final; http://tus.io/files/a http://tus.io/files/b",
|
||||
"Upload-Length": "10",
|
||||
|
|
|
@ -28,7 +28,7 @@ import (
|
|||
"github.com/nightlyone/lockfile"
|
||||
)
|
||||
|
||||
var defaultFilePerm = os.FileMode(0775)
|
||||
var defaultFilePerm = os.FileMode(0664)
|
||||
|
||||
// See the tusd.DataStore interface for documentation about the different
|
||||
// methods.
|
||||
|
|
|
@ -40,7 +40,7 @@ func TestHead(t *testing.T) {
|
|||
ReqHeader: map[string]string{
|
||||
"Tus-Resumable": "1.0.0",
|
||||
},
|
||||
Code: http.StatusNoContent,
|
||||
Code: http.StatusOK,
|
||||
ResHeader: map[string]string{
|
||||
"Upload-Offset": "11",
|
||||
"Upload-Length": "44",
|
||||
|
|
|
@ -20,7 +20,7 @@ func TestOptions(t *testing.T) {
|
|||
Method: "OPTIONS",
|
||||
Code: http.StatusOK,
|
||||
ResHeader: map[string]string{
|
||||
"Tus-Extension": "creation",
|
||||
"Tus-Extension": "creation,creation-with-upload",
|
||||
"Tus-Version": "1.0.0",
|
||||
"Tus-Resumable": "1.0.0",
|
||||
"Tus-Max-Size": "400",
|
||||
|
|
114
post_test.go
114
post_test.go
|
@ -1,44 +1,57 @@
|
|||
package tusd_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
. "github.com/tus/tusd"
|
||||
)
|
||||
|
||||
type postStore struct {
|
||||
t *testing.T
|
||||
t *assert.Assertions
|
||||
zeroStore
|
||||
}
|
||||
|
||||
func (s postStore) NewUpload(info FileInfo) (string, error) {
|
||||
if info.Size != 300 {
|
||||
s.t.Errorf("Expected size to be 300 (got %v)", info.Size)
|
||||
}
|
||||
s.t.Equal(int64(300), info.Size)
|
||||
|
||||
metaData := info.MetaData
|
||||
if len(metaData) != 2 {
|
||||
s.t.Errorf("Expected two elements in metadata")
|
||||
}
|
||||
|
||||
if v := metaData["foo"]; v != "hello" {
|
||||
s.t.Errorf("Expected foo element to be 'hello' but got %s", v)
|
||||
}
|
||||
|
||||
if v := metaData["bar"]; v != "world" {
|
||||
s.t.Errorf("Expected bar element to be 'world' but got %s", v)
|
||||
}
|
||||
s.t.Equal(2, len(metaData))
|
||||
s.t.Equal("hello", metaData["foo"])
|
||||
s.t.Equal("world", metaData["bar"])
|
||||
|
||||
return "foo", nil
|
||||
}
|
||||
|
||||
func (s postStore) WriteChunk(id string, offset int64, src io.Reader) (int64, error) {
|
||||
s.t.Equal(int64(0), offset)
|
||||
|
||||
data, err := ioutil.ReadAll(src)
|
||||
s.t.Nil(err)
|
||||
s.t.Equal("hello", string(data))
|
||||
|
||||
return 5, nil
|
||||
}
|
||||
|
||||
func (s postStore) ConcatUploads(id string, uploads []string) error {
|
||||
s.t.True(false, "concatenation should not be attempted")
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestPost(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
|
||||
handler, _ := NewHandler(Config{
|
||||
MaxSize: 400,
|
||||
BasePath: "files",
|
||||
DataStore: postStore{
|
||||
t: t,
|
||||
t: a,
|
||||
},
|
||||
})
|
||||
|
||||
|
@ -87,7 +100,7 @@ func TestPost(t *testing.T) {
|
|||
MaxSize: 400,
|
||||
BasePath: "files",
|
||||
DataStore: postStore{
|
||||
t: t,
|
||||
t: a,
|
||||
},
|
||||
RespectForwardedHeaders: true,
|
||||
})
|
||||
|
@ -141,3 +154,70 @@ func TestPost(t *testing.T) {
|
|||
},
|
||||
}).Run(handler, t)
|
||||
}
|
||||
|
||||
func TestPostWithUpload(t *testing.T) {
|
||||
a := assert.New(t)
|
||||
|
||||
handler, _ := NewHandler(Config{
|
||||
MaxSize: 400,
|
||||
BasePath: "files",
|
||||
DataStore: postStore{
|
||||
t: a,
|
||||
},
|
||||
})
|
||||
|
||||
(&httpTest{
|
||||
Name: "Successful request",
|
||||
Method: "POST",
|
||||
ReqHeader: map[string]string{
|
||||
"Tus-Resumable": "1.0.0",
|
||||
"Upload-Length": "300",
|
||||
"Content-Type": "application/offset+octet-stream",
|
||||
"Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=",
|
||||
},
|
||||
ReqBody: strings.NewReader("hello"),
|
||||
Code: http.StatusCreated,
|
||||
ResHeader: map[string]string{
|
||||
"Location": "http://tus.io/files/foo",
|
||||
"Upload-Offset": "5",
|
||||
},
|
||||
}).Run(handler, t)
|
||||
|
||||
(&httpTest{
|
||||
Name: "Exceeding upload size",
|
||||
Method: "POST",
|
||||
ReqHeader: map[string]string{
|
||||
"Tus-Resumable": "1.0.0",
|
||||
"Upload-Length": "300",
|
||||
"Content-Type": "application/offset+octet-stream",
|
||||
"Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=",
|
||||
},
|
||||
ReqBody: bytes.NewReader(make([]byte, 400)),
|
||||
Code: http.StatusRequestEntityTooLarge,
|
||||
}).Run(handler, t)
|
||||
|
||||
(&httpTest{
|
||||
Name: "Incorrect content type",
|
||||
Method: "POST",
|
||||
ReqHeader: map[string]string{
|
||||
"Tus-Resumable": "1.0.0",
|
||||
"Content-Type": "application/false",
|
||||
},
|
||||
ReqBody: strings.NewReader("hello"),
|
||||
Code: http.StatusBadRequest,
|
||||
}).Run(handler, t)
|
||||
|
||||
(&httpTest{
|
||||
Name: "Upload and final concatenation",
|
||||
Method: "POST",
|
||||
ReqHeader: map[string]string{
|
||||
"Tus-Resumable": "1.0.0",
|
||||
"Upload-Length": "300",
|
||||
"Content-Type": "application/offset+octet-stream",
|
||||
"Upload-Metadata": "foo aGVsbG8=, bar d29ybGQ=",
|
||||
"Upload-Concat": "final; http://tus.io/files/a http://tus.io/files/b",
|
||||
},
|
||||
ReqBody: strings.NewReader("hello"),
|
||||
Code: http.StatusForbidden,
|
||||
}).Run(handler, t)
|
||||
}
|
||||
|
|
|
@ -907,6 +907,16 @@ func (_mr *_MockS3APIRecorder) ListObjectsV2(arg0 interface{}) *gomock.Call {
|
|||
return _mr.mock.ctrl.RecordCall(_mr.mock, "ListObjectsV2", arg0)
|
||||
}
|
||||
|
||||
func (_m *MockS3API) ListObjectsV2Pages(_param0 *s3.ListObjectsV2Input, _param1 func(*s3.ListObjectsV2Output, bool) bool) error {
|
||||
ret := _m.ctrl.Call(_m, "ListObjectsV2Pages", _param0, _param1)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
func (_mr *_MockS3APIRecorder) ListObjectsV2Pages(arg0, arg1 interface{}) *gomock.Call {
|
||||
return _mr.mock.ctrl.RecordCall(_mr.mock, "ListObjectsV2Pages", arg0, arg1)
|
||||
}
|
||||
|
||||
func (_m *MockS3API) ListObjectsV2Request(_param0 *s3.ListObjectsV2Input) (*request.Request, *s3.ListObjectsV2Output) {
|
||||
ret := _m.ctrl.Call(_m, "ListObjectsV2Request", _param0)
|
||||
ret0, _ := ret[0].(*request.Request)
|
||||
|
|
|
@ -44,7 +44,7 @@ func TestTerminate(t *testing.T) {
|
|||
Method: "OPTIONS",
|
||||
URL: "",
|
||||
ResHeader: map[string]string{
|
||||
"Tus-Extension": "creation,termination",
|
||||
"Tus-Extension": "creation,creation-with-upload,termination",
|
||||
},
|
||||
Code: http.StatusOK,
|
||||
}).Run(handler, t)
|
||||
|
|
|
@ -89,7 +89,7 @@ func NewUnroutedHandler(config Config) (*UnroutedHandler, error) {
|
|||
}
|
||||
|
||||
// Only promote extesions using the Tus-Extension header which are implemented
|
||||
extensions := "creation"
|
||||
extensions := "creation,creation-with-upload"
|
||||
if config.StoreComposer.UsesTerminater {
|
||||
extensions += ",termination"
|
||||
}
|
||||
|
@ -191,6 +191,16 @@ func (handler *UnroutedHandler) Middleware(h http.Handler) http.Handler {
|
|||
// PostFile creates a new file upload using the datastore after validating the
|
||||
// length and parsing the metadata.
|
||||
func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request) {
|
||||
// Check for presence of application/offset+octet-stream
|
||||
containsChunk := false
|
||||
if contentType := r.Header.Get("Content-Type"); contentType != "" {
|
||||
if contentType != "application/offset+octet-stream" {
|
||||
handler.sendError(w, r, ErrInvalidContentType)
|
||||
return
|
||||
}
|
||||
containsChunk = true
|
||||
}
|
||||
|
||||
// Only use the proper Upload-Concat header if the concatenation extension
|
||||
// is even supported by the data store.
|
||||
var concatHeader string
|
||||
|
@ -210,6 +220,12 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
|
|||
// Upload-Length header)
|
||||
var size int64
|
||||
if isFinal {
|
||||
// A final upload must not contain a chunk within the creation request
|
||||
if containsChunk {
|
||||
handler.sendError(w, r, ErrModifyFinal)
|
||||
return
|
||||
}
|
||||
|
||||
size, err = handler.sizeOfUploads(partialUploads)
|
||||
if err != nil {
|
||||
handler.sendError(w, r, err)
|
||||
|
@ -246,6 +262,13 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
|
|||
return
|
||||
}
|
||||
|
||||
// Add the Location header directly after creating the new resource to even
|
||||
// include it in cases of failure when an error is returned
|
||||
url := handler.absFileURL(r, id)
|
||||
w.Header().Set("Location", url)
|
||||
|
||||
go handler.Metrics.incUploadsCreated()
|
||||
|
||||
if isFinal {
|
||||
if err := handler.composer.Concater.ConcatUploads(id, partialUploads); err != nil {
|
||||
handler.sendError(w, r, err)
|
||||
|
@ -257,15 +280,26 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
|
|||
info.ID = id
|
||||
handler.CompleteUploads <- info
|
||||
}
|
||||
|
||||
go handler.Metrics.incUploadsFinished()
|
||||
}
|
||||
|
||||
url := handler.absFileURL(r, id)
|
||||
w.Header().Set("Location", url)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
if containsChunk {
|
||||
if handler.composer.UsesLocker {
|
||||
locker := handler.composer.Locker
|
||||
if err := locker.LockUpload(id); err != nil {
|
||||
handler.sendError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
go handler.Metrics.incUploadsCreated()
|
||||
defer locker.UnlockUpload(id)
|
||||
}
|
||||
|
||||
if err := handler.writeChunk(id, info, w, r); err != nil {
|
||||
handler.sendError(w, r, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}
|
||||
|
||||
// HeadFile returns the length and offset for the HEAD request
|
||||
|
@ -313,7 +347,7 @@ func (handler *UnroutedHandler) HeadFile(w http.ResponseWriter, r *http.Request)
|
|||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Upload-Length", strconv.FormatInt(info.Size, 10))
|
||||
w.Header().Set("Upload-Offset", strconv.FormatInt(info.Offset, 10))
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
// PatchFile adds a chunk to an upload. Only allowed enough space is left.
|
||||
|
@ -372,13 +406,23 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
|
|||
return
|
||||
}
|
||||
|
||||
if err := handler.writeChunk(id, info, w, r); err != nil {
|
||||
handler.sendError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// PatchFile adds a chunk to an upload. Only allowed enough space is left.
|
||||
func (handler *UnroutedHandler) writeChunk(id string, info FileInfo, w http.ResponseWriter, r *http.Request) error {
|
||||
// Get Content-Length if possible
|
||||
length := r.ContentLength
|
||||
offset := info.Offset
|
||||
|
||||
// Test if this upload fits into the file's size
|
||||
if offset+length > info.Size {
|
||||
handler.sendError(w, r, ErrSizeExceeded)
|
||||
return
|
||||
return ErrSizeExceeded
|
||||
}
|
||||
|
||||
maxSize := info.Size - offset
|
||||
|
@ -386,13 +430,18 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
|
|||
maxSize = length
|
||||
}
|
||||
|
||||
var bytesWritten int64
|
||||
// Prevent a nil pointer derefernce when accessing the body which may not be
|
||||
// available in the case of a malicious request.
|
||||
if r.Body != nil {
|
||||
// Limit the data read from the request's body to the allowed maxiumum
|
||||
reader := io.LimitReader(r.Body, maxSize)
|
||||
|
||||
bytesWritten, err := handler.composer.Core.WriteChunk(id, offset, reader)
|
||||
var err error
|
||||
bytesWritten, err = handler.composer.Core.WriteChunk(id, offset, reader)
|
||||
if err != nil {
|
||||
handler.sendError(w, r, err)
|
||||
return
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Send new offset to client
|
||||
|
@ -405,8 +454,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
|
|||
// ... allow custom mechanism to finish and cleanup the upload
|
||||
if handler.composer.UsesFinisher {
|
||||
if err := handler.composer.Finisher.FinishUpload(id); err != nil {
|
||||
handler.sendError(w, r, err)
|
||||
return
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -419,7 +467,7 @@ func (handler *UnroutedHandler) PatchFile(w http.ResponseWriter, r *http.Request
|
|||
go handler.Metrics.incUploadsFinished()
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFile handles requests to download a file using a GET request. This is not
|
||||
|
|
|
@ -1 +0,0 @@
|
|||
Subproject commit c76e8918e8f08490e3bb154178a84a0b2bdf8d6e
|
|
@ -1 +0,0 @@
|
|||
Subproject commit f6fef66e1bf17be4f3c9855fbec6de802ca6bd7d
|
Loading…
Reference in New Issue