Merge remote-tracking branch 'origin/master' into micha
This commit is contained in:
Mischa ter Smitten
commit
f6d1f857c9
|
|
@ -1,35 +0,0 @@
|
|||
# Frey specific overrides, DO NOT EDIT THIS FILE
|
||||
---
|
||||
# apt role
|
||||
apt_manage_sources_list: true
|
||||
apt_src_enable: false
|
||||
apt_update_cache_valid_time: 86400
|
||||
apt_upgrade: false
|
||||
apt_dpkg_configure: true
|
||||
apt_install_state: present
|
||||
apt_clean: true
|
||||
apt_autoremove: false
|
||||
|
||||
# ansistrano roles
|
||||
ansistrano_shared_paths:
|
||||
- logs
|
||||
ansistrano_keep_releases: 10
|
||||
ansistrano_npm: no
|
||||
ansistrano_owner: www-data
|
||||
ansistrano_group: www-data
|
||||
ansistrano_allow_anonymous_stats: no
|
||||
|
||||
ansistrano_remove_rolled_back: no
|
||||
|
||||
# fqdn
|
||||
fqdn: "{{ lookup('env', 'FREY_DOMAIN') }}"
|
||||
hostname: "{{ fqdn.split('.')[0] }}"
|
||||
|
||||
# nodejs
|
||||
nodejs_yarn: false
|
||||
nodejs_install_npm_user: root
|
||||
nodejs_npm_global_packages:
|
||||
- name: yarn
|
||||
|
||||
# unattended-upgrades
|
||||
unattended_remove_unused_dependencies: true
|
||||
|
|
@ -3,7 +3,7 @@
|
|||
"version": "0.0.1",
|
||||
"description": "This folder is charge of launching master.tus.io via [Frey](https://github.com/kvz/frey). You could re-use bits of this, but you could also run tusd any other way you want. It's not a requirement for running a tusd server.",
|
||||
"dependencies": {
|
||||
"frey": "0.3.28"
|
||||
"frey": "0.3.31"
|
||||
},
|
||||
"devDependencies": {},
|
||||
"scripts": {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ abbrev@1:
|
|||
version "1.0.9"
|
||||
resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135"
|
||||
|
||||
ansi-escapes@^1.1.0:
|
||||
ansi-escapes@^1.0.0, ansi-escapes@^1.1.0:
|
||||
version "1.4.0"
|
||||
resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-1.4.0.tgz#d3a8a83b319aa67793662b13e761c7911422306e"
|
||||
|
||||
|
|
@ -124,12 +124,23 @@ chalk@1.1.3, chalk@^1.0.0, chalk@^1.1.1:
|
|||
strip-ansi "^3.0.0"
|
||||
supports-color "^2.0.0"
|
||||
|
||||
cli-cursor@^1.0.1:
|
||||
cli-cursor@^1.0.1, cli-cursor@^1.0.2:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-1.0.2.tgz#64da3f7d56a54412e59794bd62dc35295e8f2987"
|
||||
dependencies:
|
||||
restore-cursor "^1.0.1"
|
||||
|
||||
cli-spinners@0.1.2:
|
||||
version "0.1.2"
|
||||
resolved "https://registry.yarnpkg.com/cli-spinners/-/cli-spinners-0.1.2.tgz#bb764d88e185fb9e1e6a2a1f19772318f605e31c"
|
||||
|
||||
cli-truncate@0.2.1:
|
||||
version "0.2.1"
|
||||
resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-0.2.1.tgz#9f15cfbb0705005369216c626ac7d05ab90dd574"
|
||||
dependencies:
|
||||
slice-ansi "0.0.4"
|
||||
string-width "^1.0.1"
|
||||
|
||||
cli-width@^2.0.0:
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.1.0.tgz#b234ca209b29ef66fc518d9b98d5847b00edf00a"
|
||||
|
|
@ -162,7 +173,7 @@ concat-map@0.0.1:
|
|||
version "0.0.1"
|
||||
resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
|
||||
|
||||
concat-stream@:
|
||||
concat-stream@, concat-stream@^1.4.7:
|
||||
version "1.6.0"
|
||||
resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.0.tgz#0aac662fd52be78964d5532f694784e70110acf7"
|
||||
dependencies:
|
||||
|
|
@ -287,7 +298,7 @@ fakefile@0.0.8:
|
|||
dependencies:
|
||||
fs-extra "0.26.5"
|
||||
|
||||
figures@^1.3.5:
|
||||
figures@1.7.0, figures@^1.3.5:
|
||||
version "1.7.0"
|
||||
resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e"
|
||||
dependencies:
|
||||
|
|
@ -328,9 +339,9 @@ flat@2.0.0:
|
|||
dependencies:
|
||||
is-buffer "~1.1.2"
|
||||
|
||||
frey@0.3.28:
|
||||
version "0.3.28"
|
||||
resolved "https://registry.yarnpkg.com/frey/-/frey-0.3.28.tgz#d18c641d82faee834c18a7925388497e494d6888"
|
||||
frey@0.3.31:
|
||||
version "0.3.31"
|
||||
resolved "https://registry.yarnpkg.com/frey/-/frey-0.3.31.tgz#258aad2248af2c6c73e2854ba140ea189db7f624"
|
||||
dependencies:
|
||||
async "1.5.2"
|
||||
babel-polyfill "6.13.0"
|
||||
|
|
@ -351,12 +362,13 @@ frey@0.3.28:
|
|||
os-homedir "1.0.1"
|
||||
rc "1.1.6"
|
||||
replace "0.3.0"
|
||||
scrolex "0.0.26"
|
||||
semver "5.1.0"
|
||||
strip-indent "1.0.1"
|
||||
toml "2.3.0"
|
||||
tomlify-j0.4 "1.0.1"
|
||||
update-notifier "0.6.3"
|
||||
uuid "^3.0.0"
|
||||
uuid "3.0.0"
|
||||
yargs "4.3.2"
|
||||
|
||||
fs-extra@0.26.5:
|
||||
|
|
@ -625,6 +637,10 @@ is-stream@^1.0.0:
|
|||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
|
||||
|
||||
is-travis@1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/is-travis/-/is-travis-1.0.0.tgz#89d40ed56d9b8f8c36dfbe5811ba7e5e14944df9"
|
||||
|
||||
is-utf8@^0.2.0:
|
||||
version "0.2.1"
|
||||
resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72"
|
||||
|
|
@ -695,6 +711,10 @@ lodash.assign@^4.0.3, lodash.assign@^4.0.6:
|
|||
version "4.2.0"
|
||||
resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7"
|
||||
|
||||
lodash@4.17.4:
|
||||
version "4.17.4"
|
||||
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
|
||||
|
||||
lodash@4.6.1, lodash@^4.3.0:
|
||||
version "4.6.1"
|
||||
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.6.1.tgz#df00c1164ad236b183cfc3887a5e8d38cc63cbbc"
|
||||
|
|
@ -707,6 +727,19 @@ lodash@~2.4.1:
|
|||
version "2.4.2"
|
||||
resolved "https://registry.yarnpkg.com/lodash/-/lodash-2.4.2.tgz#fadd834b9683073da179b3eae6d9c0d15053f73e"
|
||||
|
||||
log-symbols@1.0.2:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18"
|
||||
dependencies:
|
||||
chalk "^1.0.0"
|
||||
|
||||
log-update@1.0.2:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/log-update/-/log-update-1.0.2.tgz#19929f64c4093d2d2e7075a1dad8af59c296b8d1"
|
||||
dependencies:
|
||||
ansi-escapes "^1.0.0"
|
||||
cli-cursor "^1.0.2"
|
||||
|
||||
lowercase-keys@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.0.tgz#4e3366b39e7f5457e35f1324bdf6f88d0bfc7306"
|
||||
|
|
@ -811,7 +844,11 @@ os-locale@^1.4.0:
|
|||
dependencies:
|
||||
lcid "^1.0.0"
|
||||
|
||||
os-tmpdir@^1.0.0:
|
||||
os-shim@^0.1.2:
|
||||
version "0.1.3"
|
||||
resolved "https://registry.yarnpkg.com/os-shim/-/os-shim-0.1.3.tgz#6b62c3791cf7909ea35ed46e17658bb417cb3917"
|
||||
|
||||
os-tmpdir@1.0.2, os-tmpdir@^1.0.0:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
|
||||
|
||||
|
|
@ -1006,6 +1043,25 @@ rx-lite@^3.1.2:
|
|||
version "3.1.2"
|
||||
resolved "https://registry.yarnpkg.com/rx-lite/-/rx-lite-3.1.2.tgz#19ce502ca572665f3b647b10939f97fd1615f102"
|
||||
|
||||
scrolex@0.0.26:
|
||||
version "0.0.26"
|
||||
resolved "https://registry.yarnpkg.com/scrolex/-/scrolex-0.0.26.tgz#6076fb13bf8f5953225567dc08700ab695accb1c"
|
||||
dependencies:
|
||||
chalk "1.1.3"
|
||||
cli-spinners "0.1.2"
|
||||
cli-truncate "0.2.1"
|
||||
depurar "0.3.0"
|
||||
figures "1.7.0"
|
||||
is-travis "1.0.0"
|
||||
lodash "4.17.4"
|
||||
log-symbols "1.0.2"
|
||||
log-update "1.0.2"
|
||||
os-tmpdir "1.0.2"
|
||||
slice-ansi "0.0.4"
|
||||
spawn-sync "1.0.15"
|
||||
strip-ansi "3.0.1"
|
||||
uuid "3.0.1"
|
||||
|
||||
semver-diff@^2.0.0:
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-2.1.0.tgz#4bbb8437c8d37e4b0cf1a68fd726ec6d645d6d36"
|
||||
|
|
@ -1020,10 +1076,21 @@ sigmund@~1.0.0:
|
|||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/sigmund/-/sigmund-1.0.1.tgz#3ff21f198cad2175f9f3b781853fd94d0d19b590"
|
||||
|
||||
slice-ansi@0.0.4:
|
||||
version "0.0.4"
|
||||
resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-0.0.4.tgz#edbf8903f66f7ce2f8eafd6ceed65e264c831b35"
|
||||
|
||||
slide@^1.1.5:
|
||||
version "1.1.6"
|
||||
resolved "https://registry.yarnpkg.com/slide/-/slide-1.1.6.tgz#56eb027d65b4d2dce6cb2e2d32c4d4afc9e1d707"
|
||||
|
||||
spawn-sync@1.0.15:
|
||||
version "1.0.15"
|
||||
resolved "https://registry.yarnpkg.com/spawn-sync/-/spawn-sync-1.0.15.tgz#b00799557eb7fb0c8376c29d44e8a1ea67e57476"
|
||||
dependencies:
|
||||
concat-stream "^1.4.7"
|
||||
os-shim "^0.1.2"
|
||||
|
||||
spdx-correct@~1.0.0:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-1.0.2.tgz#4b3073d933ff51f3912f03ac5519498a4150db40"
|
||||
|
|
@ -1054,7 +1121,7 @@ string_decoder@~0.10.x:
|
|||
version "0.10.31"
|
||||
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
|
||||
|
||||
strip-ansi@^3.0.0, strip-ansi@^3.0.1:
|
||||
strip-ansi@3.0.1, strip-ansi@^3.0.0, strip-ansi@^3.0.1:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf"
|
||||
dependencies:
|
||||
|
|
@ -1149,14 +1216,18 @@ util-deprecate@~1.0.1:
|
|||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
|
||||
|
||||
uuid@3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.0.tgz#6728fc0459c450d796a99c31837569bdf672d728"
|
||||
|
||||
uuid@3.0.1:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1"
|
||||
|
||||
uuid@^2.0.1:
|
||||
version "2.0.3"
|
||||
resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a"
|
||||
|
||||
uuid@^3.0.0:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1"
|
||||
|
||||
validate-npm-package-license@^3.0.1:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz#2804babe712ad3379459acfbe24746ab2c303fbc"
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
set -e
|
||||
|
||||
version=$(git rev-parse --abbrev-ref HEAD)
|
||||
version="$(git tag -l --points-at HEAD)"
|
||||
commit=$(git log --format="%H" -n 1)
|
||||
|
||||
function compile {
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ go:
|
|||
- 1.5
|
||||
- 1.6
|
||||
- 1.7
|
||||
- 1.8beta2
|
||||
- 1.8
|
||||
sudo: required
|
||||
cache:
|
||||
apt: true
|
||||
|
|
@ -33,11 +33,11 @@ deploy:
|
|||
skip_cleanup: true
|
||||
on:
|
||||
tags: true
|
||||
go: 1.7
|
||||
go: 1.8
|
||||
repo: tus/tusd
|
||||
- provider: script
|
||||
script: .scripts/deploy_current.sh
|
||||
on:
|
||||
branch: master
|
||||
go: 1.7
|
||||
go: 1.8
|
||||
repo: tus/tusd
|
||||
|
|
|
|||
|
|
@ -31,7 +31,9 @@ type Config struct {
|
|||
// NotifyTerminatedUploads indicates whether sending notifications about
|
||||
// terminated uploads using the TerminatedUploads channel should be enabled.
|
||||
NotifyTerminatedUploads bool
|
||||
NotifyUploadProgress bool
|
||||
// NotifyUploadProgress indicates whether sending notifications about
|
||||
// the upload progress using the UploadProgress channel should be enabled.
|
||||
NotifyUploadProgress bool
|
||||
// Logger is the logger to use internally, mostly for printing requests.
|
||||
Logger *log.Logger
|
||||
// Respect the X-Forwarded-Host, X-Forwarded-Proto and Forwarded headers
|
||||
|
|
|
|||
|
|
@ -69,3 +69,7 @@ This event will be triggered after an upload is fully finished, meaning that all
|
|||
### post-terminate
|
||||
|
||||
This event will be triggered after an upload has been terminated, meaning that the upload has been totally stopped and all associating chunks have been fully removed from the storage. Therefore, one is not able to retrieve the upload's content anymore and one may wish to notify further applications that this upload will never be resumed nor finished.
|
||||
|
||||
### post-receive
|
||||
|
||||
This event will be triggered for every running upload to indicate its current progress. It will occur for each open PATCH request, every second. The offset property will be set to the number of bytes which have been transfered to the server, at the time in total. Please be aware that this number may be higher than the number of bytes which have been stored by the data store!
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
package tusd_test
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
|
@ -289,4 +291,73 @@ func TestPatch(t *testing.T) {
|
|||
Code: http.StatusNoContent,
|
||||
}).Run(handler, t)
|
||||
})
|
||||
|
||||
SubTest(t, "NotifyUploadProgress", func(t *testing.T, store *MockFullDataStore) {
|
||||
gomock.InOrder(
|
||||
store.EXPECT().GetInfo("yes").Return(FileInfo{
|
||||
ID: "yes",
|
||||
Offset: 0,
|
||||
Size: 100,
|
||||
}, nil),
|
||||
store.EXPECT().WriteChunk("yes", int64(0), NewReaderMatcher("first second third")).Return(int64(18), nil),
|
||||
)
|
||||
|
||||
handler, _ := NewHandler(Config{
|
||||
DataStore: store,
|
||||
NotifyUploadProgress: true,
|
||||
})
|
||||
|
||||
c := make(chan FileInfo)
|
||||
handler.UploadProgress = c
|
||||
|
||||
reader, writer := io.Pipe()
|
||||
a := assert.New(t)
|
||||
|
||||
go func() {
|
||||
writer.Write([]byte("first "))
|
||||
|
||||
info := <-c
|
||||
a.Equal("yes", info.ID)
|
||||
a.Equal(int64(100), info.Size)
|
||||
a.Equal(int64(6), info.Offset)
|
||||
|
||||
writer.Write([]byte("second "))
|
||||
writer.Write([]byte("third"))
|
||||
|
||||
info = <-c
|
||||
a.Equal("yes", info.ID)
|
||||
a.Equal(int64(100), info.Size)
|
||||
a.Equal(int64(18), info.Offset)
|
||||
|
||||
writer.Close()
|
||||
|
||||
info = <-c
|
||||
a.Equal("yes", info.ID)
|
||||
a.Equal(int64(100), info.Size)
|
||||
a.Equal(int64(18), info.Offset)
|
||||
}()
|
||||
|
||||
(&httpTest{
|
||||
Method: "PATCH",
|
||||
URL: "yes",
|
||||
ReqHeader: map[string]string{
|
||||
"Tus-Resumable": "1.0.0",
|
||||
"Content-Type": "application/offset+octet-stream",
|
||||
"Upload-Offset": "0",
|
||||
},
|
||||
ReqBody: reader,
|
||||
Code: http.StatusNoContent,
|
||||
ResHeader: map[string]string{
|
||||
"Upload-Offset": "18",
|
||||
},
|
||||
}).Run(handler, t)
|
||||
|
||||
// Wait a short time after the request has been handled before closing the
|
||||
// channel because another goroutine may still write to the channel.
|
||||
<-time.After(10 * time.Millisecond)
|
||||
close(handler.UploadProgress)
|
||||
|
||||
_, more := <-c
|
||||
a.False(more)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
|
|
@ -84,7 +85,15 @@ type UnroutedHandler struct {
|
|||
// happen if the NotifyTerminatedUploads field is set to true in the Config
|
||||
// structure.
|
||||
TerminatedUploads chan FileInfo
|
||||
UploadProgress chan FileInfo
|
||||
// UploadProgress is used to send notifications about the progress of the
|
||||
// currently running uploads. For each open PATCH request, every second
|
||||
// a FileInfo instance will be send over this channel with the Offset field
|
||||
// being set to the number of bytes which have been transfered to the server.
|
||||
// Please be aware that this number may be higher than the number of bytes
|
||||
// which have been stored by the data store! Sending to this channel will only
|
||||
// happen if the NotifyUploadProgress field is set to true in the Config
|
||||
// structure.
|
||||
UploadProgress chan FileInfo
|
||||
// Metrics provides numbers of the usage for this handler.
|
||||
Metrics Metrics
|
||||
}
|
||||
|
|
@ -603,6 +612,14 @@ func (handler *UnroutedHandler) sendError(w http.ResponseWriter, r *http.Request
|
|||
err = ErrNotFound
|
||||
}
|
||||
|
||||
// Errors for read timeouts contain too much information which is not
|
||||
// necessary for us and makes grouping for the metrics harder. The error
|
||||
// message looks like: read tcp 127.0.0.1:1080->127.0.0.1:53673: i/o timeout
|
||||
// Therefore, we use a common error message for all of them.
|
||||
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
|
||||
err = errors.New("read tcp: i/o timeout")
|
||||
}
|
||||
|
||||
status := 500
|
||||
if statusErr, ok := err.(HTTPError); ok {
|
||||
status = statusErr.StatusCode()
|
||||
|
|
@ -654,6 +671,10 @@ func (w *progressWriter) Write(b []byte) (int, error) {
|
|||
return len(b), nil
|
||||
}
|
||||
|
||||
// sendProgressMessage will send a notification over the UploadProgress channel
|
||||
// every second, indicating how much data has been transfered to the server.
|
||||
// It will stop sending these instances once the returned channel has been
|
||||
// closed. The returned reader should be used to read the request body.
|
||||
func (handler *UnroutedHandler) sendProgressMessages(info FileInfo, reader io.Reader) (io.Reader, chan<- struct{}) {
|
||||
progress := &progressWriter{
|
||||
Offset: info.Offset,
|
||||
|
|
|
|||
Loading…
Reference in New Issue