Merge branch 'master' into metrics_race_condition

This commit is contained in:
oliverpool 2017-02-22 12:21:42 +01:00
commit 338017c9f4
12 changed files with 213 additions and 61 deletions

3
.gitignore vendored
View File

@ -1,8 +1,7 @@
tusd/data
cover.out
data/
.infra/env.sh
.infra/env.*.sh
.infra/env.*
.infra/ssh/tusd.pem
.infra/Frey-residu*
.infra/ssh/frey-tusd.pem

View File

@ -111,7 +111,22 @@ install {
name = "Install tusd"
roles {
role = "{{{init.paths.roles_dir}}}/apt/1.4.0"
apt_install = ["apg", "build-essential", "curl", "git-core", "htop", "iotop", "libpcre3", "logtail", "mlocate", "mtr", "psmisc", "telnet", "vim", "wget"]
apt_install = [
"apg",
"build-essential",
"curl",
"git-core",
"htop",
"iotop",
"libpcre3",
"logtail",
"mlocate",
"mtr",
"psmisc",
"telnet",
"vim",
"wget"
]
}
roles {
role = "{{{init.paths.roles_dir}}}/unattended-upgrades/1.3.1"
@ -167,7 +182,6 @@ setup {
}
roles {
role = "{{{init.paths.roles_dir}}}/fqdn/1.2.1"
fqdn = "{{lookup('env', 'FREY_DOMAIN')}}"
}
tasks {
file = "path=/mnt/tusd-data state=directory owner=www-data group=ubuntu mode=ug+rwX,o= recurse=yes"

View File

@ -13,6 +13,17 @@ server {
# of common primes
ssl_dhparam /etc/nginx/dhparams.pem;
# Restrict supported ciphers to prevent certain browsers from refusing to
# connect because we are offering blacklisted ciphers. This configuration has
# been generated by Mozilla's SSL Configuration Generator on the
# intermediate profile and can be accessed at:
# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=nginx-1.10.1&openssl=1.0.1e&hsts=no&profile=intermediate
# More information about blacklisted ciphers can be found at:
# http://security.stackexchange.com/questions/126775/understanding-blacklisted-ciphers-for-http2
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS';
ssl_prefer_server_ciphers on;
# Enable OCSP stapling which allows clients to verify that our certificate
# is not revoked without contacting the Certificate Authority by appending a
# CA-signed promise, that it's still valid, to the TLS handshake response.

View File

@ -1,31 +0,0 @@
# Frey specific overrides, DO NOT EDIT THIS FILE
---
# apt role
apt_manage_sources_list: true
apt_src_enable: false
apt_update_cache_valid_time: 86400
apt_upgrade: false
apt_dpkg_configure: true
apt_install_state: present
apt_clean: true
apt_autoremove: true
# ansistrano roles
ansistrano_shared_paths:
- logs
ansistrano_keep_releases: 10
ansistrano_npm: no
ansistrano_owner: www-data
ansistrano_group: www-data
ansistrano_allow_anonymous_stats: no
ansistrano_remove_rolled_back: no
# fqdn
fqdn: "{{ lookup('env', 'FREY_DOMAIN') }}"
hostname: "{{ fqdn.split('.')[0] }}"
# nodejs
nodejs_yarn: false
nodejs_npm_global_packages:
- name: yarn

View File

@ -3,7 +3,7 @@
"version": "0.0.1",
"description": "This folder is charge of launching master.tus.io via [Frey](https://github.com/kvz/frey). You could re-use bits of this, but you could also run tusd any other way you want. It's not a requirement for running a tusd server.",
"dependencies": {
"frey": "0.3.28"
"frey": "0.3.31"
},
"devDependencies": {},
"scripts": {

View File

@ -6,7 +6,7 @@ abbrev@1:
version "1.0.9"
resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135"
ansi-escapes@^1.1.0:
ansi-escapes@^1.0.0, ansi-escapes@^1.1.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-1.4.0.tgz#d3a8a83b319aa67793662b13e761c7911422306e"
@ -124,12 +124,23 @@ chalk@1.1.3, chalk@^1.0.0, chalk@^1.1.1:
strip-ansi "^3.0.0"
supports-color "^2.0.0"
cli-cursor@^1.0.1:
cli-cursor@^1.0.1, cli-cursor@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-1.0.2.tgz#64da3f7d56a54412e59794bd62dc35295e8f2987"
dependencies:
restore-cursor "^1.0.1"
cli-spinners@0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/cli-spinners/-/cli-spinners-0.1.2.tgz#bb764d88e185fb9e1e6a2a1f19772318f605e31c"
cli-truncate@0.2.1:
version "0.2.1"
resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-0.2.1.tgz#9f15cfbb0705005369216c626ac7d05ab90dd574"
dependencies:
slice-ansi "0.0.4"
string-width "^1.0.1"
cli-width@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.1.0.tgz#b234ca209b29ef66fc518d9b98d5847b00edf00a"
@ -162,7 +173,7 @@ concat-map@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
concat-stream@:
concat-stream@, concat-stream@^1.4.7:
version "1.6.0"
resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.0.tgz#0aac662fd52be78964d5532f694784e70110acf7"
dependencies:
@ -287,7 +298,7 @@ fakefile@0.0.8:
dependencies:
fs-extra "0.26.5"
figures@^1.3.5:
figures@1.7.0, figures@^1.3.5:
version "1.7.0"
resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e"
dependencies:
@ -328,9 +339,9 @@ flat@2.0.0:
dependencies:
is-buffer "~1.1.2"
frey@0.3.28:
version "0.3.28"
resolved "https://registry.yarnpkg.com/frey/-/frey-0.3.28.tgz#d18c641d82faee834c18a7925388497e494d6888"
frey@0.3.31:
version "0.3.31"
resolved "https://registry.yarnpkg.com/frey/-/frey-0.3.31.tgz#258aad2248af2c6c73e2854ba140ea189db7f624"
dependencies:
async "1.5.2"
babel-polyfill "6.13.0"
@ -351,12 +362,13 @@ frey@0.3.28:
os-homedir "1.0.1"
rc "1.1.6"
replace "0.3.0"
scrolex "0.0.26"
semver "5.1.0"
strip-indent "1.0.1"
toml "2.3.0"
tomlify-j0.4 "1.0.1"
update-notifier "0.6.3"
uuid "^3.0.0"
uuid "3.0.0"
yargs "4.3.2"
fs-extra@0.26.5:
@ -625,6 +637,10 @@ is-stream@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
is-travis@1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/is-travis/-/is-travis-1.0.0.tgz#89d40ed56d9b8f8c36dfbe5811ba7e5e14944df9"
is-utf8@^0.2.0:
version "0.2.1"
resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72"
@ -695,6 +711,10 @@ lodash.assign@^4.0.3, lodash.assign@^4.0.6:
version "4.2.0"
resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7"
lodash@4.17.4:
version "4.17.4"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
lodash@4.6.1, lodash@^4.3.0:
version "4.6.1"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.6.1.tgz#df00c1164ad236b183cfc3887a5e8d38cc63cbbc"
@ -707,6 +727,19 @@ lodash@~2.4.1:
version "2.4.2"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-2.4.2.tgz#fadd834b9683073da179b3eae6d9c0d15053f73e"
log-symbols@1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18"
dependencies:
chalk "^1.0.0"
log-update@1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/log-update/-/log-update-1.0.2.tgz#19929f64c4093d2d2e7075a1dad8af59c296b8d1"
dependencies:
ansi-escapes "^1.0.0"
cli-cursor "^1.0.2"
lowercase-keys@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.0.tgz#4e3366b39e7f5457e35f1324bdf6f88d0bfc7306"
@ -811,7 +844,11 @@ os-locale@^1.4.0:
dependencies:
lcid "^1.0.0"
os-tmpdir@^1.0.0:
os-shim@^0.1.2:
version "0.1.3"
resolved "https://registry.yarnpkg.com/os-shim/-/os-shim-0.1.3.tgz#6b62c3791cf7909ea35ed46e17658bb417cb3917"
os-tmpdir@1.0.2, os-tmpdir@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
@ -1006,6 +1043,25 @@ rx-lite@^3.1.2:
version "3.1.2"
resolved "https://registry.yarnpkg.com/rx-lite/-/rx-lite-3.1.2.tgz#19ce502ca572665f3b647b10939f97fd1615f102"
scrolex@0.0.26:
version "0.0.26"
resolved "https://registry.yarnpkg.com/scrolex/-/scrolex-0.0.26.tgz#6076fb13bf8f5953225567dc08700ab695accb1c"
dependencies:
chalk "1.1.3"
cli-spinners "0.1.2"
cli-truncate "0.2.1"
depurar "0.3.0"
figures "1.7.0"
is-travis "1.0.0"
lodash "4.17.4"
log-symbols "1.0.2"
log-update "1.0.2"
os-tmpdir "1.0.2"
slice-ansi "0.0.4"
spawn-sync "1.0.15"
strip-ansi "3.0.1"
uuid "3.0.1"
semver-diff@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-2.1.0.tgz#4bbb8437c8d37e4b0cf1a68fd726ec6d645d6d36"
@ -1020,10 +1076,21 @@ sigmund@~1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/sigmund/-/sigmund-1.0.1.tgz#3ff21f198cad2175f9f3b781853fd94d0d19b590"
slice-ansi@0.0.4:
version "0.0.4"
resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-0.0.4.tgz#edbf8903f66f7ce2f8eafd6ceed65e264c831b35"
slide@^1.1.5:
version "1.1.6"
resolved "https://registry.yarnpkg.com/slide/-/slide-1.1.6.tgz#56eb027d65b4d2dce6cb2e2d32c4d4afc9e1d707"
spawn-sync@1.0.15:
version "1.0.15"
resolved "https://registry.yarnpkg.com/spawn-sync/-/spawn-sync-1.0.15.tgz#b00799557eb7fb0c8376c29d44e8a1ea67e57476"
dependencies:
concat-stream "^1.4.7"
os-shim "^0.1.2"
spdx-correct@~1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-1.0.2.tgz#4b3073d933ff51f3912f03ac5519498a4150db40"
@ -1054,7 +1121,7 @@ string_decoder@~0.10.x:
version "0.10.31"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
strip-ansi@^3.0.0, strip-ansi@^3.0.1:
strip-ansi@3.0.1, strip-ansi@^3.0.0, strip-ansi@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf"
dependencies:
@ -1149,14 +1216,18 @@ util-deprecate@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
uuid@3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.0.tgz#6728fc0459c450d796a99c31837569bdf672d728"
uuid@3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1"
uuid@^2.0.1:
version "2.0.3"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a"
uuid@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1"
validate-npm-package-license@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz#2804babe712ad3379459acfbe24746ab2c303fbc"

View File

@ -3,7 +3,7 @@ go:
- 1.5
- 1.6
- 1.7
- 1.8beta2
- 1.8
sudo: required
cache:
apt: true
@ -33,11 +33,11 @@ deploy:
skip_cleanup: true
on:
tags: true
go: 1.7
go: 1.8
repo: tus/tusd
- provider: script
script: .scripts/deploy_current.sh
on:
branch: master
go: 1.7
go: 1.8
repo: tus/tusd

View File

@ -31,6 +31,8 @@ type Config struct {
// NotifyTerminatedUploads indicates whether sending notifications about
// terminated uploads using the TerminatedUploads channel should be enabled.
NotifyTerminatedUploads bool
// NotifyUploadProgress indicates whether sending notifications about
// the upload progress using the UploadProgress channel should be enabled.
NotifyUploadProgress bool
// Logger is the logger to use internally, mostly for printing requests.
Logger *log.Logger

View File

@ -69,3 +69,7 @@ This event will be triggered after an upload is fully finished, meaning that all
### post-terminate
This event will be triggered after an upload has been terminated, meaning that the upload has been totally stopped and all associating chunks have been fully removed from the storage. Therefore, one is not able to retrieve the upload's content anymore and one may wish to notify further applications that this upload will never be resumed nor finished.
### post-receive
This event will be triggered for every running upload to indicate its current progress. It will occur for each open PATCH request, every second. The offset property will be set to the number of bytes which have been transfered to the server, at the time in total. Please be aware that this number may be higher than the number of bytes which have been stored by the data store!

View File

@ -20,8 +20,8 @@ import (
// cheap mechanism. Locks will only exist as long as this object is kept in
// reference and will be erased if the program exits.
type MemoryLocker struct {
locks map[string]bool
mutex *sync.Mutex
locks map[string]struct{}
mutex sync.Mutex
}
// NewMemoryLocker creates a new in-memory locker. The DataStore parameter
@ -34,8 +34,7 @@ func NewMemoryLocker(_ tusd.DataStore) *MemoryLocker {
// New creates a new in-memory locker.
func New() *MemoryLocker {
return &MemoryLocker{
locks: make(map[string]bool),
mutex: new(sync.Mutex),
locks: make(map[string]struct{}),
}
}
@ -54,7 +53,7 @@ func (locker *MemoryLocker) LockUpload(id string) error {
return tusd.ErrFileLocked
}
locker.locks[id] = true
locker.locks[id] = struct{}{}
return nil
}
@ -62,11 +61,11 @@ func (locker *MemoryLocker) LockUpload(id string) error {
// UnlockUpload releases a lock. If no such lock exists, no error will be returned.
func (locker *MemoryLocker) UnlockUpload(id string) error {
locker.mutex.Lock()
defer locker.mutex.Unlock()
// Deleting a non-existing key does not end in unexpected errors or panic
// since this operation results in a no-op
delete(locker.locks, id)
locker.mutex.Unlock()
return nil
}

View File

@ -1,11 +1,13 @@
package tusd_test
import (
"io"
"io/ioutil"
"net/http"
"os"
"strings"
"testing"
"time"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
@ -289,4 +291,73 @@ func TestPatch(t *testing.T) {
Code: http.StatusNoContent,
}).Run(handler, t)
})
SubTest(t, "NotifyUploadProgress", func(t *testing.T, store *MockFullDataStore) {
gomock.InOrder(
store.EXPECT().GetInfo("yes").Return(FileInfo{
ID: "yes",
Offset: 0,
Size: 100,
}, nil),
store.EXPECT().WriteChunk("yes", int64(0), NewReaderMatcher("first second third")).Return(int64(18), nil),
)
handler, _ := NewHandler(Config{
DataStore: store,
NotifyUploadProgress: true,
})
c := make(chan FileInfo)
handler.UploadProgress = c
reader, writer := io.Pipe()
a := assert.New(t)
go func() {
writer.Write([]byte("first "))
info := <-c
a.Equal("yes", info.ID)
a.Equal(int64(100), info.Size)
a.Equal(int64(6), info.Offset)
writer.Write([]byte("second "))
writer.Write([]byte("third"))
info = <-c
a.Equal("yes", info.ID)
a.Equal(int64(100), info.Size)
a.Equal(int64(18), info.Offset)
writer.Close()
info = <-c
a.Equal("yes", info.ID)
a.Equal(int64(100), info.Size)
a.Equal(int64(18), info.Offset)
}()
(&httpTest{
Method: "PATCH",
URL: "yes",
ReqHeader: map[string]string{
"Tus-Resumable": "1.0.0",
"Content-Type": "application/offset+octet-stream",
"Upload-Offset": "0",
},
ReqBody: reader,
Code: http.StatusNoContent,
ResHeader: map[string]string{
"Upload-Offset": "18",
},
}).Run(handler, t)
// Wait a short time after the request has been handled before closing the
// channel because another goroutine may still write to the channel.
<-time.After(10 * time.Millisecond)
close(handler.UploadProgress)
_, more := <-c
a.False(more)
})
}

View File

@ -84,6 +84,14 @@ type UnroutedHandler struct {
// happen if the NotifyTerminatedUploads field is set to true in the Config
// structure.
TerminatedUploads chan FileInfo
// UploadProgress is used to send notifications about the progress of the
// currently running uploads. For each open PATCH request, every second
// a FileInfo instance will be send over this channel with the Offset field
// being set to the number of bytes which have been transfered to the server.
// Please be aware that this number may be higher than the number of bytes
// which have been stored by the data store! Sending to this channel will only
// happen if the NotifyUploadProgress field is set to true in the Config
// structure.
UploadProgress chan FileInfo
// Metrics provides numbers of the usage for this handler.
Metrics Metrics
@ -657,6 +665,10 @@ func (w *progressWriter) Write(b []byte) (int, error) {
return len(b), nil
}
// sendProgressMessage will send a notification over the UploadProgress channel
// every second, indicating how much data has been transfered to the server.
// It will stop sending these instances once the returned channel has been
// closed. The returned reader should be used to read the request body.
func (handler *UnroutedHandler) sendProgressMessages(info FileInfo, reader io.Reader) (io.Reader, chan<- struct{}) {
progress := &progressWriter{
Offset: info.Offset,