From 5a385832b63f2dda1c7cfa748225d004e62b1f78 Mon Sep 17 00:00:00 2001 From: Marius Date: Wed, 4 Jan 2017 20:15:09 +0100 Subject: [PATCH 1/6] Add notice about using HTTPS to README --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index a5062b1..fdb80df 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,12 @@ useful tools: go test -v ./... ``` +## FAQ + +### How can I access tusd using HTTPS? + +The tusd binary, once executed, listens on the provided port for only non-encrypted HTTP requests and *does not accept* HTTPS connections. This decision has been made to limit the functionality inside this repository which has to be developed, tested and maintained. If you want to send requests to tusd in a secure fashion - what we absolutely encourage, we recommend you to utilize a reverse proxy in front of tusd which accepts incoming HTTPS connections and forwards them to tusd using plain HTTP. More information about this topic, including sample configurations for Nginx and Apache, can be found in [issue #86](https://github.com/tus/tusd/issues/86#issuecomment-269569077). + ## License This project is licensed under the MIT license, see `LICENSE.txt`. From 19c9576eb8a8e9454bfdd4e331715f483637bc7c Mon Sep 17 00:00:00 2001 From: Marius Date: Wed, 4 Jan 2017 20:25:08 +0100 Subject: [PATCH 2/6] Use platform-independent paths inside file storage Closes https://github.com/tus/tusd/issues/84 --- filestore/filestore.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/filestore/filestore.go b/filestore/filestore.go index ad52a7e..5808cac 100644 --- a/filestore/filestore.go +++ b/filestore/filestore.go @@ -171,7 +171,7 @@ func (store FileStore) UnlockUpload(id string) error { // newLock contructs a new Lockfile instance. func (store FileStore) newLock(id string) (lockfile.Lockfile, error) { - path, err := filepath.Abs(store.Path + "/" + id + ".lock") + path, err := filepath.Abs(filepath.Join(store.Path, id+".lock")) if err != nil { return lockfile.Lockfile(""), err } @@ -184,12 +184,12 @@ func (store FileStore) newLock(id string) (lockfile.Lockfile, error) { // binPath returns the path to the .bin storing the binary data. func (store FileStore) binPath(id string) string { - return store.Path + "/" + id + ".bin" + return filepath.Join(store.Path, id+".bin") } // infoPath returns the path to the .info file storing the file's info. func (store FileStore) infoPath(id string) string { - return store.Path + "/" + id + ".info" + return filepath.Join(store.Path, id+".info") } // writeInfo updates the entire information. Everything will be overwritten. From 0a8a3e7e51b21a378c7c0697cd5dfb6d43a84e14 Mon Sep 17 00:00:00 2001 From: Marius Date: Wed, 4 Jan 2017 20:47:34 +0100 Subject: [PATCH 3/6] Count multiple closed connectios only once --- cmd/tusd/cli/listener.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cmd/tusd/cli/listener.go b/cmd/tusd/cli/listener.go index 4438c19..054a39f 100644 --- a/cmd/tusd/cli/listener.go +++ b/cmd/tusd/cli/listener.go @@ -37,6 +37,11 @@ type Conn struct { net.Conn ReadTimeout time.Duration WriteTimeout time.Duration + + // closeRecorded will be true if the connection has been closed and the + // corresponding prometheus counter has been decremented. It will be used to + // avoid duplicated modifications to this metric. + closeRecorded bool } func (c *Conn) Read(b []byte) (int, error) { @@ -70,7 +75,13 @@ func (c *Conn) Write(b []byte) (int, error) { } func (c *Conn) Close() error { - go MetricsOpenConnections.Dec() + // Only decremented the prometheus counter if the Close function has not been + // invoked before to avoid duplicated modifications. + if !c.closeRecorded { + c.closeRecorded = true + MetricsOpenConnections.Dec() + } + return c.Conn.Close() } From e89661cbdad0f5b34d86a510319113fef1361251 Mon Sep 17 00:00:00 2001 From: mrezai Date: Wed, 4 Jan 2017 21:30:40 +0100 Subject: [PATCH 4/6] Fix README sample code errors Closes https://github.com/tus/tusd/pull/87 --- README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index fdb80df..52a4264 100644 --- a/README.md +++ b/README.md @@ -54,14 +54,17 @@ your own Go program: package main import ( + "fmt" + "net/http" + "github.com/tus/tusd" "github.com/tus/tusd/filestore" - "net/http" ) func main() { // Create a new FileStore instance which is responsible for // storing the uploaded file on disk in the specified directory. + // This path _must_ exist before tusd will store uploads in it. // If you want to save them on a different medium, for example // a remote FTP server, you can implement your own storage backend // by implementing the tusd.DataStore interface. @@ -79,22 +82,23 @@ func main() { // Create a new HTTP handler for the tusd server by providing a configuration. // The StoreComposer property must be set to allow the handler to function. handler, err := tusd.NewHandler(tusd.Config{ - BasePath: "files/", + BasePath: "/files/", StoreComposer: composer, }) if err != nil { - panic("Unable to create handler: %s", err) + panic(fmt.Errorf("Unable to create handler: %s", err)) } // Right now, nothing has happened since we need to start the HTTP server on // our own. In the end, tusd will start listening on and accept request at // http://localhost:8080/files - http.Handle("files/", http.StripPrefix("files/", handler)) + http.Handle("/files/", http.StripPrefix("/files/", handler)) err = http.ListenAndServe(":8080", nil) if err != nil { - panic("Unable to listen: %s", err) + panic(fmt.Errorf("Unable to listen: %s", err)) } } + ``` Please consult the [online documentation](https://godoc.org/github.com/tus/tusd) From 50dbb7d7ba7ac5ccd737fdbe5fb899966130c193 Mon Sep 17 00:00:00 2001 From: Marius Date: Wed, 4 Jan 2017 23:48:55 +0100 Subject: [PATCH 5/6] Add information about reverse proxies to README --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 52a4264..fe9651e 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,17 @@ go test -v ./... The tusd binary, once executed, listens on the provided port for only non-encrypted HTTP requests and *does not accept* HTTPS connections. This decision has been made to limit the functionality inside this repository which has to be developed, tested and maintained. If you want to send requests to tusd in a secure fashion - what we absolutely encourage, we recommend you to utilize a reverse proxy in front of tusd which accepts incoming HTTPS connections and forwards them to tusd using plain HTTP. More information about this topic, including sample configurations for Nginx and Apache, can be found in [issue #86](https://github.com/tus/tusd/issues/86#issuecomment-269569077). +### Can I run tusd behind a reverse proxy? + +Yes, it is absolutely possible to do so. Firstly, you should execute the tusd binary using the `-behind-proxy` flag indicating it to pay attention to special headers which are only relevent when used in conjunction with a proxy. Furthermore, there are addtional details which should be kept in mind, depending on the used software: + +- *Disable request buffering.* Nginx, for example, reads the entire incoming HTTP request, including its body, before sending it to the backend, by default. This behavior defeats the purpose of resumability where an upload is processed while it's being transfered. Therefore, such as feature should be disabled. + +- *Adjust maximum request size.* Some proxies have default values for how big a request may be in order to protect your services. Be sure to check these settings to match the requirements of your application. + +- *Forward hostname and scheme.* If the proxy rewrites the request URL, the tusd server does not know the original URL which was used to reach the proxy. This behavior can lead to situations, where tusd returns a redirect to a URL which can not be reached by the client. To avoid this confusion, you can explicitly tell tusd which hostname and scheme to use by supplying the `X-Forwarded-Host` and `X-Forwarded-Proto` headers. + +Explicit examples for the above points can be found in the [Nginx configuration](https://github.com/tus/tusd/blob/master/.infra/files/nginx.conf) which is used to power the [master.tus.io](https://master.tus.io) instace. ## License This project is licensed under the MIT license, see `LICENSE.txt`. From 598ca78fa7b3f11a9be23f69f5fae129e38e3969 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Wed, 11 Jan 2017 10:25:28 +0100 Subject: [PATCH 6/6] Upgrade Frey to v0.3.27 /cc @tersmitten (#85) * Upgrade Frey to v0.3.27 /cc @tersmitten * Add forgotten and misspelled role variables * Make it easier to override settings and clear that this is a private file --- .infra/Freyfile.hcl | 18 +++++++++--------- .infra/group_vars/all/_frey.yml | 6 ++++++ 2 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 .infra/group_vars/all/_frey.yml diff --git a/.infra/Freyfile.hcl b/.infra/Freyfile.hcl index 4cb334c..107aa1b 100644 --- a/.infra/Freyfile.hcl +++ b/.infra/Freyfile.hcl @@ -106,11 +106,11 @@ install { hosts = "tusd" name = "Install tusd" roles { - role = "{{{init.paths.roles_dir}}}/apt/v1.0.0" - apt_packages = ["apg", "build-essential", "curl", "git-core", "htop", "iotop", "libpcre3", "logtail", "mlocate", "mtr", "psmisc", "telnet", "vim", "wget"] + role = "{{{init.paths.roles_dir}}}/apt/1.3.0" + apt_install = ["apg", "build-essential", "curl", "git-core", "htop", "iotop", "libpcre3", "logtail", "mlocate", "mtr", "psmisc", "telnet", "vim", "wget"] } roles { - role = "{{{init.paths.roles_dir}}}/unattended-upgrades/v1.2.0" + role = "{{{init.paths.roles_dir}}}/unattended-upgrades/1.3.0" } tasks { lineinfile = "dest=/home/{{{config.global.ssh.user}}}/.bashrc line=\"alias wtf='sudo tail -f /var/log/*{log,err} /var/log/{dmesg,messages,*{,/*}{log,err}}'\" owner={{{config.global.ssh.user}}} group={{{config.global.ssh.user}}} mode=0644 backup=yes" @@ -141,7 +141,7 @@ setup { hosts = "tusd" name = "Setup tusd" roles { - role = "{{{init.paths.roles_dir}}}/upstart/v1.0.0" + role = "{{{init.paths.roles_dir}}}/upstart/1.0.0" upstart_command = "./tusd -port=8080 -dir=/mnt/tusd-data -max-size=1000000000 -behind-proxy" upstart_description = "tusd server" upstart_name = "{{{config.global.appname}}}" @@ -152,7 +152,7 @@ setup { upstart_user = "www-data" } roles { - role = "{{{init.paths.roles_dir}}}/rsyslog/v3.0.1" + role = "{{{init.paths.roles_dir}}}/rsyslog/3.1.0" rsyslog_rsyslog_d_files "49-tusd" { directives = ["& stop"] rules { @@ -162,7 +162,7 @@ setup { } } roles { - role = "{{{init.paths.roles_dir}}}/fqdn/v1.0.0" + role = "{{{init.paths.roles_dir}}}/fqdn/1.0.0" fqdn = "{{lookup('env', 'FREY_DOMAIN')}}" } tasks { @@ -198,7 +198,7 @@ deploy { hosts = "tusd" name = "Deploy tusd" roles { - role = "{{{init.paths.roles_dir}}}/deploy/v1.4.0" + role = "{{{init.paths.roles_dir}}}/deploy/1.4.0" ansistrano_deploy_from = "./files/tusd_linux_amd64.tar.gz" ansistrano_deploy_to = "{{{config.global.approot}}}" ansistrano_deploy_via = "copy_unarchive" @@ -217,8 +217,8 @@ deploy { hosts = "tusd" name = "Deploy nginx" roles { - role = "{{{init.paths.roles_dir}}}/apt/v1.0.0" - apt_packages = ["nginx-light"] + role = "{{{init.paths.roles_dir}}}/apt/1.3.0" + apt_install = ["nginx-light"] } tasks { name = "nginx | Create nginx configuration" diff --git a/.infra/group_vars/all/_frey.yml b/.infra/group_vars/all/_frey.yml new file mode 100644 index 0000000..9eb5f20 --- /dev/null +++ b/.infra/group_vars/all/_frey.yml @@ -0,0 +1,6 @@ +# Frey specific overrides +--- +# apt role +apt_manage_sources_list: true +apt_src_enable: false +apt_install_state: present