diff --git a/README.md b/README.md index 083713b..dc0da1d 100644 --- a/README.md +++ b/README.md @@ -202,20 +202,20 @@ func main() { ``` -Please consult the [online documentation](https://godoc.org/github.com/tus/tusd) +Please consult the [online documentation](https://godoc.org/github.com/tus/tusd/pkg) for more details about tusd's APIs and its sub-packages. ## Implementing own storages The tusd server is built to be as flexible as possible and to allow the use of different upload storage mechanisms. By default the tusd binary includes -[`filestore`](https://godoc.org/github.com/tus/tusd/filestore) which will save every upload +[`filestore`](https://godoc.org/github.com/tus/tusd/pkg/filestore) which will save every upload to a specific directory on disk. If you have different requirements, you can build your own storage backend which will save the files to S3, a remote FTP server or similar. Doing so -is as simple as implementing the [`tusd.DataStore`](https://godoc.org/github.com/tus/tusd/#DataStore) -interface and using the new struct in the [configuration object](https://godoc.org/github.com/tus/tusd/#Config). +is as simple as implementing the [`tusd.DataStore`](https://godoc.org/github.com/tus/tusd/pkg/#DataStore) +interface and using the new struct in the [configuration object](https://godoc.org/github.com/tus/tusd/pkg/#Config). Please consult the documentation about detailed information about the required methods. @@ -224,17 +224,18 @@ required methods. This repository does not only contain the HTTP server's code but also other useful tools: -* [**s3store**](https://godoc.org/github.com/tus/tusd/s3store): A storage backend using AWS S3 -* [**filestore**](https://godoc.org/github.com/tus/tusd/filestore): A storage backend using the local file system -* [**gcsstore**](https://godoc.org/github.com/tus/tusd/gcsstore): A storage backend using Google cloud storage -* [**memorylocker**](https://godoc.org/github.com/tus/tusd/memorylocker): An in-memory locker for handling concurrent uploads -* [**etcd3locker**](https://godoc.org/github.com/tus/tusd/etcd3locker): A locker using the distributed KV etcd3 store +* [**s3store**](https://godoc.org/github.com/tus/tusd/pkg/s3store): A storage backend using AWS S3 +* [**filestore**](https://godoc.org/github.com/tus/tusd/pkg/filestore): A storage backend using the local file system +* [**gcsstore**](https://godoc.org/github.com/tus/tusd/pkg/gcsstore): A storage backend using Google cloud storage +* [**memorylocker**](https://godoc.org/github.com/tus/tusd/pkg/memorylocker): An in-memory locker for handling concurrent uploads +* [**filelocker**](https://godoc.org/github.com/tus/tusd/pkg/filelocker): A disk-based locker for handling concurrent uploads ### 3rd-Party tusd Packages The following packages are supported by 3rd-party maintainers outside of this repository. Please file issues respective to the packages in their respective repositories. * [**tusd-dynamo-locker**](https://github.com/chen-anders/tusd-dynamo-locker): A locker using AWS DynamoDB store +* [**tusd-etcd3-locker**](https://github.com/tus/tusd-etcd3-locker): A locker using the distributed KV etcd3 store ## Running the testsuite diff --git a/go.mod b/go.mod index b85e200..cf4ba6a 100644 --- a/go.mod +++ b/go.mod @@ -6,47 +6,12 @@ require ( cloud.google.com/go v0.40.0 github.com/aws/aws-sdk-go v1.20.1 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 - github.com/chen-anders/go-etcd-harness v0.0.0-20181107211618-49dd980f448b - github.com/coreos/bbolt v1.3.3 // indirect - github.com/coreos/etcd v3.3.13+incompatible - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd v0.0.0-20190612170431-362f06ec6bc1 // indirect - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect - github.com/davecgh/go-spew v1.1.1 - github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b - github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/golang/mock v1.3.1 - github.com/golang/protobuf v1.3.1 - github.com/googleapis/gax-go v2.0.2+incompatible - github.com/gorilla/websocket v1.4.0 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.9.1 // indirect - github.com/jonboulle/clockwork v0.1.0 // indirect - github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.0.0 github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 - github.com/soheilhy/cmux v0.1.4 // indirect github.com/stretchr/testify v1.3.0 - github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/vimeo/go-util v1.2.0 - github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.etcd.io/bbolt v1.3.3 // indirect - go.etcd.io/etcd v3.3.13+incompatible // indirect - go.uber.org/atomic v1.4.0 // indirect - go.uber.org/multierr v1.1.0 // indirect - go.uber.org/zap v1.10.0 // indirect - go4.org v0.0.0-20190313082347-94abd6928b1d - golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 - golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c - golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 - golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b - golang.org/x/text v0.3.2 google.golang.org/api v0.6.0 - google.golang.org/appengine v1.5.0 - google.golang.org/genproto v0.0.0-20190530194941-fb225487d101 - google.golang.org/grpc v1.20.1 gopkg.in/Acconut/lockfile.v1 v1.1.0 gopkg.in/h2non/gock.v1 v1.0.14 ) diff --git a/go.sum b/go.sum index 6d4bf4f..2a66c9f 100644 --- a/go.sum +++ b/go.sum @@ -13,27 +13,11 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 h1:y4B3+GPxKlrigF1ha5FFErxK+sr6sWxQovRMzwMhejo= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= -github.com/chen-anders/go-etcd-harness v0.0.0-20181107211618-49dd980f448b h1:cteA/H4jA2ssisyqiZpcSzUkh0EHFhMTqrP9HIk+mJE= -github.com/chen-anders/go-etcd-harness v0.0.0-20181107211618-49dd980f448b/go.mod h1:CoVQD9YyaUuOjBn45IGuywrCkRXPPY2qZIDIoVqtvMQ= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY= -github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190612170431-362f06ec6bc1 h1:3zpwNeYYPkliiyBLhusrKrqlfLr8QbIrtQjs6SY3V9U= -github.com/coreos/go-systemd v0.0.0-20190612170431-362f06ec6bc1/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -41,8 +25,6 @@ github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= @@ -58,18 +40,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= -github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.1 h1:LgwPEIdyJmF9Ug9nINVNspG6Z6P8/TM0yKdQ5h3VQaQ= -github.com/grpc-ecosystem/grpc-gateway v1.9.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -77,8 +49,6 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -86,11 +56,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -115,38 +80,19 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 h1:X9XMOYjxEfAYSy3xK1DzO5dMkkWhs9E9UCcS1IERx2k= github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/vimeo/go-util v1.2.0 h1:YHzwOnM+V2tc6r67K9fXpYqUiRwXp0TgFKuyj+A5bsg= github.com/vimeo/go-util v1.2.0/go.mod h1:s13SMDTSO7AjH1nbgp707mfN5JFIWUFDU5MDDuRRtKs= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v3.3.13+incompatible h1:jCejD5EMnlGxFvcGRyEV4VGlENZc7oPQX6o0t7n3xbw= -go.etcd.io/etcd v3.3.13+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go4.org v0.0.0-20190313082347-94abd6928b1d h1:JkRdGP3zvTtTbabWSAC6n67ka30y7gOzWAah4XYJSfw= -go4.org v0.0.0-20190313082347-94abd6928b1d/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -159,7 +105,6 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -176,7 +121,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA= @@ -193,6 +137,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c h1:97SnQk1GYRXJgvwZ8fadnxDOWfKvkNQHH3CtZntPSrM= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.6.0 h1:2tJEkRfnZL5g1GeBUlITh/rqT5HG3sFcoVCUUxmgJ2g= @@ -215,12 +160,8 @@ gopkg.in/Acconut/lockfile.v1 v1.1.0/go.mod h1:6UCz3wJ8tSFUsPR6uP/j8uegEtDuEEqFxl gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/etcd3locker/lock.go b/pkg/etcd3locker/lock.go deleted file mode 100644 index 233028e..0000000 --- a/pkg/etcd3locker/lock.go +++ /dev/null @@ -1,60 +0,0 @@ -// Package etcd3locker provides a locking mechanism using an etcd3 cluster. -// Tested on etcd 3.1/3.2./3.3 -package etcd3locker - -import ( - "context" - "time" - - "github.com/coreos/etcd/clientv3/concurrency" - "github.com/tus/tusd/pkg/handler" -) - -type etcd3Lock struct { - Id string - Mutex *concurrency.Mutex - Session *concurrency.Session - - isHeld bool -} - -func newEtcd3Lock(session *concurrency.Session, id string) *etcd3Lock { - return &etcd3Lock{ - Mutex: concurrency.NewMutex(session, id), - Session: session, - } -} - -// Acquires a lock from etcd3 -func (lock *etcd3Lock) Lock() error { - if lock.isHeld { - return handler.ErrFileLocked - } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - // this is a blocking call; if we receive DeadlineExceeded - // the lock is most likely already taken - if err := lock.Mutex.Lock(ctx); err != nil { - if err == context.DeadlineExceeded { - return handler.ErrFileLocked - } else { - return err - } - } - - lock.isHeld = true - return nil -} - -// Releases a lock from etcd3 -func (lock *etcd3Lock) Unlock() error { - if !lock.isHeld { - return ErrLockNotHeld - } - - lock.isHeld = false - defer lock.Session.Close() - return lock.Mutex.Unlock(context.Background()) -} diff --git a/pkg/etcd3locker/locker.go b/pkg/etcd3locker/locker.go deleted file mode 100644 index 95e2c19..0000000 --- a/pkg/etcd3locker/locker.go +++ /dev/null @@ -1,106 +0,0 @@ -// Package etcd3locker provides a locking mechanism using an etcd3 cluster -// -// To initialize a locker, a pre-existing connected etcd3 client must be present -// -// client, err := clientv3.New(clientv3.Config{ -// Endpoints: []string{harness.Endpoint}, -// DialTimeout: 5 * time.Second, -// }) -// -// For the most basic locker (e.g. non-shared etcd3 cluster / use default TTLs), -// a locker can be instantiated like the following: -// -// locker, err := etcd3locker.New(client) -// if err != nil { -// return nil, fmt.Errorf("Failed to create etcd locker: %v", err.Error()) -// } -// -// The locker will need to be included in composer that is used by tusd: -// -// composer := handler.NewStoreComposer() -// locker.UseIn(composer) -// -// For a shared etcd3 cluster, you may want to modify the prefix that etcd3locker uses: -// -// locker, err := etcd3locker.NewWithPrefix(client, "my-prefix") -// if err != nil { -// return nil, fmt.Errorf("Failed to create etcd locker: %v", err.Error()) -// } -// -// -// For full control over all options, an etcd3.LockerOptions may be passed into -// etcd3.NewWithLockerOptions like the following example: -// -// ttl := 15 // seconds -// options := etcd3locker.NewLockerOptions(ttl, "my-prefix") -// locker, err := etcd3locker.NewWithLockerOptions(client, options) -// if err != nil { -// return nil, fmt.Errorf("Failed to create etcd locker: %v", err.Error()) -// } -// -// Tested on etcd 3.1/3.2/3.3 -// -package etcd3locker - -import ( - "errors" - "time" - - etcd3 "github.com/coreos/etcd/clientv3" - "github.com/coreos/etcd/clientv3/concurrency" - "github.com/tus/tusd/pkg/handler" -) - -var ( - ErrLockNotHeld = errors.New("Lock not held") - GrantTimeout = 1500 * time.Millisecond -) - -type Etcd3Locker struct { - // etcd3 client session - Client *etcd3.Client - - prefix string - sessionTtl int -} - -// New constructs a new locker using the provided client. -func New(client *etcd3.Client) (*Etcd3Locker, error) { - return NewWithLockerOptions(client, DefaultLockerOptions()) -} - -// This method may be used if a different prefix is required for multi-tenant etcd clusters -func NewWithPrefix(client *etcd3.Client, prefix string) (*Etcd3Locker, error) { - lockerOptions := DefaultLockerOptions() - lockerOptions.SetPrefix(prefix) - return NewWithLockerOptions(client, lockerOptions) -} - -// This method may be used if we want control over both prefix/session TTLs. This is used for testing in particular. -func NewWithLockerOptions(client *etcd3.Client, opts LockerOptions) (*Etcd3Locker, error) { - return &Etcd3Locker{Client: client, prefix: opts.Prefix(), sessionTtl: opts.Ttl()}, nil -} - -// UseIn adds this locker to the passed composer. -func (locker *Etcd3Locker) UseIn(composer *handler.StoreComposer) { - composer.UseLocker(locker) -} - -func (locker *Etcd3Locker) NewLock(id string) (handler.Lock, error) { - session, err := locker.createSession() - if err != nil { - return nil, err - } - - lock := newEtcd3Lock(session, locker.getId(id)) - - return lock, nil -} - -func (locker *Etcd3Locker) createSession() (*concurrency.Session, error) { - return concurrency.NewSession(locker.Client, concurrency.WithTTL(locker.sessionTtl)) -} - -func (locker *Etcd3Locker) getId(id string) string { - return locker.prefix + id -} diff --git a/pkg/etcd3locker/locker_options.go b/pkg/etcd3locker/locker_options.go deleted file mode 100644 index dcbfc73..0000000 --- a/pkg/etcd3locker/locker_options.go +++ /dev/null @@ -1,66 +0,0 @@ -package etcd3locker - -import ( - "strings" -) - -var ( - DefaultTtl = 60 - DefaultPrefix = "/tusd" -) - -type LockerOptions struct { - ttl int - prefix string -} - -// DefaultLockerOptions() instantiates an instance of LockerOptions -// with default 60 second time to live and an etcd3 prefix of "/tusd" -func DefaultLockerOptions() LockerOptions { - return LockerOptions{ - ttl: 60, - prefix: "/tusd", - } -} - -// NewLockerOptions instantiates an instance of LockerOptions with a -// provided TTL (time to live) and string prefix for keys to be stored in etcd3 -func NewLockerOptions(ttl int, prefix string) LockerOptions { - return LockerOptions{ - ttl: ttl, - prefix: prefix, - } -} - -// Returns the TTL (time to live) of sessions in etcd3 -func (l *LockerOptions) Ttl() int { - if l.ttl == 0 { - return DefaultTtl - } else { - return l.ttl - } -} - -// Returns the string prefix used to store keys in etcd3 -func (l *LockerOptions) Prefix() string { - prefix := l.prefix - if !strings.HasPrefix(prefix, "/") { - prefix = "/" + prefix - } - - if prefix == "" { - return DefaultPrefix - } else { - return prefix - } -} - -// Set etcd3 session TTL (time to live) -func (l *LockerOptions) SetTtl(ttl int) { - l.ttl = ttl -} - -// Set string prefix to be used in keys stored into etcd3 by the locker -func (l *LockerOptions) SetPrefix(prefix string) { - l.prefix = prefix -} diff --git a/pkg/etcd3locker/locker_test.go b/pkg/etcd3locker/locker_test.go deleted file mode 100644 index aa03f5a..0000000 --- a/pkg/etcd3locker/locker_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package etcd3locker - -import ( - "os" - "testing" - "time" - - etcd_harness "github.com/chen-anders/go-etcd-harness" - "github.com/coreos/etcd/clientv3" - - "github.com/stretchr/testify/assert" - "github.com/tus/tusd/pkg/handler" -) - -var _ handler.Locker = &Etcd3Locker{} - -func TestEtcd3Locker(t *testing.T) { - a := assert.New(t) - - harness, err := etcd_harness.New(os.Stderr) - if err != nil { - t.Fatalf("failed starting etcd harness: %v", err) - } - t.Logf("will use etcd harness endpoint: %v", harness.Endpoint) - defer func() { - harness.Stop() - t.Logf("cleaned up etcd harness") - }() - - client, err := clientv3.New(clientv3.Config{ - Endpoints: []string{harness.Endpoint}, - DialTimeout: 5 * time.Second, - }) - if err != nil { - t.Fatalf("Unable to connect to etcd3: %v", err) - } - defer client.Close() - - shortTTL := 3 - testPrefix := "/test-tusd" - - lockerOptions := NewLockerOptions(shortTTL, testPrefix) - locker, err := NewWithLockerOptions(client, lockerOptions) - a.NoError(err) - - lock1, err := locker.NewLock("one") - a.NoError(err) - a.NoError(lock1.Lock()) - - //a.Equal(handler.ErrFileLocked, lock1.Lock()) - time.Sleep(5 * time.Second) - // test that we can't take over the upload via a different etcd3 session - // while an upload is already taking place; testing etcd3 session KeepAlive - lock2, err := locker.NewLock("one") - a.NoError(err) - a.Equal(handler.ErrFileLocked, lock2.Lock()) - a.NoError(lock1.Unlock()) - a.Equal(ErrLockNotHeld, lock1.Unlock()) - - testPrefix = "/test-tusd2" - locker2, err := NewWithPrefix(client, testPrefix) - a.NoError(err) - - lock3, err := locker2.NewLock("one") - a.NoError(err) - - a.NoError(lock3.Lock()) - a.Equal(handler.ErrFileLocked, lock3.Lock()) - a.Equal(handler.ErrFileLocked, lock3.Lock()) - a.NoError(lock3.Unlock()) - a.Equal(ErrLockNotHeld, lock3.Unlock()) -}