Merge branch 'master' of github.com:tus/tusd into v2
This commit is contained in:
Marius
commit
387b04a2e2
|
|
@ -5,8 +5,9 @@ name: Test
|
|||
jobs:
|
||||
test:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
go-version: [1.12.x, 1.13.x]
|
||||
go-version: [1.16.x, 1.17.x]
|
||||
platform: [ubuntu-latest, macos-latest, windows-latest]
|
||||
runs-on: ${{ matrix.platform }}
|
||||
steps:
|
||||
|
|
@ -26,14 +27,14 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')
|
||||
steps:
|
||||
- name: Install Go 1.13.1
|
||||
- name: Install Go 1.17.2
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: '1.13.1'
|
||||
go-version: '1.17.2'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v2
|
||||
- name: Build TUSD
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
env:
|
||||
GO111MODULE: on
|
||||
run: ./scripts/build_all.sh
|
||||
|
|
@ -43,7 +44,7 @@ jobs:
|
|||
with:
|
||||
files: tusd_*.*
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GH_RELEASE_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ secrets.GH_RELEASE_TOKEN }}
|
||||
- name: Deploy to heroku
|
||||
uses: akhileshns/heroku-deploy@v3.4.6
|
||||
with:
|
||||
|
|
@ -53,8 +54,8 @@ jobs:
|
|||
- uses: azure/docker-login@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Build and push docker image
|
||||
run: |
|
||||
docker build -t tusproject/tusd:$GITHUB_SHA .
|
||||
docker push tusproject/tusd:$GITHUB_SHA
|
||||
docker push tusproject/tusd:$GITHUB_SHA
|
||||
|
|
|
|||
|
|
@ -3,3 +3,4 @@ cover.out
|
|||
data/
|
||||
node_modules/
|
||||
.DS_Store
|
||||
./tusd
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
FROM golang:1.13-alpine AS builder
|
||||
FROM golang:1.16-alpine AS builder
|
||||
|
||||
# Copy in the git repo from the build context
|
||||
COPY . /go/src/github.com/tus/tusd/
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/tus/tusd/pkg/azurestore"
|
||||
"github.com/tus/tusd/pkg/filelocker"
|
||||
"github.com/tus/tusd/pkg/filestore"
|
||||
"github.com/tus/tusd/pkg/gcsstore"
|
||||
|
|
@ -16,6 +17,8 @@ import (
|
|||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
var Composer *handler.StoreComposer
|
||||
|
|
@ -96,6 +99,49 @@ func CreateComposer() {
|
|||
|
||||
locker := memorylocker.New()
|
||||
locker.UseIn(Composer)
|
||||
} else if Flags.AzStorage != "" {
|
||||
|
||||
accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
|
||||
if accountName == "" {
|
||||
stderr.Fatalf("No service account name for Azure BlockBlob Storage using the AZURE_STORAGE_ACCOUNT environment variable.\n")
|
||||
}
|
||||
|
||||
accountKey := os.Getenv("AZURE_STORAGE_KEY")
|
||||
if accountKey == "" {
|
||||
stderr.Fatalf("No service account key for Azure BlockBlob Storage using the AZURE_STORAGE_KEY environment variable.\n")
|
||||
}
|
||||
|
||||
azureEndpoint := Flags.AzEndpoint
|
||||
// Enables support for using Azurite as a storage emulator without messing with proxies and stuff
|
||||
// e.g. http://127.0.0.1:10000/devstoreaccount1
|
||||
if azureEndpoint == "" {
|
||||
azureEndpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
|
||||
stdout.Printf("Custom Azure Endpoint not specified in flag variable azure-endpoint.\n"+
|
||||
"Using endpoint %s\n", azureEndpoint)
|
||||
} else {
|
||||
stdout.Printf("Using Azure endpoint %s\n", azureEndpoint)
|
||||
}
|
||||
|
||||
azConfig := &azurestore.AzConfig{
|
||||
AccountName: accountName,
|
||||
AccountKey: accountKey,
|
||||
ContainerName: Flags.AzStorage,
|
||||
ContainerAccessType: Flags.AzContainerAccessType,
|
||||
BlobAccessTier: Flags.AzBlobAccessTier,
|
||||
Endpoint: azureEndpoint,
|
||||
}
|
||||
|
||||
azService, err := azurestore.NewAzureService(azConfig)
|
||||
if err != nil {
|
||||
stderr.Fatalf(err.Error())
|
||||
}
|
||||
|
||||
store := azurestore.New(azService)
|
||||
store.ObjectPrefix = Flags.AzObjectPrefix
|
||||
store.Container = Flags.AzStorage
|
||||
|
||||
store.UseIn(Composer)
|
||||
|
||||
} else {
|
||||
dir, err := filepath.Abs(Flags.UploadDir)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,11 @@ var Flags struct {
|
|||
S3ConcurrentPartUploads int
|
||||
GCSBucket string
|
||||
GCSObjectPrefix string
|
||||
AzStorage string
|
||||
AzContainerAccessType string
|
||||
AzBlobAccessTier string
|
||||
AzObjectPrefix string
|
||||
AzEndpoint string
|
||||
EnabledHooksString string
|
||||
FileHooksDir string
|
||||
HttpHooksEndpoint string
|
||||
|
|
@ -78,6 +83,11 @@ func ParseFlags() {
|
|||
flag.IntVar(&Flags.S3ConcurrentPartUploads, "s3-concurrent-part-uploads", 10, "Number of concurrent part uploads to S3 (experimental and may be removed in the future)")
|
||||
flag.StringVar(&Flags.GCSBucket, "gcs-bucket", "", "Use Google Cloud Storage with this bucket as storage backend (requires the GCS_SERVICE_ACCOUNT_FILE environment variable to be set)")
|
||||
flag.StringVar(&Flags.GCSObjectPrefix, "gcs-object-prefix", "", "Prefix for GCS object names (can't contain underscore character)")
|
||||
flag.StringVar(&Flags.AzStorage, "azure-storage", "", "Use Azure BlockBlob Storage with this container name as a storage backend (requires the AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY environment variable to be set)")
|
||||
flag.StringVar(&Flags.AzContainerAccessType, "azure-container-access-type", "", "Access type when creating a new container if it does not exist (possible values: blob, container, '')")
|
||||
flag.StringVar(&Flags.AzBlobAccessTier, "azure-blob-access-tier", "", "Blob access tier when uploading new files (possible values: archive, cool, hot, '')")
|
||||
flag.StringVar(&Flags.AzObjectPrefix, "azure-object-prefix", "", "Prefix for Azure object names")
|
||||
flag.StringVar(&Flags.AzEndpoint, "azure-endpoint", "", "Custom Endpoint to use for Azure BlockBlob Storage (requires azure-storage to be pass)")
|
||||
flag.StringVar(&Flags.EnabledHooksString, "hooks-enabled-events", "pre-create,post-create,post-receive,post-terminate,post-finish", "Comma separated list of enabled hook events (e.g. post-create,post-finish). Leave empty to enable default events")
|
||||
flag.StringVar(&Flags.FileHooksDir, "hooks-dir", "", "Directory to search for available hooks scripts")
|
||||
flag.StringVar(&Flags.HttpHooksEndpoint, "hooks-http", "", "An HTTP endpoint to which hook events will be sent to")
|
||||
|
|
|
|||
6
go.mod
6
go.mod
|
|
@ -1,9 +1,13 @@
|
|||
module github.com/tus/tusd
|
||||
|
||||
go 1.12
|
||||
// Specify the Go version needed for the Heroku deployment
|
||||
// See https://github.com/heroku/heroku-buildpack-go#go-module-specifics
|
||||
// +heroku goVersion go1.16
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.40.0
|
||||
github.com/Azure/azure-storage-blob-go v0.13.0
|
||||
github.com/aws/aws-sdk-go v1.20.1
|
||||
github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40
|
||||
github.com/golang/mock v1.3.1
|
||||
|
|
|
|||
29
go.sum
29
go.sum
|
|
@ -3,6 +3,17 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
|
|||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.40.0 h1:FjSY7bOj+WzJe6TZRVtXI2b9kAYvtNg4lMbcH2+MUkk=
|
||||
cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
|
||||
github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U=
|
||||
github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
|
||||
github.com/Azure/azure-storage-blob-go v0.12.0 h1:7bFXA1QB+lOK2/ASWHhp6/vnxjaeeZq6t8w1Jyp0Iaw=
|
||||
github.com/Azure/azure-storage-blob-go v0.12.0/go.mod h1:A0u4VjtpgZJ7Y7um/+ix2DHBuEKFC6sEIlj0xc13a4Q=
|
||||
github.com/Azure/azure-storage-blob-go v0.13.0 h1:lgWHvFh+UYBNVQLFHXkvul2f6yOPA9PIH82RTG2cSwc=
|
||||
github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nUdBD+e64lEuc4sVnuOfNs=
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE=
|
||||
github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
|
||||
github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
|
||||
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
|
|
@ -20,6 +31,7 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
|
|||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
|
|
@ -51,6 +63,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
|
|||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg=
|
||||
|
|
@ -71,6 +85,10 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
|
|||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
|
||||
github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
|
|
@ -80,10 +98,12 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
|
|||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
|
|
@ -122,6 +142,7 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
|||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
|
|
@ -134,8 +155,11 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r
|
|||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20191112182307-2180aed22343 h1:00ohfJ4K98s3m6BGUoBd8nyfp4Yl0GoIKvw5abItTjI=
|
||||
golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
|
||||
|
|
@ -149,9 +173,13 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
|
|||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200828194041-157a740278f4 h1:kCCpuwSAoYJPkNc6x0xT9yTtV4oKtARo4RGBQWOfg9E=
|
||||
golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
|
|
@ -196,6 +224,7 @@ gopkg.in/Acconut/lockfile.v1 v1.1.0/go.mod h1:6UCz3wJ8tSFUsPR6uP/j8uegEtDuEEqFxl
|
|||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM=
|
||||
gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
|
||||
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,310 @@
|
|||
// Package azurestore provides a Azure Blob Storage based backend
|
||||
|
||||
// AzureStore is a storage backend that uses the AzService interface in order to store uploads in Azure Blob Storage.
|
||||
// It stores the uploads in a container specified in two different BlockBlob: The `[id].info` blobs are used to store the fileinfo in JSON format. The `[id]` blobs without an extension contain the raw binary data uploaded.
|
||||
// If the upload is not finished within a week, the uncommited blocks will be discarded.
|
||||
|
||||
// Support for setting the default Continaer access type and Blob access tier varies on your Azure Storage Account and its limits.
|
||||
// More information about Container access types and limts
|
||||
// https://docs.microsoft.com/en-us/azure/storage/blobs/anonymous-read-access-configure?tabs=portal
|
||||
|
||||
// More information about Blob access tiers and limits
|
||||
// https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-performance-tiers
|
||||
// https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#access-tiers-for-block-blob-data
|
||||
|
||||
package azurestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-storage-blob-go/azblob"
|
||||
)
|
||||
|
||||
const (
|
||||
InfoBlobSuffix string = ".info"
|
||||
MaxBlockBlobSize int64 = azblob.BlockBlobMaxBlocks * azblob.BlockBlobMaxStageBlockBytes
|
||||
MaxBlockBlobChunkSize int64 = azblob.BlockBlobMaxStageBlockBytes
|
||||
)
|
||||
|
||||
type azService struct {
|
||||
BlobAccessTier azblob.AccessTierType
|
||||
ContainerURL *azblob.ContainerURL
|
||||
ContainerName string
|
||||
}
|
||||
|
||||
type AzService interface {
|
||||
NewBlob(ctx context.Context, name string) (AzBlob, error)
|
||||
}
|
||||
|
||||
type AzConfig struct {
|
||||
AccountName string
|
||||
AccountKey string
|
||||
BlobAccessTier string
|
||||
ContainerName string
|
||||
ContainerAccessType string
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
type AzBlob interface {
|
||||
// Delete the blob
|
||||
Delete(ctx context.Context) error
|
||||
// Upload the blob
|
||||
Upload(ctx context.Context, body io.ReadSeeker) error
|
||||
// Download the contents of the blob
|
||||
Download(ctx context.Context) ([]byte, error)
|
||||
// Get the offset of the blob and its indexes
|
||||
GetOffset(ctx context.Context) (int64, error)
|
||||
// Commit the uploaded blocks to the BlockBlob
|
||||
Commit(ctx context.Context) error
|
||||
}
|
||||
|
||||
type BlockBlob struct {
|
||||
Blob *azblob.BlockBlobURL
|
||||
AccessTier azblob.AccessTierType
|
||||
Indexes []int
|
||||
}
|
||||
|
||||
type InfoBlob struct {
|
||||
Blob *azblob.BlockBlobURL
|
||||
}
|
||||
|
||||
// New Azure service for communication to Azure BlockBlob Storage API
|
||||
func NewAzureService(config *AzConfig) (AzService, error) {
|
||||
// struct to store your credentials.
|
||||
credential, err := azblob.NewSharedKeyCredential(config.AccountName, config.AccountKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Might be limited by the storage account
|
||||
// "" or default inherits the access type from the Storage Account
|
||||
var containerAccessType azblob.PublicAccessType
|
||||
switch config.ContainerAccessType {
|
||||
case "container":
|
||||
containerAccessType = azblob.PublicAccessContainer
|
||||
case "blob":
|
||||
containerAccessType = azblob.PublicAccessBlob
|
||||
case "":
|
||||
default:
|
||||
containerAccessType = azblob.PublicAccessNone
|
||||
}
|
||||
|
||||
// Does not support the premium access tiers
|
||||
var blobAccessTierType azblob.AccessTierType
|
||||
switch config.BlobAccessTier {
|
||||
case "archive":
|
||||
blobAccessTierType = azblob.AccessTierArchive
|
||||
case "cool":
|
||||
blobAccessTierType = azblob.AccessTierCool
|
||||
case "hot":
|
||||
blobAccessTierType = azblob.AccessTierHot
|
||||
case "":
|
||||
default:
|
||||
blobAccessTierType = azblob.DefaultAccessTier
|
||||
}
|
||||
|
||||
// The pipeline specifies things like retry policies, logging, deserialization of HTTP response payloads, and more.
|
||||
p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
|
||||
cURL, _ := url.Parse(fmt.Sprintf("%s/%s", config.Endpoint, config.ContainerName))
|
||||
|
||||
// Get the ContainerURL URL
|
||||
containerURL := azblob.NewContainerURL(*cURL, p)
|
||||
// Do not care about response since it will fail if container exists and create if it does not.
|
||||
_, _ = containerURL.Create(context.Background(), azblob.Metadata{}, containerAccessType)
|
||||
|
||||
return &azService{
|
||||
BlobAccessTier: blobAccessTierType,
|
||||
ContainerURL: &containerURL,
|
||||
ContainerName: config.ContainerName,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Determine if we return a InfoBlob or BlockBlob, based on the name
|
||||
func (service *azService) NewBlob(ctx context.Context, name string) (AzBlob, error) {
|
||||
var fileBlob AzBlob
|
||||
bb := service.ContainerURL.NewBlockBlobURL(name)
|
||||
if strings.HasSuffix(name, InfoBlobSuffix) {
|
||||
fileBlob = &InfoBlob{
|
||||
Blob: &bb,
|
||||
}
|
||||
} else {
|
||||
fileBlob = &BlockBlob{
|
||||
Blob: &bb,
|
||||
Indexes: []int{},
|
||||
AccessTier: service.BlobAccessTier,
|
||||
}
|
||||
}
|
||||
return fileBlob, nil
|
||||
}
|
||||
|
||||
// Delete the blockBlob from Azure Blob Storage
|
||||
func (blockBlob *BlockBlob) Delete(ctx context.Context) error {
|
||||
_, err := blockBlob.Blob.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{})
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload a block to Azure Blob Storage and add it to the indexes to be after upload is finished
|
||||
func (blockBlob *BlockBlob) Upload(ctx context.Context, body io.ReadSeeker) error {
|
||||
// Keep track of the indexes
|
||||
var index int
|
||||
if len(blockBlob.Indexes) == 0 {
|
||||
index = 0
|
||||
} else {
|
||||
index = blockBlob.Indexes[len(blockBlob.Indexes)-1] + 1
|
||||
}
|
||||
blockBlob.Indexes = append(blockBlob.Indexes, index)
|
||||
|
||||
_, err := blockBlob.Blob.StageBlock(ctx, blockIDIntToBase64(index), body, azblob.LeaseAccessConditions{}, nil, azblob.ClientProvidedKeyOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download the blockBlob from Azure Blob Storage
|
||||
func (blockBlob *BlockBlob) Download(ctx context.Context) (data []byte, err error) {
|
||||
downloadResponse, err := blockBlob.Blob.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
|
||||
|
||||
// If the file does not exist, it will not return an error, but a 404 status and body
|
||||
if downloadResponse != nil && downloadResponse.StatusCode() == 404 {
|
||||
return nil, fmt.Errorf("File %s does not exist", blockBlob.Blob.ToBlockBlobURL())
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bodyStream := downloadResponse.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20})
|
||||
downloadedData := bytes.Buffer{}
|
||||
|
||||
_, err = downloadedData.ReadFrom(bodyStream)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return downloadedData.Bytes(), nil
|
||||
}
|
||||
|
||||
func (blockBlob *BlockBlob) GetOffset(ctx context.Context) (int64, error) {
|
||||
// Get the offset of the file from azure storage
|
||||
// For the blob, show each block (ID and size) that is a committed part of it.
|
||||
var indexes []int
|
||||
var offset int64
|
||||
|
||||
getBlock, err := blockBlob.Blob.GetBlockList(ctx, azblob.BlockListAll, azblob.LeaseAccessConditions{})
|
||||
if err != nil {
|
||||
if err.(azblob.StorageError).ServiceCode() == azblob.ServiceCodeBlobNotFound {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Need committed blocks to be added to offset to know how big the file really is
|
||||
for _, block := range getBlock.CommittedBlocks {
|
||||
offset += block.Size
|
||||
indexes = append(indexes, blockIDBase64ToInt(block.Name))
|
||||
}
|
||||
|
||||
// Need to get the uncommitted blocks so that we can commit them
|
||||
for _, block := range getBlock.UncommittedBlocks {
|
||||
offset += block.Size
|
||||
indexes = append(indexes, blockIDBase64ToInt(block.Name))
|
||||
}
|
||||
|
||||
// Sort the block IDs in ascending order. This is required as Azure returns the block lists alphabetically
|
||||
// and we store the indexes as base64 encoded ints.
|
||||
sort.Ints(indexes)
|
||||
blockBlob.Indexes = indexes
|
||||
|
||||
return offset, nil
|
||||
}
|
||||
|
||||
// After all the blocks have been uploaded, we commit the unstaged blocks by sending a Block List
|
||||
func (blockBlob *BlockBlob) Commit(ctx context.Context) error {
|
||||
base64BlockIDs := make([]string, len(blockBlob.Indexes))
|
||||
for index, id := range blockBlob.Indexes {
|
||||
base64BlockIDs[index] = blockIDIntToBase64(id)
|
||||
}
|
||||
|
||||
_, err := blockBlob.Blob.CommitBlockList(ctx, base64BlockIDs, azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{}, blockBlob.AccessTier, nil, azblob.ClientProvidedKeyOptions{})
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete the infoBlob from Azure Blob Storage
|
||||
func (infoBlob *InfoBlob) Delete(ctx context.Context) error {
|
||||
_, err := infoBlob.Blob.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{})
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload the infoBlob to Azure Blob Storage
|
||||
// Because the info file is presumed to be smaller than azblob.BlockBlobMaxUploadBlobBytes (256MiB), we can upload it all in one go
|
||||
// New uploaded data will create a new, or overwrite the existing block blob
|
||||
func (infoBlob *InfoBlob) Upload(ctx context.Context, body io.ReadSeeker) error {
|
||||
_, err := infoBlob.Blob.Upload(ctx, body, azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{}, azblob.DefaultAccessTier, nil, azblob.ClientProvidedKeyOptions{})
|
||||
return err
|
||||
}
|
||||
|
||||
// Download the infoBlob from Azure Blob Storage
|
||||
func (infoBlob *InfoBlob) Download(ctx context.Context) ([]byte, error) {
|
||||
downloadResponse, err := infoBlob.Blob.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
|
||||
|
||||
// If the file does not exist, it will not return an error, but a 404 status and body
|
||||
if downloadResponse != nil && downloadResponse.StatusCode() == 404 {
|
||||
return nil, fmt.Errorf("File %s does not exist", infoBlob.Blob.ToBlockBlobURL())
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bodyStream := downloadResponse.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20})
|
||||
downloadedData := bytes.Buffer{}
|
||||
|
||||
_, err = downloadedData.ReadFrom(bodyStream)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return downloadedData.Bytes(), nil
|
||||
}
|
||||
|
||||
// infoBlob does not utilise offset, so just return 0, nil
|
||||
func (infoBlob *InfoBlob) GetOffset(ctx context.Context) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// infoBlob does not have uncommited blocks, so just return nil
|
||||
func (infoBlob *InfoBlob) Commit(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// === Helper Functions ===
|
||||
// These helper functions convert a binary block ID to a base-64 string and vice versa
|
||||
// NOTE: The blockID must be <= 64 bytes and ALL blockIDs for the block must be the same length
|
||||
func blockIDBinaryToBase64(blockID []byte) string {
|
||||
return base64.StdEncoding.EncodeToString(blockID)
|
||||
}
|
||||
|
||||
func blockIDBase64ToBinary(blockID string) []byte {
|
||||
binary, _ := base64.StdEncoding.DecodeString(blockID)
|
||||
return binary
|
||||
}
|
||||
|
||||
// These helper functions convert an int block ID to a base-64 string and vice versa
|
||||
func blockIDIntToBase64(blockID int) string {
|
||||
binaryBlockID := (&[4]byte{})[:] // All block IDs are 4 bytes long
|
||||
binary.LittleEndian.PutUint32(binaryBlockID, uint32(blockID))
|
||||
return blockIDBinaryToBase64(binaryBlockID)
|
||||
}
|
||||
|
||||
func blockIDBase64ToInt(blockID string) int {
|
||||
blockIDBase64ToBinary(blockID)
|
||||
return int(binary.LittleEndian.Uint32(blockIDBase64ToBinary(blockID)))
|
||||
}
|
||||
|
|
@ -0,0 +1,232 @@
|
|||
package azurestore
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/tus/tusd/internal/uid"
|
||||
"github.com/tus/tusd/pkg/handler"
|
||||
)
|
||||
|
||||
type AzureStore struct {
|
||||
Service AzService
|
||||
ObjectPrefix string
|
||||
Container string
|
||||
}
|
||||
|
||||
type AzUpload struct {
|
||||
ID string
|
||||
InfoBlob AzBlob
|
||||
BlockBlob AzBlob
|
||||
InfoHandler *handler.FileInfo
|
||||
}
|
||||
|
||||
func New(service AzService) *AzureStore {
|
||||
return &AzureStore{
|
||||
Service: service,
|
||||
}
|
||||
}
|
||||
|
||||
// UseIn sets this store as the core data store in the passed composer and adds
|
||||
// all possible extension to it.
|
||||
func (store AzureStore) UseIn(composer *handler.StoreComposer) {
|
||||
composer.UseCore(store)
|
||||
composer.UseTerminater(store)
|
||||
composer.UseLengthDeferrer(store)
|
||||
}
|
||||
|
||||
func (store AzureStore) NewUpload(ctx context.Context, info handler.FileInfo) (handler.Upload, error) {
|
||||
if info.ID == "" {
|
||||
info.ID = uid.Uid()
|
||||
}
|
||||
|
||||
if info.Size > MaxBlockBlobSize {
|
||||
return nil, fmt.Errorf("azurestore: max upload of %v bytes exceeded MaxBlockBlobSize of %v bytes",
|
||||
info.Size, MaxBlockBlobSize)
|
||||
}
|
||||
|
||||
blockBlob, err := store.Service.NewBlob(ctx, store.keyWithPrefix(info.ID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
infoFile := store.keyWithPrefix(store.infoPath(info.ID))
|
||||
infoBlob, err := store.Service.NewBlob(ctx, infoFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info.Storage = map[string]string{
|
||||
"Type": "azurestore",
|
||||
"Container": store.Container,
|
||||
"Key": store.keyWithPrefix(info.ID),
|
||||
}
|
||||
|
||||
azUpload := &AzUpload{
|
||||
ID: info.ID,
|
||||
InfoHandler: &info,
|
||||
InfoBlob: infoBlob,
|
||||
BlockBlob: blockBlob,
|
||||
}
|
||||
|
||||
err = azUpload.writeInfo(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("azurestore: unable to create InfoHandler file:\n%s", err)
|
||||
}
|
||||
|
||||
return azUpload, nil
|
||||
}
|
||||
|
||||
func (store AzureStore) GetUpload(ctx context.Context, id string) (handle handler.Upload, err error) {
|
||||
info := handler.FileInfo{}
|
||||
infoFile := store.keyWithPrefix(store.infoPath(id))
|
||||
infoBlob, err := store.Service.NewBlob(ctx, infoFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Download the info file from Azure Storage
|
||||
data, err := infoBlob.Download(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info.Size > MaxBlockBlobSize {
|
||||
return nil, fmt.Errorf("azurestore: max upload of %v bytes exceeded MaxBlockBlobSize of %v bytes",
|
||||
info.Size, MaxBlockBlobSize)
|
||||
}
|
||||
|
||||
blockBlob, err := store.Service.NewBlob(ctx, store.keyWithPrefix(info.ID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
offset, err := blockBlob.GetOffset(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info.Offset = offset
|
||||
|
||||
return &AzUpload{
|
||||
ID: id,
|
||||
InfoHandler: &info,
|
||||
InfoBlob: infoBlob,
|
||||
BlockBlob: blockBlob,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (store AzureStore) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload {
|
||||
return upload.(*AzUpload)
|
||||
}
|
||||
|
||||
func (store AzureStore) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload {
|
||||
return upload.(*AzUpload)
|
||||
}
|
||||
|
||||
func (upload *AzUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) {
|
||||
r := bufio.NewReader(src)
|
||||
buf := new(bytes.Buffer)
|
||||
n, err := r.WriteTo(buf)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
chunkSize := int64(binary.Size(buf.Bytes()))
|
||||
if chunkSize > MaxBlockBlobChunkSize {
|
||||
return 0, fmt.Errorf("azurestore: Chunk of size %v too large. Max chunk size is %v", chunkSize, MaxBlockBlobChunkSize)
|
||||
}
|
||||
|
||||
re := bytes.NewReader(buf.Bytes())
|
||||
err = upload.BlockBlob.Upload(ctx, re)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
upload.InfoHandler.Offset += n
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (upload *AzUpload) GetInfo(ctx context.Context) (handler.FileInfo, error) {
|
||||
info := handler.FileInfo{}
|
||||
|
||||
if upload.InfoHandler != nil {
|
||||
return *upload.InfoHandler, nil
|
||||
}
|
||||
|
||||
data, err := upload.InfoBlob.Download(ctx)
|
||||
if err != nil {
|
||||
return info, err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &info); err != nil {
|
||||
return info, err
|
||||
}
|
||||
|
||||
upload.InfoHandler = &info
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// Get the uploaded file from the Azure storage
|
||||
func (upload *AzUpload) GetReader(ctx context.Context) (io.Reader, error) {
|
||||
b, err := upload.BlockBlob.Download(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.NewReader(b), nil
|
||||
}
|
||||
|
||||
// Finish the file upload and commit the block list
|
||||
func (upload *AzUpload) FinishUpload(ctx context.Context) error {
|
||||
return upload.BlockBlob.Commit(ctx)
|
||||
}
|
||||
|
||||
func (upload *AzUpload) Terminate(ctx context.Context) error {
|
||||
// Delete info file
|
||||
err := upload.InfoBlob.Delete(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete file
|
||||
return upload.BlockBlob.Delete(ctx)
|
||||
}
|
||||
|
||||
func (upload *AzUpload) DeclareLength(ctx context.Context, length int64) error {
|
||||
upload.InfoHandler.Size = length
|
||||
upload.InfoHandler.SizeIsDeferred = false
|
||||
return upload.writeInfo(ctx)
|
||||
}
|
||||
|
||||
func (store AzureStore) infoPath(id string) string {
|
||||
return id + InfoBlobSuffix
|
||||
}
|
||||
|
||||
func (upload *AzUpload) writeInfo(ctx context.Context) error {
|
||||
data, err := json.Marshal(upload.InfoHandler)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(data)
|
||||
return upload.InfoBlob.Upload(ctx, reader)
|
||||
}
|
||||
|
||||
func (store *AzureStore) keyWithPrefix(key string) string {
|
||||
prefix := store.ObjectPrefix
|
||||
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
||||
prefix += "/"
|
||||
}
|
||||
|
||||
return prefix + key
|
||||
}
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
// Code generated by MockGen. DO NOT EDIT.
|
||||
// Source: github.com/tus/tusd/pkg/azurestore (interfaces: AzService,AzBlob)
|
||||
|
||||
// Package azurestore_test is a generated GoMock package.
|
||||
package azurestore_test
|
||||
|
||||
import (
|
||||
context "context"
|
||||
gomock "github.com/golang/mock/gomock"
|
||||
azurestore "github.com/tus/tusd/pkg/azurestore"
|
||||
io "io"
|
||||
reflect "reflect"
|
||||
)
|
||||
|
||||
// MockAzService is a mock of AzService interface
|
||||
type MockAzService struct {
|
||||
ctrl *gomock.Controller
|
||||
recorder *MockAzServiceMockRecorder
|
||||
}
|
||||
|
||||
// MockAzServiceMockRecorder is the mock recorder for MockAzService
|
||||
type MockAzServiceMockRecorder struct {
|
||||
mock *MockAzService
|
||||
}
|
||||
|
||||
// NewMockAzService creates a new mock instance
|
||||
func NewMockAzService(ctrl *gomock.Controller) *MockAzService {
|
||||
mock := &MockAzService{ctrl: ctrl}
|
||||
mock.recorder = &MockAzServiceMockRecorder{mock}
|
||||
return mock
|
||||
}
|
||||
|
||||
// EXPECT returns an object that allows the caller to indicate expected use
|
||||
func (m *MockAzService) EXPECT() *MockAzServiceMockRecorder {
|
||||
return m.recorder
|
||||
}
|
||||
|
||||
// NewBlob mocks base method
|
||||
func (m *MockAzService) NewBlob(arg0 context.Context, arg1 string) (azurestore.AzBlob, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "NewBlob", arg0, arg1)
|
||||
ret0, _ := ret[0].(azurestore.AzBlob)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// NewBlob indicates an expected call of NewBlob
|
||||
func (mr *MockAzServiceMockRecorder) NewBlob(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewBlob", reflect.TypeOf((*MockAzService)(nil).NewBlob), arg0, arg1)
|
||||
}
|
||||
|
||||
// MockAzBlob is a mock of AzBlob interface
|
||||
type MockAzBlob struct {
|
||||
ctrl *gomock.Controller
|
||||
recorder *MockAzBlobMockRecorder
|
||||
}
|
||||
|
||||
// MockAzBlobMockRecorder is the mock recorder for MockAzBlob
|
||||
type MockAzBlobMockRecorder struct {
|
||||
mock *MockAzBlob
|
||||
}
|
||||
|
||||
// NewMockAzBlob creates a new mock instance
|
||||
func NewMockAzBlob(ctrl *gomock.Controller) *MockAzBlob {
|
||||
mock := &MockAzBlob{ctrl: ctrl}
|
||||
mock.recorder = &MockAzBlobMockRecorder{mock}
|
||||
return mock
|
||||
}
|
||||
|
||||
// EXPECT returns an object that allows the caller to indicate expected use
|
||||
func (m *MockAzBlob) EXPECT() *MockAzBlobMockRecorder {
|
||||
return m.recorder
|
||||
}
|
||||
|
||||
// Commit mocks base method
|
||||
func (m *MockAzBlob) Commit(arg0 context.Context) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "Commit", arg0)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// Commit indicates an expected call of Commit
|
||||
func (mr *MockAzBlobMockRecorder) Commit(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Commit", reflect.TypeOf((*MockAzBlob)(nil).Commit), arg0)
|
||||
}
|
||||
|
||||
// Delete mocks base method
|
||||
func (m *MockAzBlob) Delete(arg0 context.Context) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "Delete", arg0)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// Delete indicates an expected call of Delete
|
||||
func (mr *MockAzBlobMockRecorder) Delete(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockAzBlob)(nil).Delete), arg0)
|
||||
}
|
||||
|
||||
// Download mocks base method
|
||||
func (m *MockAzBlob) Download(arg0 context.Context) ([]byte, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "Download", arg0)
|
||||
ret0, _ := ret[0].([]byte)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// Download indicates an expected call of Download
|
||||
func (mr *MockAzBlobMockRecorder) Download(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Download", reflect.TypeOf((*MockAzBlob)(nil).Download), arg0)
|
||||
}
|
||||
|
||||
// GetOffset mocks base method
|
||||
func (m *MockAzBlob) GetOffset(arg0 context.Context) (int64, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetOffset", arg0)
|
||||
ret0, _ := ret[0].(int64)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetOffset indicates an expected call of GetOffset
|
||||
func (mr *MockAzBlobMockRecorder) GetOffset(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOffset", reflect.TypeOf((*MockAzBlob)(nil).GetOffset), arg0)
|
||||
}
|
||||
|
||||
// Upload mocks base method
|
||||
func (m *MockAzBlob) Upload(arg0 context.Context, arg1 io.ReadSeeker) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "Upload", arg0, arg1)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// Upload indicates an expected call of Upload
|
||||
func (mr *MockAzBlobMockRecorder) Upload(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upload", reflect.TypeOf((*MockAzBlob)(nil).Upload), arg0, arg1)
|
||||
}
|
||||
|
|
@ -0,0 +1,426 @@
|
|||
package azurestore_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/azure-storage-blob-go/azblob"
|
||||
"github.com/golang/mock/gomock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/tus/tusd/pkg/azurestore"
|
||||
"github.com/tus/tusd/pkg/handler"
|
||||
)
|
||||
|
||||
//go:generate mockgen -destination=./azurestore_mock_test.go -package=azurestore_test github.com/tus/tusd/pkg/azurestore AzService,AzBlob
|
||||
|
||||
// Test interface implementations
|
||||
var _ handler.DataStore = azurestore.AzureStore{}
|
||||
var _ handler.TerminaterDataStore = azurestore.AzureStore{}
|
||||
var _ handler.LengthDeferrerDataStore = azurestore.AzureStore{}
|
||||
|
||||
const mockID = "123456789abcdefghijklmnopqrstuvwxyz"
|
||||
const mockContainer = "tusd"
|
||||
const mockSize int64 = 4096
|
||||
const mockReaderData = "Hello World"
|
||||
|
||||
var mockTusdInfo = handler.FileInfo{
|
||||
ID: mockID,
|
||||
Size: mockSize,
|
||||
MetaData: map[string]string{
|
||||
"foo": "bar",
|
||||
},
|
||||
Storage: map[string]string{
|
||||
"Type": "azurestore",
|
||||
"Container": mockContainer,
|
||||
"Key": mockID,
|
||||
},
|
||||
}
|
||||
|
||||
func TestNewUpload(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
assert := assert.New(t)
|
||||
ctx := context.Background()
|
||||
|
||||
service := NewMockAzService(mockCtrl)
|
||||
store := azurestore.New(service)
|
||||
store.Container = mockContainer
|
||||
|
||||
infoBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(infoBlob)
|
||||
|
||||
data, err := json.Marshal(mockTusdInfo)
|
||||
assert.Nil(err)
|
||||
|
||||
r := bytes.NewReader(data)
|
||||
|
||||
gomock.InOrder(
|
||||
service.EXPECT().NewBlob(ctx, mockID).Return(NewMockAzBlob(mockCtrl), nil).Times(1),
|
||||
service.EXPECT().NewBlob(ctx, mockID+".info").Return(infoBlob, nil).Times(1),
|
||||
infoBlob.EXPECT().Upload(ctx, r).Return(nil).Times(1),
|
||||
)
|
||||
|
||||
upload, err := store.NewUpload(context.Background(), mockTusdInfo)
|
||||
assert.Nil(err)
|
||||
assert.NotNil(upload)
|
||||
}
|
||||
|
||||
func TestNewUploadWithPrefix(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
assert := assert.New(t)
|
||||
ctx := context.Background()
|
||||
|
||||
objectPrefix := "/path/to/file/"
|
||||
|
||||
service := NewMockAzService(mockCtrl)
|
||||
store := azurestore.New(service)
|
||||
store.Container = mockContainer
|
||||
store.ObjectPrefix = objectPrefix
|
||||
|
||||
infoBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(infoBlob)
|
||||
|
||||
info := mockTusdInfo
|
||||
info.Storage = map[string]string{
|
||||
"Type": "azurestore",
|
||||
"Container": mockContainer,
|
||||
"Key": objectPrefix + mockID,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(info)
|
||||
assert.Nil(err)
|
||||
|
||||
r := bytes.NewReader(data)
|
||||
|
||||
gomock.InOrder(
|
||||
service.EXPECT().NewBlob(ctx, objectPrefix+mockID).Return(NewMockAzBlob(mockCtrl), nil).Times(1),
|
||||
service.EXPECT().NewBlob(ctx, objectPrefix+mockID+".info").Return(infoBlob, nil).Times(1),
|
||||
infoBlob.EXPECT().Upload(ctx, r).Return(nil).Times(1),
|
||||
)
|
||||
|
||||
upload, err := store.NewUpload(context.Background(), mockTusdInfo)
|
||||
assert.Nil(err)
|
||||
assert.NotNil(upload)
|
||||
}
|
||||
|
||||
func TestNewUploadTooLargeBlob(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
assert := assert.New(t)
|
||||
ctx := context.Background()
|
||||
|
||||
service := NewMockAzService(mockCtrl)
|
||||
store := azurestore.New(service)
|
||||
store.Container = mockContainer
|
||||
|
||||
infoBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(infoBlob)
|
||||
|
||||
info := mockTusdInfo
|
||||
info.Size = azurestore.MaxBlockBlobSize + 1
|
||||
|
||||
upload, err := store.NewUpload(ctx, info)
|
||||
assert.Nil(upload)
|
||||
assert.NotNil(err)
|
||||
assert.Contains(err.Error(), "exceeded MaxBlockBlobSize")
|
||||
assert.Contains(err.Error(), "209715200000001")
|
||||
}
|
||||
|
||||
func TestGetUpload(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
service := NewMockAzService(mockCtrl)
|
||||
store := azurestore.New(service)
|
||||
store.Container = mockContainer
|
||||
|
||||
blockBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(blockBlob)
|
||||
|
||||
infoBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(infoBlob)
|
||||
|
||||
data, err := json.Marshal(mockTusdInfo)
|
||||
assert.Nil(err)
|
||||
|
||||
gomock.InOrder(
|
||||
service.EXPECT().NewBlob(ctx, mockID+".info").Return(infoBlob, nil).Times(1),
|
||||
infoBlob.EXPECT().Download(ctx).Return(data, nil).Times(1),
|
||||
service.EXPECT().NewBlob(ctx, mockID).Return(blockBlob, nil).Times(1),
|
||||
blockBlob.EXPECT().GetOffset(ctx).Return(int64(0), nil).Times(1),
|
||||
)
|
||||
|
||||
upload, err := store.GetUpload(ctx, mockID)
|
||||
assert.Nil(err)
|
||||
|
||||
info, err := upload.GetInfo(ctx)
|
||||
assert.Nil(err)
|
||||
assert.NotNil(info)
|
||||
cancel()
|
||||
}
|
||||
|
||||
func TestGetUploadTooLargeBlob(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
service := NewMockAzService(mockCtrl)
|
||||
store := azurestore.New(service)
|
||||
store.Container = mockContainer
|
||||
|
||||
infoBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(infoBlob)
|
||||
|
||||
info := mockTusdInfo
|
||||
info.Size = azurestore.MaxBlockBlobSize + 1
|
||||
data, err := json.Marshal(info)
|
||||
assert.Nil(err)
|
||||
|
||||
gomock.InOrder(
|
||||
service.EXPECT().NewBlob(ctx, mockID+".info").Return(infoBlob, nil).Times(1),
|
||||
infoBlob.EXPECT().Download(ctx).Return(data, nil).Times(1),
|
||||
)
|
||||
|
||||
upload, err := store.GetUpload(ctx, mockID)
|
||||
assert.Nil(upload)
|
||||
assert.NotNil(err)
|
||||
assert.Contains(err.Error(), "exceeded MaxBlockBlobSize")
|
||||
assert.Contains(err.Error(), "209715200000001")
|
||||
cancel()
|
||||
}
|
||||
|
||||
func TestGetUploadNotFound(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
assert := assert.New(t)
|
||||
|
||||
service := NewMockAzService(mockCtrl)
|
||||
store := azurestore.New(service)
|
||||
store.Container = mockContainer
|
||||
|
||||
infoBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(infoBlob)
|
||||
|
||||
ctx := context.Background()
|
||||
gomock.InOrder(
|
||||
service.EXPECT().NewBlob(ctx, mockID+".info").Return(infoBlob, nil).Times(1),
|
||||
infoBlob.EXPECT().Download(ctx).Return(nil, errors.New(string(azblob.StorageErrorCodeBlobNotFound))).Times(1),
|
||||
)
|
||||
|
||||
_, err := store.GetUpload(context.Background(), mockID)
|
||||
assert.NotNil(err)
|
||||
assert.Equal(err.Error(), "BlobNotFound")
|
||||
}
|
||||
|
||||
func TestGetReader(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
service := NewMockAzService(mockCtrl)
|
||||
store := azurestore.New(service)
|
||||
store.Container = mockContainer
|
||||
|
||||
blockBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(blockBlob)
|
||||
|
||||
infoBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(infoBlob)
|
||||
|
||||
data, err := json.Marshal(mockTusdInfo)
|
||||
assert.Nil(err)
|
||||
|
||||
gomock.InOrder(
|
||||
service.EXPECT().NewBlob(ctx, mockID+".info").Return(infoBlob, nil).Times(1),
|
||||
infoBlob.EXPECT().Download(ctx).Return(data, nil).Times(1),
|
||||
service.EXPECT().NewBlob(ctx, mockID).Return(blockBlob, nil).Times(1),
|
||||
blockBlob.EXPECT().GetOffset(ctx).Return(int64(0), nil).Times(1),
|
||||
blockBlob.EXPECT().Download(ctx).Return([]byte(mockReaderData), nil).Times(1),
|
||||
)
|
||||
|
||||
upload, err := store.GetUpload(ctx, mockID)
|
||||
assert.Nil(err)
|
||||
|
||||
reader, err := upload.GetReader(ctx)
|
||||
assert.Nil(err)
|
||||
assert.NotNil(reader)
|
||||
cancel()
|
||||
}
|
||||
|
||||
func TestWriteChunk(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
service := NewMockAzService(mockCtrl)
|
||||
store := azurestore.New(service)
|
||||
store.Container = mockContainer
|
||||
|
||||
blockBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(blockBlob)
|
||||
|
||||
infoBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(infoBlob)
|
||||
|
||||
data, err := json.Marshal(mockTusdInfo)
|
||||
assert.Nil(err)
|
||||
|
||||
var offset int64 = mockSize / 2
|
||||
|
||||
gomock.InOrder(
|
||||
service.EXPECT().NewBlob(ctx, mockID+".info").Return(infoBlob, nil).Times(1),
|
||||
infoBlob.EXPECT().Download(ctx).Return(data, nil).Times(1),
|
||||
service.EXPECT().NewBlob(ctx, mockID).Return(blockBlob, nil).Times(1),
|
||||
blockBlob.EXPECT().GetOffset(ctx).Return(offset, nil).Times(1),
|
||||
blockBlob.EXPECT().Upload(ctx, bytes.NewReader([]byte(mockReaderData))).Return(nil).Times(1),
|
||||
)
|
||||
|
||||
upload, err := store.GetUpload(ctx, mockID)
|
||||
assert.Nil(err)
|
||||
|
||||
_, err = upload.WriteChunk(ctx, offset, bytes.NewReader([]byte(mockReaderData)))
|
||||
assert.Nil(err)
|
||||
cancel()
|
||||
}
|
||||
|
||||
func TestFinishUpload(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
service := NewMockAzService(mockCtrl)
|
||||
store := azurestore.New(service)
|
||||
store.Container = mockContainer
|
||||
|
||||
blockBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(blockBlob)
|
||||
|
||||
infoBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(infoBlob)
|
||||
|
||||
data, err := json.Marshal(mockTusdInfo)
|
||||
assert.Nil(err)
|
||||
|
||||
var offset int64 = mockSize / 2
|
||||
|
||||
gomock.InOrder(
|
||||
service.EXPECT().NewBlob(ctx, mockID+".info").Return(infoBlob, nil).Times(1),
|
||||
infoBlob.EXPECT().Download(ctx).Return(data, nil).Times(1),
|
||||
service.EXPECT().NewBlob(ctx, mockID).Return(blockBlob, nil).Times(1),
|
||||
blockBlob.EXPECT().GetOffset(ctx).Return(offset, nil).Times(1),
|
||||
blockBlob.EXPECT().Commit(ctx).Return(nil).Times(1),
|
||||
)
|
||||
|
||||
upload, err := store.GetUpload(ctx, mockID)
|
||||
assert.Nil(err)
|
||||
|
||||
err = upload.FinishUpload(ctx)
|
||||
assert.Nil(err)
|
||||
cancel()
|
||||
}
|
||||
|
||||
func TestTerminate(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
service := NewMockAzService(mockCtrl)
|
||||
store := azurestore.New(service)
|
||||
store.Container = mockContainer
|
||||
|
||||
blockBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(blockBlob)
|
||||
|
||||
infoBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(infoBlob)
|
||||
|
||||
data, err := json.Marshal(mockTusdInfo)
|
||||
assert.Nil(err)
|
||||
|
||||
gomock.InOrder(
|
||||
service.EXPECT().NewBlob(ctx, mockID+".info").Return(infoBlob, nil).Times(1),
|
||||
infoBlob.EXPECT().Download(ctx).Return(data, nil).Times(1),
|
||||
service.EXPECT().NewBlob(ctx, mockID).Return(blockBlob, nil).Times(1),
|
||||
blockBlob.EXPECT().GetOffset(ctx).Return(int64(0), nil).Times(1),
|
||||
infoBlob.EXPECT().Delete(ctx).Return(nil).Times(1),
|
||||
blockBlob.EXPECT().Delete(ctx).Return(nil).Times(1),
|
||||
)
|
||||
|
||||
upload, err := store.GetUpload(ctx, mockID)
|
||||
assert.Nil(err)
|
||||
|
||||
err = store.AsTerminatableUpload(upload).Terminate(ctx)
|
||||
assert.Nil(err)
|
||||
cancel()
|
||||
}
|
||||
|
||||
func TestDeclareLength(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
service := NewMockAzService(mockCtrl)
|
||||
store := azurestore.New(service)
|
||||
store.Container = mockContainer
|
||||
|
||||
blockBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(blockBlob)
|
||||
|
||||
infoBlob := NewMockAzBlob(mockCtrl)
|
||||
assert.NotNil(infoBlob)
|
||||
|
||||
info := mockTusdInfo
|
||||
info.Size = mockSize * 2
|
||||
|
||||
data, err := json.Marshal(info)
|
||||
assert.Nil(err)
|
||||
|
||||
r := bytes.NewReader(data)
|
||||
|
||||
gomock.InOrder(
|
||||
service.EXPECT().NewBlob(ctx, mockID+".info").Return(infoBlob, nil).Times(1),
|
||||
infoBlob.EXPECT().Download(ctx).Return(data, nil).Times(1),
|
||||
service.EXPECT().NewBlob(ctx, mockID).Return(blockBlob, nil).Times(1),
|
||||
blockBlob.EXPECT().GetOffset(ctx).Return(int64(0), nil).Times(1),
|
||||
infoBlob.EXPECT().Upload(ctx, r).Return(nil).Times(1),
|
||||
)
|
||||
|
||||
upload, err := store.GetUpload(ctx, mockID)
|
||||
assert.Nil(err)
|
||||
|
||||
err = store.AsLengthDeclarableUpload(upload).DeclareLength(ctx, mockSize*2)
|
||||
assert.Nil(err)
|
||||
|
||||
info, err = upload.GetInfo(ctx)
|
||||
assert.Nil(err)
|
||||
assert.NotNil(info)
|
||||
assert.Equal(info.Size, mockSize*2)
|
||||
|
||||
cancel()
|
||||
}
|
||||
|
|
@ -497,14 +497,16 @@ func TestPatch(t *testing.T) {
|
|||
defer ctrl.Finish()
|
||||
upload := NewMockFullUpload(ctrl)
|
||||
|
||||
// We simulate that the upload has already an offset of 10 bytes. Therefore, the progress notifications
|
||||
// must be the sum of the exisiting offset and the newly read bytes.
|
||||
gomock.InOrder(
|
||||
store.EXPECT().GetUpload(context.Background(), "yes").Return(upload, nil),
|
||||
upload.EXPECT().GetInfo(context.Background()).Return(FileInfo{
|
||||
ID: "yes",
|
||||
Offset: 0,
|
||||
Offset: 10,
|
||||
Size: 100,
|
||||
}, nil),
|
||||
upload.EXPECT().WriteChunk(context.Background(), int64(0), NewReaderMatcher("first second third")).Return(int64(18), nil),
|
||||
upload.EXPECT().WriteChunk(context.Background(), int64(10), NewReaderMatcher("first second third")).Return(int64(18), nil),
|
||||
)
|
||||
|
||||
handler, _ := NewHandler(Config{
|
||||
|
|
@ -525,7 +527,7 @@ func TestPatch(t *testing.T) {
|
|||
info := event.Upload
|
||||
a.Equal("yes", info.ID)
|
||||
a.Equal(int64(100), info.Size)
|
||||
a.Equal(int64(6), info.Offset)
|
||||
a.Equal(int64(16), info.Offset)
|
||||
|
||||
writer.Write([]byte("second "))
|
||||
writer.Write([]byte("third"))
|
||||
|
|
@ -534,7 +536,7 @@ func TestPatch(t *testing.T) {
|
|||
info = event.Upload
|
||||
a.Equal("yes", info.ID)
|
||||
a.Equal(int64(100), info.Size)
|
||||
a.Equal(int64(18), info.Offset)
|
||||
a.Equal(int64(28), info.Offset)
|
||||
|
||||
writer.Close()
|
||||
|
||||
|
|
@ -548,12 +550,12 @@ func TestPatch(t *testing.T) {
|
|||
ReqHeader: map[string]string{
|
||||
"Tus-Resumable": "1.0.0",
|
||||
"Content-Type": "application/offset+octet-stream",
|
||||
"Upload-Offset": "0",
|
||||
"Upload-Offset": "10",
|
||||
},
|
||||
ReqBody: reader,
|
||||
Code: http.StatusNoContent,
|
||||
ResHeader: map[string]string{
|
||||
"Upload-Offset": "18",
|
||||
"Upload-Offset": "28",
|
||||
},
|
||||
}).Run(handler, t)
|
||||
|
||||
|
|
|
|||
|
|
@ -986,20 +986,21 @@ func (handler *UnroutedHandler) absFileURL(r *http.Request, id string) string {
|
|||
// closed.
|
||||
func (handler *UnroutedHandler) sendProgressMessages(hook HookEvent, reader *bodyReader) chan<- struct{} {
|
||||
previousOffset := int64(0)
|
||||
originalOffset := hook.Upload.Offset
|
||||
stop := make(chan struct{}, 1)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
hook.Upload.Offset = reader.bytesRead()
|
||||
hook.Upload.Offset = originalOffset + reader.bytesRead()
|
||||
if hook.Upload.Offset != previousOffset {
|
||||
handler.UploadProgress <- hook
|
||||
previousOffset = hook.Upload.Offset
|
||||
}
|
||||
return
|
||||
case <-time.After(1 * time.Second):
|
||||
hook.Upload.Offset = reader.bytesRead()
|
||||
hook.Upload.Offset = originalOffset + reader.bytesRead()
|
||||
if hook.Upload.Offset != previousOffset {
|
||||
handler.UploadProgress <- hook
|
||||
previousOffset = hook.Upload.Offset
|
||||
|
|
|
|||
|
|
@ -662,7 +662,9 @@ func (upload s3Upload) fetchInfo(ctx context.Context) (info handler.FileInfo, pa
|
|||
// when the multipart upload has already been completed or aborted. Since
|
||||
// we already found the info object, we know that the upload has been
|
||||
// completed and therefore can ensure the the offset is the size.
|
||||
if isAwsError(err, "NoSuchUpload") {
|
||||
// AWS S3 returns NoSuchUpload, but other implementations, such as DigitalOcean
|
||||
// Spaces, can also return NoSuchKey.
|
||||
if isAwsError(err, "NoSuchUpload") || isAwsError(err, "NoSuchKey") {
|
||||
info.Offset = info.Size
|
||||
err = nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ compile linux 386
|
|||
compile linux amd64
|
||||
compile linux arm
|
||||
compile linux arm64
|
||||
compile darwin 386
|
||||
compile darwin amd64
|
||||
compile darwin arm64
|
||||
compile windows 386 .exe
|
||||
compile windows amd64 .exe
|
||||
|
||||
|
|
@ -19,8 +19,8 @@ maketar linux 386
|
|||
maketar linux amd64
|
||||
maketar linux arm
|
||||
maketar linux arm64
|
||||
makezip darwin 386
|
||||
makezip darwin amd64
|
||||
makezip darwin arm64
|
||||
makezip windows 386 .exe
|
||||
makezip windows amd64 .exe
|
||||
makedep amd64
|
||||
|
|
|
|||
Loading…
Reference in New Issue