tusd/pkg/s3store/calcpartsize_test.go

166 lines
6.4 KiB
Go
Raw Normal View History

s3store: calculate part size based on upload size See https://github.com/tus/tusd/issues/149 and https://github.com/tus/tusd/pull/150 for more details. Squashed commit of the following: commit 78312ab26ea7ee664038e5b5d362bd534bfe0e37 Author: Marius <maerious@gmail.com> Date: Fri Sep 1 19:49:48 2017 +0200 Correct error assertions for exceeding max part size commit 9350712c0a46651e6a7a91d8819307ba4b08ec7e Author: Marius <maerious@gmail.com> Date: Fri Sep 1 19:44:28 2017 +0200 Make CalcOptimalPartSize unexported commit 593f3b2d37d16c51f229572c1d6b39fc2a234079 Author: Marius <maerious@gmail.com> Date: Fri Sep 1 19:38:46 2017 +0200 Add more output for debugging tests commit b7193bfe67b535c9b9dd441610b41af11fe4538f Author: Marius <maerious@gmail.com> Date: Fri Sep 1 19:35:48 2017 +0200 Extract size assertions into own function commit 7521de23194652519fbbf3d61a41ef0b44b005fa Author: Marius <maerious@gmail.com> Date: Fri Sep 1 19:26:48 2017 +0200 Move tests for CalcPartSize into own file commit 6c483de7710cc119c870271ccad629c98c15c9a3 Author: Marius <maerious@gmail.com> Date: Fri Sep 1 19:13:02 2017 +0200 Use same assertions in AllUploadSizes test commit 7b0290a07e7def09ea8ed982e7817a2ea7cd468a Author: Marius <maerious@gmail.com> Date: Fri Sep 1 18:30:02 2017 +0200 Split negative test case from TestCalcOptimalPartSize into own test commit 79c0a20d7bc71b494bc0824ad2aa8879b0c2900b Merge: 5240f9b 997961f Author: Marius <maerious@gmail.com> Date: Fri Sep 1 17:32:31 2017 +0200 Merge branch 'f-s3-part-size' of https://github.com/flaneurtv/tusd into flaneurtv-f-s3-part-size commit 997961ff5cc3d59c9055e54335926929f9ed570a Author: Markus Kienast <mark@rickkiste.at> Date: Fri Sep 1 00:59:38 2017 +0200 TestNewUploadLargerMaxObjectSize commit 0831bd79f8d34c0d5d079d1750852754db36ecdf Author: Markus Kienast <mark@rickkiste.at> Date: Thu Aug 31 23:08:03 2017 +0200 fmt.Sprintf removed, range from 0 - MaxObjectSize+1 commit 1be708152409f6a918a8dee86d704d1a9f25690b Author: Markus Kienast <mark@rickkiste.at> Date: Tue Aug 29 10:23:50 2017 +0200 turn off debug mode commit be9a9bec10e0681572c6a1aa588fd158d913c008 Author: Markus Kienast <mark@rickkiste.at> Date: Tue Aug 29 10:12:20 2017 +0200 moved MaxObjectSize check to NewUpload, refined tests * moved MaxObjectSize check to NewUpload * removed MaxObjectSize check from CalcOptimalPartSize * switched to assert in tests * added TestAllPartSizes, excluded in short mode TODO: TestNewUploadLargerMaxObjectSize needs to fail if MaxObjectSize > size commit 7c22847a451c8f70ef3b312111267b699ff00693 Author: Markus Kienast <mark@rickkiste.at> Date: Sat Aug 26 12:55:07 2017 +0200 adding debug code to TestCalcOptimalPartSize commit 5240f9b549000fac34be79ddfbe6e82404387f6b Merge: 63c011e 5b116e7 Author: Marius <maerious@gmail.com> Date: Sat Aug 26 12:50:51 2017 +0200 Merge branch 'f-s3-part-size' of https://github.com/flaneurtv/tusd into flaneurtv-f-s3-part-size commit 63c011ef768db42e99004df921c2b9e5c4776fd2 Author: Marius <maerious@gmail.com> Date: Sat Aug 26 12:50:45 2017 +0200 Format s3store_test commit 5b116e70875789eff95e490353b5d232c21f8660 Author: Markus Kienast <mark@rickkiste.at> Date: Sat Aug 26 12:24:22 2017 +0200 restructuring tests to accommodate optimalPartSize of 0 commit 93134a5696e3755d35e3ab360ba393cd3e52b6f0 Author: Markus Kienast <mark@rickkiste.at> Date: Sat Aug 26 12:03:18 2017 +0200 moving MaxObjectSize check to top commit 68e6bb8c41f92e29dfbf37bcb391bc5256ebb90a Author: Markus Kienast <mark@rickkiste.at> Date: Sat Aug 26 02:31:27 2017 +0200 enhance readability, comments and errors commit 8831a98c34d0a702fe24b6450928830569dac351 Author: Markus Kienast <mark@rickkiste.at> Date: Thu Aug 24 02:27:57 2017 +0200 separated partsize calc and error handling commit f059acc7ccc443405cf9d465f619319483ed947a Author: Markus Kienast <mark@rickkiste.at> Date: Thu Aug 24 01:29:26 2017 +0200 fixed edge cases; pre-cleanup commit e2e3b9ffe4aeef70ed44bc63b6afc5e4c353c159 Author: Markus Kienast <mark@rickkiste.at> Date: Wed Aug 23 13:28:59 2017 +0200 added error, when size > MaxObjectSize; additional case in algorithm + tests; go fmt commit 381d3326cb17a173b13eced781a1ae6efc88773b Author: Markus Kienast <mark@rickkiste.at> Date: Thu Aug 17 16:32:25 2017 +0200 calculating PartSize based on size of upload simplified algorithm, respect MaxObjectSize, updated tests, go fmt commit 1ad6187d6dc199018605e3e3a7d9d4c7d2c37cf8 Author: koenvo <info@koenvossen.nl> Date: Thu Aug 17 21:31:37 2017 +0200 Take IsTruncated field of S3 ListParts API response into account (#148) * Take IsTruncated field of S3 ListParts API response into account * Rename s3store.ListParts to ListAllParts * Use proper formatting + make listAllParts private + test listAllParts through TestGetInfo * Update TestFinishUpload to also test paged ListParts response
2017-09-03 08:57:06 +00:00
package s3store
import (
"fmt"
"testing"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
)
const enableTestDebugOutput = false
func assertCalculatedPartSize(store S3Store, assert *assert.Assertions, size int64) {
optimalPartSize, err := store.calcOptimalPartSize(size)
assert.Nil(err, "Size %d, no error should be returned.\n", size)
// Number of parts with the same size
equalparts := size / optimalPartSize
// Size of the last part (or 0 if no spare part is needed)
lastpartSize := size % optimalPartSize
prelude := fmt.Sprintf("Size %d, %d parts of size %d, lastpart %d: ", size, equalparts, optimalPartSize, lastpartSize)
assert.False(optimalPartSize < store.MinPartSize, prelude+"optimalPartSize < MinPartSize %d.\n", store.MinPartSize)
assert.False(optimalPartSize > store.MaxPartSize, prelude+"optimalPartSize > MaxPartSize %d.\n", store.MaxPartSize)
assert.False(lastpartSize == 0 && equalparts > store.MaxMultipartParts, prelude+"more parts than MaxMultipartParts %d.\n", store.MaxMultipartParts)
assert.False(lastpartSize > 0 && equalparts > store.MaxMultipartParts-1, prelude+"more parts than MaxMultipartParts %d.\n", store.MaxMultipartParts)
assert.False(lastpartSize > store.MaxPartSize, prelude+"lastpart > MaxPartSize %d.\n", store.MaxPartSize)
assert.False(lastpartSize > optimalPartSize, prelude+"lastpart > optimalPartSize %d.\n", optimalPartSize)
assert.True(size <= optimalPartSize*store.MaxMultipartParts, prelude+"upload does not fit in %d parts.\n", store.MaxMultipartParts)
if enableTestDebugOutput {
fmt.Printf(prelude+"does exceed MaxObjectSize: %t.\n", size > store.MaxObjectSize)
}
}
func TestCalcOptimalPartSize(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
assert := assert.New(t)
s3obj := NewMockS3API(mockCtrl)
store := New("bucket", s3obj)
// If you quickly want to override the default values in this test
/*
store.MinPartSize = 2
store.MaxPartSize = 10
store.MaxMultipartParts = 20
store.MaxObjectSize = 200
*/
// sanity check
if store.MaxObjectSize > store.MaxPartSize*store.MaxMultipartParts {
t.Errorf("MaxObjectSize %v can never be achieved, as MaxMultipartParts %v and MaxPartSize %v only allow for an upload of %v bytes total.\n", store.MaxObjectSize, store.MaxMultipartParts, store.MaxPartSize, store.MaxMultipartParts*store.MaxPartSize)
}
HighestApplicablePartSize := store.MaxObjectSize / store.MaxMultipartParts
if store.MaxObjectSize%store.MaxMultipartParts > 0 {
HighestApplicablePartSize++
}
RemainderWithHighestApplicablePartSize := store.MaxObjectSize % HighestApplicablePartSize
// some of these tests are actually duplicates, as they specify the same size
// in bytes - two ways to describe the same thing. That is wanted, in order
// to provide a full picture from any angle.
testcases := []int64{
0,
1,
store.MinPartSize - 1,
store.MinPartSize,
store.MinPartSize + 1,
store.MinPartSize*(store.MaxMultipartParts-1) - 1,
store.MinPartSize * (store.MaxMultipartParts - 1),
store.MinPartSize*(store.MaxMultipartParts-1) + 1,
store.MinPartSize*store.MaxMultipartParts - 1,
store.MinPartSize * store.MaxMultipartParts,
store.MinPartSize*store.MaxMultipartParts + 1,
store.MinPartSize*(store.MaxMultipartParts+1) - 1,
store.MinPartSize * (store.MaxMultipartParts + 1),
store.MinPartSize*(store.MaxMultipartParts+1) + 1,
(HighestApplicablePartSize-1)*store.MaxMultipartParts - 1,
(HighestApplicablePartSize - 1) * store.MaxMultipartParts,
(HighestApplicablePartSize-1)*store.MaxMultipartParts + 1,
HighestApplicablePartSize*(store.MaxMultipartParts-1) - 1,
HighestApplicablePartSize * (store.MaxMultipartParts - 1),
HighestApplicablePartSize*(store.MaxMultipartParts-1) + 1,
HighestApplicablePartSize*(store.MaxMultipartParts-1) + RemainderWithHighestApplicablePartSize - 1,
HighestApplicablePartSize*(store.MaxMultipartParts-1) + RemainderWithHighestApplicablePartSize,
HighestApplicablePartSize*(store.MaxMultipartParts-1) + RemainderWithHighestApplicablePartSize + 1,
store.MaxObjectSize - 1,
store.MaxObjectSize,
store.MaxObjectSize + 1,
(store.MaxObjectSize/store.MaxMultipartParts)*(store.MaxMultipartParts-1) - 1,
(store.MaxObjectSize / store.MaxMultipartParts) * (store.MaxMultipartParts - 1),
(store.MaxObjectSize/store.MaxMultipartParts)*(store.MaxMultipartParts-1) + 1,
store.MaxPartSize*(store.MaxMultipartParts-1) - 1,
store.MaxPartSize * (store.MaxMultipartParts - 1),
store.MaxPartSize*(store.MaxMultipartParts-1) + 1,
store.MaxPartSize*store.MaxMultipartParts - 1,
store.MaxPartSize * store.MaxMultipartParts,
// We cannot calculate a part size for store.MaxPartSize*store.MaxMultipartParts + 1
// This case is tested in TestCalcOptimalPartSize_ExceedingMaxPartSize
}
for _, size := range testcases {
assertCalculatedPartSize(store, assert, size)
}
if enableTestDebugOutput {
fmt.Println("HighestApplicablePartSize", HighestApplicablePartSize)
fmt.Println("RemainderWithHighestApplicablePartSize", RemainderWithHighestApplicablePartSize)
}
}
func TestCalcOptimalPartSize_AllUploadSizes(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
assert := assert.New(t)
s3obj := NewMockS3API(mockCtrl)
store := New("bucket", s3obj)
store.MinPartSize = 5
store.MaxPartSize = 5 * 1024
store.MaxMultipartParts = 1000
store.MaxObjectSize = store.MaxPartSize * store.MaxMultipartParts
// sanity check
if store.MaxObjectSize > store.MaxPartSize*store.MaxMultipartParts {
t.Errorf("MaxObjectSize %v can never be achieved, as MaxMultipartParts %v and MaxPartSize %v only allow for an upload of %v bytes total.\n", store.MaxObjectSize, store.MaxMultipartParts, store.MaxPartSize, store.MaxMultipartParts*store.MaxPartSize)
}
for size := int64(0); size <= store.MaxObjectSize; size++ {
assertCalculatedPartSize(store, assert, size)
}
}
func TestCalcOptimalPartSize_ExceedingMaxPartSize(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
assert := assert.New(t)
s3obj := NewMockS3API(mockCtrl)
store := New("bucket", s3obj)
size := store.MaxPartSize*store.MaxMultipartParts + 1
optimalPartSize, err := store.calcOptimalPartSize(size)
assert.NotNil(err)
assert.EqualError(err, fmt.Sprintf("calcOptimalPartSize: to upload %v bytes optimalPartSize %v must exceed MaxPartSize %v", size, optimalPartSize, store.MaxPartSize))
}