LumeWeb
/
tusd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed edge cases; pre-cleanup

This commit is contained in:
Markus Kienast 2017-08-24 01:29:26 +02:00
parent e2e3b9ffe4
commit f059acc7cc
2 changed files with 90 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
s3store/s3store.go
View File

@ -565,27 +565,33 @@ func (store S3Store) CalcOptimalPartSize(size int64) (int64, error) {
switch { switch {
// We can only manage files up to MaxObjectSize, else we need to fail. // We can only manage files up to MaxObjectSize, else we need to fail.
case size > store.MaxObjectSize: case size > store.MaxObjectSize:
fmt.Print("0 ")
return 0, fmt.Errorf("CalcOptimalPartSize: size of %v bytes exceeds MaxObjectSize of %v bytes", size, store.MaxObjectSize) return 0, fmt.Errorf("CalcOptimalPartSize: size of %v bytes exceeds MaxObjectSize of %v bytes", size, store.MaxObjectSize)
// When upload is smaller or equal MinPartSize, we upload in just one part. // When upload is smaller or equal MinPartSize, we upload in just one part.
case size <= store.MinPartSize: case size <= store.MinPartSize:
fmt.Print("A ")
return store.MinPartSize, nil return store.MinPartSize, nil
// When we need 9999 parts or less with MinPartSize. // When we need 9999 parts or less with MinPartSize.
case size/store.MinPartSize < store.MaxMultipartParts: case size/store.MinPartSize < store.MaxMultipartParts:
fmt.Print("B ")
return store.MinPartSize, nil return store.MinPartSize, nil
// When we can divide our upload into exactly MaxMultipartParts parts with // If our upload divides up exactly into MaxMultipartParts parts with
// no bytes leftover, we will not need an spare last part. // no bytes leftover, we will not need an spare last part. So we can go with
// Also, when MaxObjectSize is equal to MaxPartSize * MaxMultipartParts // a straight division. Otherwise we would exceed MaxPartSize in a scenario,
// where MaxObjectSize is equal to MaxPartSize * MaxMultipartParts
// (which is not the case with the current AWS S3 API specification, but // (which is not the case with the current AWS S3 API specification, but
// might be in the future or with other S3-aware stores), we need this in // might be in the future or with other S3-aware stores).
// order for our Multipart-Upload to reach full MaxObjectSize.
case size%store.MaxMultipartParts == 0: case size%store.MaxMultipartParts == 0:
fmt.Print("C ")
return size / store.MaxMultipartParts, nil return size / store.MaxMultipartParts, nil
// If the last part would be larger than MaxPartSize, which is only the case // In all other cases, we need to round up to the next integer, as long as we
// when we are close to MaxObjectSize, we have to go with MaxPartSize. // stay below MaxPartSize.
case size%(store.MaxMultipartParts-1) > store.MaxPartSize: case size/store.MaxMultipartParts < store.MaxPartSize:
return store.MaxPartSize, nil fmt.Print("D ")
// In all other cases, we need a spare last piece for the remaining bytes. return size/store.MaxMultipartParts + 1, nil
// If non of the above matches, we have exceeded the MaxPartSize limit above.
default: default:
return size / (store.MaxMultipartParts - 1), nil fmt.Print("X ")
return size/store.MaxMultipartParts + 1, fmt.Errorf("CalcOptimalPartSize: to upload %v bytes optimalPartSize %v must exceed MaxPartSize %v", size, (size/store.MaxMultipartParts + 1), store.MaxPartSize)
} }
} }

118
s3store/s3store_test.go
View File

@ -3,11 +3,10 @@ package s3store_test
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"io/ioutil"
"testing"
"github.com/golang/mock/gomock" "github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"io/ioutil"
"testing"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/awserr"
@ -36,23 +35,32 @@ func TestCalcOptimalPartSize(t *testing.T) {
assert.Equal("bucket", store.Bucket) assert.Equal("bucket", store.Bucket)
assert.Equal(s3obj, store.Service) assert.Equal(s3obj, store.Service)
// If you quickly want to override the default values in this test
store.MinPartSize = 2
store.MaxPartSize = 10
store.MaxMultipartParts = 20
store.MaxObjectSize = 201
var MinPartSize = store.MinPartSize var MinPartSize = store.MinPartSize
var MaxPartSize = store.MaxPartSize var MaxPartSize = store.MaxPartSize
var MaxMultipartParts = store.MaxMultipartParts var MaxMultipartParts = store.MaxMultipartParts
var MaxObjectSize = store.MaxObjectSize var MaxObjectSize = store.MaxObjectSize
// sanity check
if MaxObjectSize > MaxPartSize*MaxMultipartParts {
t.Errorf("%v parts with %v bytes each is %v bytes, which is less than MaxObjectSize=%v.\n", MaxMultipartParts, MaxPartSize, MaxMultipartParts*MaxPartSize, MaxObjectSize)
}
var LimitedMaxPartSize = MaxObjectSize / (MaxMultipartParts - 1)
// the size of the last part, when upload has MaxObjectSize and we use
// LimitedMaxPartSize for uploading
var LastPartSize int64 = MaxObjectSize % (MaxMultipartParts - 1)
var equalparts, lastpartsize int64 var equalparts, lastpartsize int64
var err string var err string
// sanity check
if MaxObjectSize > MaxPartSize*MaxMultipartParts {
t.Errorf("MaxObjectSize %v can never be achieved, as MaxMultipartParts %v and MaxPartSize %v only allow for an upload of %v bytes total.\n", MaxObjectSize, MaxMultipartParts, MaxPartSize, MaxMultipartParts*MaxPartSize)
}
var HighestApplicablePartSize int64 = MaxObjectSize / MaxMultipartParts
if MaxObjectSize%MaxMultipartParts > 0 {
HighestApplicablePartSize++
}
var RemainderWithHighestApplicablePartSize int64 = MaxObjectSize % HighestApplicablePartSize
fmt.Println("HighestApplicablePartSize", HighestApplicablePartSize)
fmt.Println("RemainderWithHighestApplicablePartSize", RemainderWithHighestApplicablePartSize)
// some of these tests are actually duplicates, as they specify the same size // some of these tests are actually duplicates, as they specify the same size
// in bytes - two ways to describe the same thing. That is wanted, in order // in bytes - two ways to describe the same thing. That is wanted, in order
// to provide a full picture from any angle. // to provide a full picture from any angle.
@ -61,58 +69,78 @@ func TestCalcOptimalPartSize(t *testing.T) {
MinPartSize - 1, MinPartSize - 1,
MinPartSize, MinPartSize,
MinPartSize + 1, MinPartSize + 1,
MinPartSize * 9999,
MinPartSize*10000 - 1, MinPartSize*(MaxMultipartParts-1) - 1,
MinPartSize * 10000, MinPartSize * (MaxMultipartParts - 1),
MinPartSize*10000 + 1, MinPartSize*(MaxMultipartParts-1) + 1,
MinPartSize * 10001,
LimitedMaxPartSize*9999 - 1, MinPartSize*MaxMultipartParts - 1,
LimitedMaxPartSize * 9999, MinPartSize * MaxMultipartParts,
LimitedMaxPartSize*9999 + 1, MinPartSize*MaxMultipartParts + 1,
LimitedMaxPartSize*9999 + LastPartSize - 1,
LimitedMaxPartSize*9999 + LastPartSize, MinPartSize*(MaxMultipartParts+1) - 1,
LimitedMaxPartSize*9999 + LastPartSize + 1, MinPartSize * (MaxMultipartParts + 1),
MinPartSize*(MaxMultipartParts+1) + 1,
(HighestApplicablePartSize-1)*MaxMultipartParts - 1,
(HighestApplicablePartSize - 1) * MaxMultipartParts,
(HighestApplicablePartSize-1)*MaxMultipartParts + 1,
HighestApplicablePartSize*(MaxMultipartParts-1) - 1,
HighestApplicablePartSize * (MaxMultipartParts - 1),
HighestApplicablePartSize*(MaxMultipartParts-1) + 1,
HighestApplicablePartSize*(MaxMultipartParts-1) + RemainderWithHighestApplicablePartSize - 1,
HighestApplicablePartSize*(MaxMultipartParts-1) + RemainderWithHighestApplicablePartSize,
HighestApplicablePartSize*(MaxMultipartParts-1) + RemainderWithHighestApplicablePartSize + 1,
MaxObjectSize - 1, MaxObjectSize - 1,
MaxObjectSize, MaxObjectSize,
MaxObjectSize + 1, MaxObjectSize + 1,
MaxPartSize*9999 - 1,
MaxPartSize * 9999, (MaxObjectSize/MaxMultipartParts)*(MaxMultipartParts-1) - 1,
MaxPartSize*9999 + 1, (MaxObjectSize / MaxMultipartParts) * (MaxMultipartParts - 1),
MaxPartSize*10000 - 1, (MaxObjectSize/MaxMultipartParts)*(MaxMultipartParts-1) + 1,
MaxPartSize * 10000,
MaxPartSize*10000 + 1, MaxPartSize*(MaxMultipartParts-1) - 1,
MaxPartSize * (MaxMultipartParts - 1),
MaxPartSize*(MaxMultipartParts-1) + 1,
MaxPartSize*MaxMultipartParts - 1,
MaxPartSize * MaxMultipartParts,
MaxPartSize*MaxMultipartParts + 1,
} }
for index, size := range testcases { for index, size := range testcases {
optimalPartSize, calcError := store.CalcOptimalPartSize(size) optimalPartSize, calcError := store.CalcOptimalPartSize(size)
equalparts, lastpartsize = 0, 0
err = ""
if size > MaxObjectSize && calcError == nil { if size > MaxObjectSize && calcError == nil {
err += fmt.Sprintf("Testcase #%v: size=%v exceeds MaxObjectSize=%v but CalcOptimalPartSize did not throw an error\n", index, size, MaxObjectSize) err += fmt.Sprintf("Testcase #%v size %v: size exceeds MaxObjectSize=%v but no error returned\n", index, size, MaxObjectSize)
}
if optimalPartSize*(MaxMultipartParts-1) > MaxObjectSize {
err += fmt.Sprintf("Testcase #%v: optimalPartSize=%v, exceeds MaxPartSize=%v\n", index, optimalPartSize, MaxPartSize)
}
if optimalPartSize > MaxPartSize {
err += fmt.Sprintf("Testcase #%v: optimalPartSize=%v exceeds MaxPartSize=%v\n", index, optimalPartSize, MaxPartSize)
} }
if calcError == nil { if calcError == nil {
equalparts = size / optimalPartSize equalparts = size / optimalPartSize
lastpartsize = size % optimalPartSize lastpartsize = size % optimalPartSize
if optimalPartSize < MinPartSize { if optimalPartSize < MinPartSize {
err += fmt.Sprintf("Testcase #%v: optimalPartSize=%v is below MinPartSize=%v\n", index, optimalPartSize, MinPartSize) err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: optimalPartSize < MinPartSize %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MinPartSize)
} }
if optimalPartSize > MaxPartSize { if optimalPartSize > MaxPartSize {
err += fmt.Sprintf("Testcase #%v: optimalPartSize=%v is larger than MaxPartSize=%v\n", index, optimalPartSize, MaxPartSize) err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: optimalPartSize > MaxPartSize %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MaxPartSize)
}
if size%optimalPartSize == 0 && equalparts > MaxMultipartParts {
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: more parts than MaxMultipartParts %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MaxMultipartParts)
}
if size%optimalPartSize > 0 && equalparts > MaxMultipartParts-1 {
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: more parts than MaxMultipartParts %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MaxMultipartParts)
} }
if lastpartsize > MaxPartSize { if lastpartsize > MaxPartSize {
err += fmt.Sprintf("Testcase #%v: size of last part %v is exceeding MaxPartSize limit of %v\n", index, lastpartsize, MaxPartSize) err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: lastpart > MaxPartSize %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MaxPartSize)
} }
if equalparts > 10000 { if lastpartsize > optimalPartSize {
err += fmt.Sprintf("Testcase #%v: max-parts=%v exceeds limit of 10.000 parts. %v equal parts of size %v, size of last part %v\n", index, equalparts, equalparts, optimalPartSize, lastpartsize) err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: lastpart > optimalPartSize %v\n", index, size, equalparts, optimalPartSize, lastpartsize, optimalPartSize)
}
if equalparts == 10000 && lastpartsize > 0 {
err += fmt.Sprintf("Testcase #%v: max-parts=%v exceeds limit of 10.000 parts. %v equal parts of size %v, size of last part %v\n", index, equalparts+1, equalparts, optimalPartSize, lastpartsize)
} }
} }
fmt.Printf("Testcase #%v size %v, %v parts of size %v, lastpart %v\n", index, size, equalparts, optimalPartSize, lastpartsize)
if len(err) > 0 { if len(err) > 0 {
t.Errorf(err) t.Errorf(err)
} }