separated partsize calc and error handling
This commit is contained in:
parent
f059acc7cc
commit
8831a98c34
|
@ -235,7 +235,7 @@ func (store S3Store) WriteChunk(id string, offset int64, src io.Reader) (int64,
|
|||
bytesUploaded := int64(0)
|
||||
optimalPartSize, err := store.CalcOptimalPartSize(size)
|
||||
if err != nil {
|
||||
return bytesUploaded, nil
|
||||
return bytesUploaded, err
|
||||
}
|
||||
|
||||
// Get number of parts to generate next number
|
||||
|
@ -561,20 +561,16 @@ func isAwsError(err error, code string) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
func (store S3Store) CalcOptimalPartSize(size int64) (int64, error) {
|
||||
func (store S3Store) CalcOptimalPartSize(size int64) (optimalPartSize int64, err error) {
|
||||
var errorstrings []string
|
||||
|
||||
switch {
|
||||
// We can only manage files up to MaxObjectSize, else we need to fail.
|
||||
case size > store.MaxObjectSize:
|
||||
fmt.Print("0 ")
|
||||
return 0, fmt.Errorf("CalcOptimalPartSize: size of %v bytes exceeds MaxObjectSize of %v bytes", size, store.MaxObjectSize)
|
||||
// When upload is smaller or equal MinPartSize, we upload in just one part.
|
||||
case size <= store.MinPartSize:
|
||||
fmt.Print("A ")
|
||||
return store.MinPartSize, nil
|
||||
optimalPartSize = store.MinPartSize
|
||||
// When we need 9999 parts or less with MinPartSize.
|
||||
case size/store.MinPartSize < store.MaxMultipartParts:
|
||||
fmt.Print("B ")
|
||||
return store.MinPartSize, nil
|
||||
optimalPartSize = store.MinPartSize
|
||||
// If our upload divides up exactly into MaxMultipartParts parts with
|
||||
// no bytes leftover, we will not need an spare last part. So we can go with
|
||||
// a straight division. Otherwise we would exceed MaxPartSize in a scenario,
|
||||
|
@ -582,16 +578,22 @@ func (store S3Store) CalcOptimalPartSize(size int64) (int64, error) {
|
|||
// (which is not the case with the current AWS S3 API specification, but
|
||||
// might be in the future or with other S3-aware stores).
|
||||
case size%store.MaxMultipartParts == 0:
|
||||
fmt.Print("C ")
|
||||
return size / store.MaxMultipartParts, nil
|
||||
// In all other cases, we need to round up to the next integer, as long as we
|
||||
// stay below MaxPartSize.
|
||||
case size/store.MaxMultipartParts < store.MaxPartSize:
|
||||
fmt.Print("D ")
|
||||
return size/store.MaxMultipartParts + 1, nil
|
||||
// If non of the above matches, we have exceeded the MaxPartSize limit above.
|
||||
optimalPartSize = size / store.MaxMultipartParts
|
||||
// In all other cases, we need to round up to the next integer.
|
||||
default:
|
||||
fmt.Print("X ")
|
||||
return size/store.MaxMultipartParts + 1, fmt.Errorf("CalcOptimalPartSize: to upload %v bytes optimalPartSize %v must exceed MaxPartSize %v", size, (size/store.MaxMultipartParts + 1), store.MaxPartSize)
|
||||
optimalPartSize = size/store.MaxMultipartParts + 1
|
||||
}
|
||||
|
||||
// an upload larger than MaxObjectSize must throw an error
|
||||
if size > store.MaxObjectSize {
|
||||
errorstrings = append(errorstrings, fmt.Sprintf("CalcOptimalPartSize: upload size of %v bytes exceeds MaxObjectSize of %v bytes", size, store.MaxObjectSize))
|
||||
}
|
||||
// optimalPartSize must never exceed MaxPartSize
|
||||
if optimalPartSize > store.MaxPartSize {
|
||||
errorstrings = append(errorstrings, fmt.Sprintf("CalcOptimalPartSize: to upload %v bytes optimalPartSize %v must exceed MaxPartSize %v", size, optimalPartSize, store.MaxPartSize))
|
||||
}
|
||||
if len(errorstrings) > 0 {
|
||||
err = fmt.Errorf(strings.Join(errorstrings, "\n"))
|
||||
}
|
||||
return optimalPartSize, err
|
||||
}
|
||||
|
|
|
@ -36,10 +36,12 @@ func TestCalcOptimalPartSize(t *testing.T) {
|
|||
assert.Equal(s3obj, store.Service)
|
||||
|
||||
// If you quickly want to override the default values in this test
|
||||
store.MinPartSize = 2
|
||||
store.MaxPartSize = 10
|
||||
store.MaxMultipartParts = 20
|
||||
store.MaxObjectSize = 201
|
||||
/*
|
||||
store.MinPartSize = 2
|
||||
store.MaxPartSize = 10
|
||||
store.MaxMultipartParts = 20
|
||||
store.MaxObjectSize = 200
|
||||
*/
|
||||
|
||||
var MinPartSize = store.MinPartSize
|
||||
var MaxPartSize = store.MaxPartSize
|
||||
|
@ -58,9 +60,6 @@ func TestCalcOptimalPartSize(t *testing.T) {
|
|||
}
|
||||
var RemainderWithHighestApplicablePartSize int64 = MaxObjectSize % HighestApplicablePartSize
|
||||
|
||||
fmt.Println("HighestApplicablePartSize", HighestApplicablePartSize)
|
||||
fmt.Println("RemainderWithHighestApplicablePartSize", RemainderWithHighestApplicablePartSize)
|
||||
|
||||
// some of these tests are actually duplicates, as they specify the same size
|
||||
// in bytes - two ways to describe the same thing. That is wanted, in order
|
||||
// to provide a full picture from any angle.
|
||||
|
@ -112,39 +111,38 @@ func TestCalcOptimalPartSize(t *testing.T) {
|
|||
}
|
||||
|
||||
for index, size := range testcases {
|
||||
optimalPartSize, calcError := store.CalcOptimalPartSize(size)
|
||||
equalparts, lastpartsize = 0, 0
|
||||
err = ""
|
||||
optimalPartSize, calcError := store.CalcOptimalPartSize(size)
|
||||
equalparts = size / optimalPartSize
|
||||
lastpartsize = size % optimalPartSize
|
||||
if size > MaxObjectSize && calcError == nil {
|
||||
err += fmt.Sprintf("Testcase #%v size %v: size exceeds MaxObjectSize=%v but no error returned\n", index, size, MaxObjectSize)
|
||||
}
|
||||
if calcError == nil {
|
||||
equalparts = size / optimalPartSize
|
||||
lastpartsize = size % optimalPartSize
|
||||
if optimalPartSize < MinPartSize {
|
||||
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: optimalPartSize < MinPartSize %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MinPartSize)
|
||||
}
|
||||
if optimalPartSize > MaxPartSize {
|
||||
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: optimalPartSize > MaxPartSize %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MaxPartSize)
|
||||
}
|
||||
if size%optimalPartSize == 0 && equalparts > MaxMultipartParts {
|
||||
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: more parts than MaxMultipartParts %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MaxMultipartParts)
|
||||
}
|
||||
if size%optimalPartSize > 0 && equalparts > MaxMultipartParts-1 {
|
||||
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: more parts than MaxMultipartParts %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MaxMultipartParts)
|
||||
}
|
||||
if lastpartsize > MaxPartSize {
|
||||
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: lastpart > MaxPartSize %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MaxPartSize)
|
||||
}
|
||||
if lastpartsize > optimalPartSize {
|
||||
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: lastpart > optimalPartSize %v\n", index, size, equalparts, optimalPartSize, lastpartsize, optimalPartSize)
|
||||
}
|
||||
if optimalPartSize < MinPartSize {
|
||||
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: optimalPartSize < MinPartSize %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MinPartSize)
|
||||
}
|
||||
fmt.Printf("Testcase #%v size %v, %v parts of size %v, lastpart %v\n", index, size, equalparts, optimalPartSize, lastpartsize)
|
||||
if optimalPartSize > MaxPartSize && calcError == nil {
|
||||
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: optimalPartSize > MaxPartSize %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MaxPartSize)
|
||||
}
|
||||
if size%optimalPartSize == 0 && equalparts > MaxMultipartParts {
|
||||
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: more parts than MaxMultipartParts %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MaxMultipartParts)
|
||||
}
|
||||
if size%optimalPartSize > 0 && equalparts > MaxMultipartParts-1 {
|
||||
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: more parts than MaxMultipartParts %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MaxMultipartParts)
|
||||
}
|
||||
if lastpartsize > MaxPartSize {
|
||||
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: lastpart > MaxPartSize %v\n", index, size, equalparts, optimalPartSize, lastpartsize, MaxPartSize)
|
||||
}
|
||||
if lastpartsize > optimalPartSize {
|
||||
err += fmt.Sprintf("Testcase #%v size %v, %v parts of size %v, lastpart %v: lastpart > optimalPartSize %v\n", index, size, equalparts, optimalPartSize, lastpartsize, optimalPartSize)
|
||||
}
|
||||
// fmt.Printf("Testcase #%v size %v, %v parts of size %v, lastpart %v\n", index, size, equalparts, optimalPartSize, lastpartsize)
|
||||
if len(err) > 0 {
|
||||
t.Errorf(err)
|
||||
}
|
||||
}
|
||||
// fmt.Println("HighestApplicablePartSize", HighestApplicablePartSize)
|
||||
// fmt.Println("RemainderWithHighestApplicablePartSize", RemainderWithHighestApplicablePartSize)
|
||||
}
|
||||
|
||||
func TestNewUpload(t *testing.T) {
|
||||
|
|
Loading…
Reference in New Issue