From 973c40afb45e01374f92c1e1a9ecca431ec52767 Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Mon, 11 Mar 2024 07:50:07 -0400 Subject: [PATCH] refactor: switch to using decimals in db, a fork of the siacentral api to return decimals, strings for currency settings, and rats to do big number math safety --- config/sia.go | 63 ++++++++++++++--------- db/models/sc_price_history.go | 6 ++- go.mod | 3 +- go.sum | 20 +------- renter/price_tracker.go | 95 ++++++++++++++++++++++------------- renter/renter.go | 2 +- 6 files changed, 108 insertions(+), 81 deletions(-) diff --git a/config/sia.go b/config/sia.go index 739fb86..ec72ac0 100644 --- a/config/sia.go +++ b/config/sia.go @@ -9,14 +9,14 @@ var _ Validator = (*SiaConfig)(nil) var _ Defaults = (*SiaConfig)(nil) type SiaConfig struct { - Key string `mapstructure:"key"` - URL string `mapstructure:"url"` - PriceHistoryDays uint64 `mapstructure:"price_history_days"` - MaxUploadPrice float64 `mapstructure:"max_upload_price"` - MaxDownloadPrice float64 `mapstructure:"max_download_price"` - MaxStoragePrice float64 `mapstructure:"max_storage_price"` - MaxContractSCPrice float64 `mapstructure:"max_contract_sc_price"` - MaxRPCSCPrice string `mapstructure:"max_rpc_sc_price"` + Key string `mapstructure:"key"` + URL string `mapstructure:"url"` + PriceHistoryDays uint64 `mapstructure:"price_history_days"` + MaxUploadPrice string `mapstructure:"max_upload_price"` + MaxDownloadPrice string `mapstructure:"max_download_price"` + MaxStoragePrice string `mapstructure:"max_storage_price"` + MaxContractSCPrice string `mapstructure:"max_contract_sc_price"` + MaxRPCSCPrice string `mapstructure:"max_rpc_sc_price"` } func (s SiaConfig) Defaults() map[string]interface{} { @@ -35,27 +35,40 @@ func (s SiaConfig) Validate() error { return errors.New("core.storage.sia.url is required") } - if s.MaxUploadPrice <= 0 { - return errors.New("core.storage.sia.max_upload_price must be greater than 0") - } - - if s.MaxDownloadPrice <= 0 { - return errors.New("core.storage.sia.max_download_price must be greater than 0") - } - - if s.MaxStoragePrice <= 0 { - return errors.New("core.storage.sia.max_storage_price must be greater than 0") - } - - err := errors.New("failed to parse core.storage.sia.max_rpc_sc_price ") - - rat, ok := new(big.Rat).SetString(s.MaxRPCSCPrice) - if !ok { + if err := validateStringNumber(s.MaxUploadPrice, "core.storage.sia.max_upload_price"); err != nil { return err } + if err := validateStringNumber(s.MaxDownloadPrice, "core.storage.sia.max_download_price"); err != nil { + return err + } + + if err := validateStringNumber(s.MaxStoragePrice, "core.storage.sia.max_storage_price"); err != nil { + return err + } + + if err := validateStringNumber(s.MaxContractSCPrice, "core.storage.sia.max_contract_sc_price"); err != nil { + return err + } + + if err := validateStringNumber(s.MaxRPCSCPrice, "core.storage.sia.max_rpc_sc_price"); err != nil { + return err + } + return nil +} + +func validateStringNumber(s string, name string) error { + if s == "" { + return errors.New(name + " is required") + } + + rat, ok := new(big.Rat).SetString(s) + if !ok { + return errors.New("failed to parse " + name) + } + if rat.Cmp(new(big.Rat).SetUint64(0)) <= 0 { - return err + return errors.New(name + " must be greater than 0") } return nil diff --git a/db/models/sc_price_history.go b/db/models/sc_price_history.go index 6b75f6c..998b186 100644 --- a/db/models/sc_price_history.go +++ b/db/models/sc_price_history.go @@ -3,6 +3,8 @@ package models import ( "time" + "github.com/shopspring/decimal" + "gorm.io/gorm" "gorm.io/gorm/schema" ) @@ -15,8 +17,8 @@ func init() { type SCPriceHistory struct { gorm.Model - CreatedAt time.Time `gorm:"index:idx_rate"` - Rate float64 `gorm:"index:idx_rate"` + CreatedAt time.Time `gorm:"index:idx_rate"` + Rate decimal.Decimal `gorm:"type:DECIMAL(10,6);index:idx_rate"` } func (SCPriceHistory) TableName() string { diff --git a/go.mod b/go.mod index e7c90f3..cc31dba 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21.6 require ( git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310131713-ee6f140b7e98 github.com/AfterShip/email-verifier v1.4.0 + github.com/LumeWeb/siacentral-api v0.0.0-20240311114304-4ff40c07bce5 github.com/aws/aws-sdk-go-v2 v1.25.1 github.com/aws/aws-sdk-go-v2/config v1.27.2 github.com/aws/aws-sdk-go-v2/credentials v1.17.2 @@ -27,6 +28,7 @@ require ( github.com/redis/go-redis/v9 v9.5.1 github.com/rs/cors v1.10.1 github.com/samber/lo v1.39.0 + github.com/shopspring/decimal v1.3.1 github.com/siacentral/apisdkgo v0.2.10 github.com/spf13/viper v1.18.2 github.com/tus/tusd/v2 v2.2.3-0.20240125123123-9080d351525d @@ -112,7 +114,6 @@ require ( github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/shopspring/decimal v1.3.1 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect diff --git a/go.sum b/go.sum index 2abb41a..d197286 100644 --- a/go.sum +++ b/go.sum @@ -1,22 +1,4 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240309121554-42fa773b52b6 h1:MeyICcSVT4vPLptqVA452XBpKdxDk/eVKeiW5qx6ABw= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240309121554-42fa773b52b6/go.mod h1:h/1HUjA2mWT14QsRprG6o7WDeP3paAB23tkh/OczmUk= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240309181222-438e76dfb848 h1:L721A8/dqEO5E2GxCzQ6JA8To4i4jh1hupNwEbB2ck4= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240309181222-438e76dfb848/go.mod h1:h/1HUjA2mWT14QsRprG6o7WDeP3paAB23tkh/OczmUk= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310104618-48ef3e3a2a45 h1:eB6C/koZkujqQ6E23xDTwbHWjjg6c9dWNHKWGQdbRdI= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310104618-48ef3e3a2a45/go.mod h1:h/1HUjA2mWT14QsRprG6o7WDeP3paAB23tkh/OczmUk= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310112448-1f8d383da743 h1:ejVAkYNBBn4Q/TFlShx0AgUV8h1WrZojZJOZ152b75E= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310112448-1f8d383da743/go.mod h1:h/1HUjA2mWT14QsRprG6o7WDeP3paAB23tkh/OczmUk= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310113844-71cb44dc6169 h1:jxRUeSu1CrDcPnhtq2Xc9uRdkTv4Z0tPk8XxL5dvCUQ= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310113844-71cb44dc6169/go.mod h1:h/1HUjA2mWT14QsRprG6o7WDeP3paAB23tkh/OczmUk= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310124044-45a483989c5b h1:O9r+O5tcOv0Jm8CvnXTD3jaPnxHaJPrMRTYRhcBLMYQ= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310124044-45a483989c5b/go.mod h1:h/1HUjA2mWT14QsRprG6o7WDeP3paAB23tkh/OczmUk= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310125555-2a6c661b494e h1:DHegJ3QU5sYZM7Q0p8TOvQok/nvjuSGAY6h2vzelm4s= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310125555-2a6c661b494e/go.mod h1:h/1HUjA2mWT14QsRprG6o7WDeP3paAB23tkh/OczmUk= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310125732-195abfdf20f2 h1:JVKxP93YkoO4Ck4ThAxjlgf2BEuAa0rzOmGHg1Hahjg= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310125732-195abfdf20f2/go.mod h1:h/1HUjA2mWT14QsRprG6o7WDeP3paAB23tkh/OczmUk= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310130929-bb1b43958ac7 h1:SK+yS5y6PZwzkWYdkTHqVIROKe1wmy1Md8f0CNSPeKI= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310130929-bb1b43958ac7/go.mod h1:h/1HUjA2mWT14QsRprG6o7WDeP3paAB23tkh/OczmUk= git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310131713-ee6f140b7e98 h1:sFemHrtW2qtRYmNfF8o5YTwk60/5cJ4YiOj1iZeI0NQ= git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240310131713-ee6f140b7e98/go.mod h1:h/1HUjA2mWT14QsRprG6o7WDeP3paAB23tkh/OczmUk= github.com/Acconut/go-httptest-recorder v1.0.0 h1:TAv2dfnqp/l+SUvIaMAUK4GeN4+wqb6KZsFFFTGhoJg= @@ -24,6 +6,8 @@ github.com/Acconut/go-httptest-recorder v1.0.0/go.mod h1:CwQyhTH1kq/gLyWiRieo7c0 github.com/AfterShip/email-verifier v1.4.0 h1:DoQplvVFVhZUfS5fPiVnmCQDr5i1tv+ivUV0TFd2AZo= github.com/AfterShip/email-verifier v1.4.0/go.mod h1:JNPV1KZpTq4TArmss1NAOJsTD8JRa/ZElbCAJCEgikg= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/LumeWeb/siacentral-api v0.0.0-20240311114304-4ff40c07bce5 h1:PLu5lde5k11bZcv1gFOHTw3j1JSDg6NE11g8yGml4U4= +github.com/LumeWeb/siacentral-api v0.0.0-20240311114304-4ff40c07bce5/go.mod h1:uxlJTAnfhfebzHGPvKlyaGmS2G7QOX+xRcdmpMiLfXE= github.com/LumeWeb/tusd/v2 v2.2.3-0.20240224143554-96925dd43120 h1:+FYQ83a3c0p6Y/sqp3373CJ2m/b0mskdG3ma/opxtlk= github.com/LumeWeb/tusd/v2 v2.2.3-0.20240224143554-96925dd43120/go.mod h1:lqzUzWTG5OwezKgA1HJ+uwyjJusv6StTYdLTIvo0nxE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/renter/price_tracker.go b/renter/price_tracker.go index 816a4d2..16280a0 100644 --- a/renter/price_tracker.go +++ b/renter/price_tracker.go @@ -6,16 +6,18 @@ import ( "math/big" "time" + "github.com/shopspring/decimal" + "github.com/docker/go-units" "git.lumeweb.com/LumeWeb/portal/db/models" - "github.com/siacentral/apisdkgo" + siasdk "github.com/LumeWeb/siacentral-api" "git.lumeweb.com/LumeWeb/portal/config" "git.lumeweb.com/LumeWeb/portal/cron" + siasdksia "github.com/LumeWeb/siacentral-api/sia" "github.com/go-co-op/gocron/v2" - siasdk "github.com/siacentral/apisdkgo/sia" "go.sia.tech/core/types" "go.uber.org/fx" "go.uber.org/zap" @@ -34,7 +36,7 @@ type PriceTracker struct { cron *cron.CronServiceDefault db *gorm.DB renter *RenterDefault - api *siasdk.APIClient + api *siasdksia.APIClient } func (p PriceTracker) LoadInitialTasks(cron cron.CronService) error { @@ -77,7 +79,7 @@ func (p PriceTracker) recordRate() { } func (p PriceTracker) updatePrices() error { - var averageRate float64 + var averageRate decimal.Decimal days := p.config.Config().Core.Storage.Sia.PriceHistoryDays sql := ` SELECT AVG(rate) as average_rate FROM ( @@ -94,7 +96,7 @@ SELECT AVG(rate) as average_rate FROM ( return err } - if averageRate == 0 { + if averageRate.Equal(decimal.Zero) { p.logger.Error("average rate is 0") return errors.New("average rate is 0") } @@ -115,38 +117,46 @@ SELECT AVG(rate) as average_rate FROM ( return err } - maxDownloadPrice := p.config.Config().Core.Storage.Sia.MaxDownloadPrice / averageRate - - gouge.MaxDownloadPrice, err = siacoinsFromFloat(maxDownloadPrice) + maxDownloadPrice, err := computeByRate(p.config.Config().Core.Storage.Sia.MaxDownloadPrice, averageRate, "max download price") if err != nil { return err } - maxUploadPrice := p.config.Config().Core.Storage.Sia.MaxUploadPrice / averageRate - - p.logger.Debug("Setting max upload price", zap.Float64("maxUploadPrice", maxUploadPrice)) - - gouge.MaxUploadPrice, err = siacoinsFromFloat(maxUploadPrice) + gouge.MaxDownloadPrice, err = siacoinsFromRat(maxDownloadPrice) if err != nil { return err } - maxContractPrice := p.config.Config().Core.Storage.Sia.MaxContractSCPrice - - p.logger.Debug("Setting max contract price", zap.Float64("maxContractPrice", maxContractPrice)) - - gouge.MaxContractPrice, err = siacoinsFromFloat(maxContractPrice) + maxUploadPrice, err := computeByRate(p.config.Config().Core.Storage.Sia.MaxUploadPrice, averageRate, "max upload price") if err != nil { return err } - maxRPCPrice, ok := new(big.Rat).SetString(p.config.Config().Core.Storage.Sia.MaxRPCSCPrice) + p.logger.Debug("Setting max upload price", zap.String("maxUploadPrice", maxUploadPrice.FloatString(decimalsInSiacoin))) - if !ok { - return errors.New("failed to parse max rpc price") + gouge.MaxUploadPrice, err = siacoinsFromRat(maxUploadPrice) + if err != nil { + return err } - maxRPCPrice = new(big.Rat).Quo(maxRPCPrice, new(big.Rat).SetUint64(1_000_000)) + maxContractPrice, err := computeByRate(p.config.Config().Core.Storage.Sia.MaxContractSCPrice, averageRate, "max contract price") + if err != nil { + return err + } + + p.logger.Debug("Setting max contract price", zap.String("maxContractPrice", maxContractPrice.FloatString(decimalsInSiacoin))) + + gouge.MaxContractPrice, err = siacoinsFromRat(maxContractPrice) + if err != nil { + return err + } + + maxRPCPrice, err := computeByRate(p.config.Config().Core.Storage.Sia.MaxRPCSCPrice, averageRate, "max rpc price") + if err != nil { + return err + } + + maxRPCPrice = ratDivide(maxRPCPrice, 1_000_000) p.logger.Debug("Setting max RPC price", zap.String("maxRPCPrice", maxRPCPrice.FloatString(decimalsInSiacoin))) @@ -155,15 +165,18 @@ SELECT AVG(rate) as average_rate FROM ( return err } - maxStoragePrice := p.config.Config().Core.Storage.Sia.MaxStoragePrice - maxStoragePrice = maxStoragePrice / redundancy.Redundancy() - maxStoragePrice = maxStoragePrice / units.TiB - maxStoragePrice = maxStoragePrice / blocksPerMonth - maxStoragePrice = maxStoragePrice / averageRate + maxStoragePrice, err := computeByRate(p.config.Config().Core.Storage.Sia.MaxStoragePrice, averageRate, "max storage price") + if err != nil { + return err + } - p.logger.Debug("Setting max storage price", zap.Float64("maxStoragePrice", maxStoragePrice)) + maxStoragePrice = ratDivideFloat(maxStoragePrice, redundancy.Redundancy()) + maxStoragePrice = ratDivide(maxStoragePrice, units.TiB) + maxStoragePrice = ratDivide(maxStoragePrice, blocksPerMonth) - gouge.MaxStoragePrice, err = siacoinsFromFloat(maxStoragePrice) + p.logger.Debug("Setting max storage price", zap.String("maxStoragePrice", maxStoragePrice.FloatString(decimalsInSiacoin))) + + gouge.MaxStoragePrice, err = siacoinsFromRat(maxStoragePrice) if err != nil { return err } @@ -233,12 +246,12 @@ type PriceTrackerParams struct { Cron *cron.CronServiceDefault Db *gorm.DB Renter *RenterDefault - PriceApi *siasdk.APIClient + PriceApi *siasdksia.APIClient } func (p PriceTracker) init() error { p.cron.RegisterService(p) - p.api = apisdkgo.NewSiaClient() + p.api = siasdk.NewSiaClient() go func() { err := p.importPrices() @@ -277,7 +290,21 @@ func siacoinsFromRat(r *big.Rat) (types.Currency, error) { return types.NewCurrency(i.Uint64(), new(big.Int).Rsh(i, 64).Uint64()), nil } -func siacoinsFromFloat(f float64) (types.Currency, error) { - r := new(big.Rat).SetFloat64(f) - return siacoinsFromRat(r) +func computeByRate(num string, rate decimal.Decimal, name string) (*big.Rat, error) { + parsedNum, ok := new(big.Rat).SetString(num) + + if !ok { + return nil, errors.New("failed to parse " + name) + } + + parsedRate := new(big.Rat).Quo(parsedNum, rate.Rat()) + + return parsedRate, nil +} + +func ratDivide(a *big.Rat, b uint64) *big.Rat { + return new(big.Rat).Quo(a, new(big.Rat).SetUint64(b)) +} +func ratDivideFloat(a *big.Rat, b float64) *big.Rat { + return new(big.Rat).Quo(a, new(big.Rat).SetFloat64(b)) } diff --git a/renter/renter.go b/renter/renter.go index 2944423..5049b02 100644 --- a/renter/renter.go +++ b/renter/renter.go @@ -15,7 +15,7 @@ import ( "git.lumeweb.com/LumeWeb/portal/config" "git.lumeweb.com/LumeWeb/portal/cron" - sia "github.com/siacentral/apisdkgo" + sia "github.com/LumeWeb/siacentral-api" rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/renterd/api" busClient "go.sia.tech/renterd/bus/client"