From 7f12ee5b0df623c4fe7f59796fe2e6f807380240 Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Thu, 22 Feb 2024 02:09:31 -0500 Subject: [PATCH] refactor: implement new configuration management system --- CHANGELOG.md | 4 +- api/account/account.go | 7 +- api/api.go | 13 ++- api/middleware/middleware.go | 11 ++- api/s5/s5.go | 11 ++- cmd/portal/main.go | 11 ++- config/config.go | 169 ++++++++++++++++++++++----------- config/core.go | 14 +++ config/database.go | 10 ++ config/log.go | 5 + config/protocol.go | 5 + config/s3.go | 9 ++ config/sia.go | 6 ++ config/storage.go | 5 + go.mod | 2 +- go.sum | 4 +- protocols/protocols.go | 14 ++- protocols/registry/registry.go | 3 + protocols/s5/config.go | 26 +++++ protocols/s5/s5.go | 86 ++++++++--------- protocols/s5/tus.go | 29 +++--- 21 files changed, 296 insertions(+), 148 deletions(-) create mode 100644 config/core.go create mode 100644 config/database.go create mode 100644 config/log.go create mode 100644 config/protocol.go create mode 100644 config/s3.go create mode 100644 config/sia.go create mode 100644 config/storage.go create mode 100644 protocols/s5/config.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e1e0d4..4b14067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,11 +40,11 @@ * if we have an existing upload, just return it as if successful ([90170e5](https://git.lumeweb.com/LumeWeb/portal/commit/90170e5b81831f3d768291fd37c7c13e32d522fe)) * iris context.User needs to be embedded in our User struct for type checking to properly work ([1cfc222](https://git.lumeweb.com/LumeWeb/portal/commit/1cfc2223a6df614f26fd0337ced68d92e774589f)) * just use the any route ([e100429](https://git.lumeweb.com/LumeWeb/portal/commit/e100429b60e783f6c7c3ddecab7bb9b4dd599726)) -* load config before db ([58165e0](https://git.lumeweb.com/LumeWeb/portal/commit/58165e01af9f2b183d654d3d8809cbd1eda0a9bb)) +* load awsConfig before db ([58165e0](https://git.lumeweb.com/LumeWeb/portal/commit/58165e01af9f2b183d654d3d8809cbd1eda0a9bb)) * make an attempt to look for the token before adding to db ([f11b285](https://git.lumeweb.com/LumeWeb/portal/commit/f11b285d4e255c1c4c95f6ac15aa904d7a5730e4)) * missing setting SetTusComposer ([80561f8](https://git.lumeweb.com/LumeWeb/portal/commit/80561f89e92dfa86887ada8361e0046ee6288234)) * newer gorm version causes db rebuilds every boot ([72255eb](https://git.lumeweb.com/LumeWeb/portal/commit/72255eb3c50892aa5f2cfdc4cb1daa5883f0affc)) -* only panic if the error is other than a missing config file ([6e0ec8a](https://git.lumeweb.com/LumeWeb/portal/commit/6e0ec8aaf90e86bcb7cb6c8c53f6569e6885e0aa)) +* only panic if the error is other than a missing awsConfig file ([6e0ec8a](https://git.lumeweb.com/LumeWeb/portal/commit/6e0ec8aaf90e86bcb7cb6c8c53f6569e6885e0aa)) * output error info ([cfa7ceb](https://git.lumeweb.com/LumeWeb/portal/commit/cfa7ceb2f422a6e594a424315c8eaeffc6572926)) * PostPubkeyChallenge should be lowercasing the pubkey for consistency ([d680f06](https://git.lumeweb.com/LumeWeb/portal/commit/d680f0660f910e323356a1169ee13ef2e647a015)) * PostPubkeyChallenge should be using ChallengeRequest ([36745bb](https://git.lumeweb.com/LumeWeb/portal/commit/36745bb55b1d7cd464b085e410333089504591c1)) diff --git a/api/account/account.go b/api/account/account.go index 45e1776..8cee50f 100644 --- a/api/account/account.go +++ b/api/account/account.go @@ -5,6 +5,8 @@ import ( "crypto/ed25519" "net/http" + "git.lumeweb.com/LumeWeb/portal/config" + "go.uber.org/zap" "github.com/julienschmidt/httprouter" @@ -12,7 +14,6 @@ import ( "git.lumeweb.com/LumeWeb/portal/account" "git.lumeweb.com/LumeWeb/portal/api/middleware" "git.lumeweb.com/LumeWeb/portal/api/registry" - "github.com/spf13/viper" "go.sia.tech/jape" "go.uber.org/fx" ) @@ -22,7 +23,7 @@ var ( ) type AccountAPI struct { - config *viper.Viper + config *config.Manager accounts *account.AccountServiceDefault identity ed25519.PrivateKey logger *zap.Logger @@ -30,7 +31,7 @@ type AccountAPI struct { type AccountAPIParams struct { fx.In - Config *viper.Viper + Config *config.Manager Accounts *account.AccountServiceDefault Identity ed25519.PrivateKey Logger *zap.Logger diff --git a/api/api.go b/api/api.go index b2ac798..92581b0 100644 --- a/api/api.go +++ b/api/api.go @@ -4,18 +4,21 @@ import ( "context" "slices" + "git.lumeweb.com/LumeWeb/portal/config" + "git.lumeweb.com/LumeWeb/portal/api/middleware" "git.lumeweb.com/LumeWeb/portal/api/registry" - "github.com/spf13/viper" "go.uber.org/fx" ) -func BuildApis(config *viper.Viper) fx.Option { +var alwaysEnabled = []string{"account"} + +func BuildApis(cm *config.Manager) fx.Option { var options []fx.Option - enabledProtocols := config.GetStringSlice("core.protocols") + enabledProtocols := cm.Viper().GetStringSlice("core.protocols") for _, entry := range registry.GetRegistry() { - if slices.Contains(enabledProtocols, entry.Key) { + if slices.Contains(enabledProtocols, entry.Key) || slices.Contains(alwaysEnabled, entry.Key) { options = append(options, entry.Module) } } @@ -42,7 +45,7 @@ func BuildApis(config *viper.Viper) fx.Option { if err != nil { return err } - middleware.RegisterProtocolSubdomain(config, routes, protocol.Name()) + middleware.RegisterProtocolSubdomain(cm, routes, protocol.Name()) } return nil diff --git a/api/middleware/middleware.go b/api/middleware/middleware.go index 64c8918..7349bea 100644 --- a/api/middleware/middleware.go +++ b/api/middleware/middleware.go @@ -8,11 +8,12 @@ import ( "strconv" "strings" + "git.lumeweb.com/LumeWeb/portal/config" + "git.lumeweb.com/LumeWeb/portal/account" "git.lumeweb.com/LumeWeb/portal/api/registry" "github.com/golang-jwt/jwt/v5" "github.com/julienschmidt/httprouter" - "github.com/spf13/viper" "go.sia.tech/jape" ) @@ -64,9 +65,9 @@ func ApplyMiddlewares(handler jape.Handler, middlewares ...interface{}) jape.Han } return handler } -func RegisterProtocolSubdomain(config *viper.Viper, mux *httprouter.Router, name string) { +func RegisterProtocolSubdomain(config *config.Manager, mux *httprouter.Router, name string) { router := registry.GetRouter() - domain := config.GetString("core.domain") + domain := config.Config().Core.Domain (router)[name+"."+domain] = mux } @@ -103,7 +104,7 @@ type AuthMiddlewareOptions struct { FindToken FindAuthTokenFunc Purpose account.JWTPurpose AuthContextKey string - Config *viper.Viper + Config *config.Manager } func AuthMiddleware(options AuthMiddlewareOptions) func(http.Handler) http.Handler { @@ -114,7 +115,7 @@ func AuthMiddleware(options AuthMiddlewareOptions) func(http.Handler) http.Handl panic("purpose is missing") } - domain := options.Config.GetString("core.domain") + domain := options.Config.Config().Core.Domain return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/api/s5/s5.go b/api/s5/s5.go index 866a0f3..b4b2468 100644 --- a/api/s5/s5.go +++ b/api/s5/s5.go @@ -19,6 +19,8 @@ import ( "strings" "time" + "git.lumeweb.com/LumeWeb/portal/config" + "git.lumeweb.com/LumeWeb/portal/api/swagger" "git.lumeweb.com/LumeWeb/portal/metadata" @@ -48,7 +50,6 @@ import ( protoRegistry "git.lumeweb.com/LumeWeb/portal/protocols/registry" "git.lumeweb.com/LumeWeb/portal/protocols/s5" "github.com/rs/cors" - "github.com/spf13/viper" "go.sia.tech/jape" "go.uber.org/fx" ) @@ -61,7 +62,7 @@ var ( var swagSpec []byte type S5API struct { - config *viper.Viper + config *config.Manager identity ed25519.PrivateKey accounts *account.AccountServiceDefault storage storage.StorageService @@ -75,7 +76,7 @@ type S5API struct { type APIParams struct { fx.In - Config *viper.Viper + Config *config.Manager Identity ed25519.PrivateKey Accounts *account.AccountServiceDefault Storage storage.StorageService @@ -355,7 +356,7 @@ func (s *S5API) prepareFileUpload(jc jape.Context) (file io.ReadSeekCloser, s5Er // Handle multipart form data uploads if strings.HasPrefix(contentType, "multipart/form-data") { - if err := r.ParseMultipartForm(s.config.GetInt64("core.post-upload-limit")); err != nil { + if err := r.ParseMultipartForm(s.config.Config().Core.PostUploadLimit); err != nil { return nil, NewS5Error(ErrKeyFileUploadFailed, err) } @@ -745,7 +746,7 @@ func (s *S5API) directoryUpload(jc jape.Context) { } // Parse multipart form with size limit from config - if err := jc.Request.ParseMultipartForm(s.config.GetInt64("core.post-upload-limit")); err != nil { + if err := jc.Request.ParseMultipartForm(s.config.Config().Core.PostUploadLimit); err != nil { s.sendErrorResponse(jc, NewS5Error(ErrKeyInvalidOperation, err)) return } diff --git a/cmd/portal/main.go b/cmd/portal/main.go index 257079c..28ac3fe 100644 --- a/cmd/portal/main.go +++ b/cmd/portal/main.go @@ -4,9 +4,10 @@ import ( "flag" "net/http" + "git.lumeweb.com/LumeWeb/portal/config" + "git.lumeweb.com/LumeWeb/portal/account" "git.lumeweb.com/LumeWeb/portal/api" - _config "git.lumeweb.com/LumeWeb/portal/config" "git.lumeweb.com/LumeWeb/portal/cron" "git.lumeweb.com/LumeWeb/portal/db" _logger "git.lumeweb.com/LumeWeb/portal/logger" @@ -23,7 +24,7 @@ import ( func main() { logger := _logger.NewLogger() - config, err := _config.NewConfig(logger) + cfg, err := config.NewManager(logger) if err != nil { logger.Fatal("Failed to load config", zap.Error(err)) @@ -48,7 +49,7 @@ func main() { } fx.New( - fx.Supply(config), + fx.Supply(cfg), fx.Supply(logger), fxLogger, fx.Invoke(initCheckRequiredConfig), @@ -59,8 +60,8 @@ func main() { cron.Module, account.Module, metadata.Module, - protocols.BuildProtocols(config), - api.BuildApis(config), + protocols.BuildProtocols(cfg), + api.BuildApis(cfg), fx.Provide(api.NewCasbin), fx.Invoke(protocols.SetupLifecycles), fx.Invoke(api.SetupLifecycles), diff --git a/config/config.go b/config/config.go index 82c8688..c056e86 100644 --- a/config/config.go +++ b/config/config.go @@ -2,7 +2,10 @@ package config import ( "errors" + "fmt" + _logger "git.lumeweb.com/LumeWeb/portal/logger" + "github.com/docker/go-units" "github.com/spf13/viper" "go.uber.org/zap" ) @@ -15,7 +18,106 @@ var ( } ) -func NewConfig(logger *zap.Logger) (*viper.Viper, error) { +type Config struct { + Core CoreConfig `mapstructure:"core"` +} + +type Manager struct { + viper *viper.Viper + root *Config + changes bool +} + +func NewManager(logger *zap.Logger) (*Manager, error) { + v, err := newConfig(logger) + if err != nil { + return nil, err + } + + var config Config + + m := &Manager{ + viper: v, + root: &config, + } + + m.setDefaults(m.coreDefaults(), "") + err = m.maybeSave() + if err != nil { + return nil, err + } + + err = v.Unmarshal(config) + if err != nil { + return nil, err + } + + return nil, nil +} + +func (m *Manager) ConfigureProtocol(name string, cfg ProtocolConfig) error { + defaults := cfg.Defaults() + + m.setDefaults(defaults, fmt.Sprintf("protocol.%s", name)) + err := m.maybeSave() + if err != nil { + return err + } + + return m.viper.Unmarshal(cfg) +} + +func (m *Manager) setDefaults(defaults map[string]interface{}, prefix string) { + for key, value := range defaults { + if prefix != "" { + key = fmt.Sprintf("%s.%s", prefix, key) + } + if m.setDefault(key, value) { + m.changes = true + } + } +} + +func (m *Manager) setDefault(key string, value interface{}) bool { + if !m.viper.IsSet(key) { + m.viper.SetDefault(key, value) + return true + } + + return false +} + +func (m *Manager) maybeSave() error { + if m.changes { + ret := m.viper.WriteConfig() + if ret != nil { + return ret + } + m.changes = false + } + + return nil +} + +func (m *Manager) coreDefaults() map[string]interface{} { + return map[string]interface{}{ + "core.post-upload-limit": units.MiB * 100, + "core.log.level": "info", + "core.db.charset": "utf8mb4", + "core.db.port": 3306, + "core.db.name": "portal", + } +} + +func (m *Manager) Config() *Config { + return m.root +} + +func (m *Manager) Viper() *viper.Viper { + return m.viper +} + +func newConfig(logger *zap.Logger) (*viper.Viper, error) { if logger == nil { logger = _logger.NewFallbackLogger() } @@ -32,64 +134,19 @@ func NewConfig(logger *zap.Logger) (*viper.Viper, error) { err := viper.ReadInConfig() if err != nil { - if errors.As(err, &viper.ConfigFileNotFoundError{}) { - logger.Info("Config file not found, using default settings.") - err := viper.SafeWriteConfig() - if err != nil { - return nil, err - } - err = writeDefaults() - if err != nil { - return nil, err - } - - return viper.GetViper(), nil + if !errors.Is(err, &viper.ConfigFileNotFoundError{}) { + return nil, err } - return nil, err - } - err = writeDefaults() - if err != nil { - return nil, err + logger.Info("Config file not found, using default settings.") + err := viper.SafeWriteConfig() + if err != nil { + return nil, err + } + + return viper.GetViper(), nil + } return viper.GetViper(), nil } - -func writeDefaults() error { - defaults := map[string]interface{}{ - "core.post-upload-limit": 1024 * 1024 * 1000, - "core.log.level": "info", - "core.db.charset": "utf8mb4", - "core.db.port": 3306, - "core.db.name": "portal", - "protocol.s5.p2p.maxOutgoingPeerFailures": 10, - "protocol.s5.p2p.network": "", - } - - changes := false - - for key, value := range defaults { - if writeDefault(key, value) { - changes = true - } - } - - if changes { - err := viper.WriteConfig() - if err != nil { - return err - } - } - - return nil -} - -func writeDefault(key string, value interface{}) bool { - if !viper.IsSet(key) { - viper.SetDefault(key, value) - return true - } - - return false -} diff --git a/config/core.go b/config/core.go new file mode 100644 index 0000000..4bd8bb8 --- /dev/null +++ b/config/core.go @@ -0,0 +1,14 @@ +package config + +type CoreConfig struct { + DB DatabaseConfig `mapstructure:"db"` + Domain string `mapstructure:"domain"` + ExternalPort uint `mapstructure:"external_port"` + Identity string `mapstructure:"identity"` + Log LogConfig `mapstructure:"log"` + Port uint `mapstructure:"port"` + PostUploadLimit int64 `mapstructure:"post_upload_limit"` + Sia SiaConfig `mapstructure:"sia"` + Storage StorageConfig `mapstructure:"storage"` + Protocols []string `mapstructure:"protocols"` +} diff --git a/config/database.go b/config/database.go new file mode 100644 index 0000000..68983aa --- /dev/null +++ b/config/database.go @@ -0,0 +1,10 @@ +package config + +type DatabaseConfig struct { + Charset string `mapstructure:"charset"` + Host string `mapstructure:"host"` + Name string `mapstructure:"name"` + Password string `mapstructure:"password"` + Port int `mapstructure:"port"` + Username string `mapstructure:"username"` +} diff --git a/config/log.go b/config/log.go new file mode 100644 index 0000000..4a83a2b --- /dev/null +++ b/config/log.go @@ -0,0 +1,5 @@ +package config + +type LogConfig struct { + Level string `mapstructure:"level"` +} diff --git a/config/protocol.go b/config/protocol.go new file mode 100644 index 0000000..c0cef98 --- /dev/null +++ b/config/protocol.go @@ -0,0 +1,5 @@ +package config + +type ProtocolConfig interface { + Defaults() map[string]interface{} +} diff --git a/config/s3.go b/config/s3.go new file mode 100644 index 0000000..8b5f312 --- /dev/null +++ b/config/s3.go @@ -0,0 +1,9 @@ +package config + +type S3Config struct { + BufferBucket string `mapstructure:"buffer_bucket"` + Endpoint string `mapstructure:"endpoint"` + Region string `mapstructure:"region"` + AccessKey string `mapstructure:"access_key"` + SecretKey string `mapstructure:"secret_key"` +} diff --git a/config/sia.go b/config/sia.go new file mode 100644 index 0000000..696978f --- /dev/null +++ b/config/sia.go @@ -0,0 +1,6 @@ +package config + +type SiaConfig struct { + Key string `mapstructure:"key"` + URL string `mapstructure:"url"` +} diff --git a/config/storage.go b/config/storage.go new file mode 100644 index 0000000..b8ca374 --- /dev/null +++ b/config/storage.go @@ -0,0 +1,5 @@ +package config + +type StorageConfig struct { + S3 S3Config `mapstructure:"s3"` +} diff --git a/go.mod b/go.mod index fd2dd0e..2f8f58a 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240201012059-dfeb8b29a8e4 github.com/AfterShip/email-verifier v1.4.0 github.com/aws/aws-sdk-go-v2 v1.24.0 - github.com/aws/aws-sdk-go-v2/config v1.26.2 + github.com/aws/aws-sdk-go-v2/cm v1.26.2 github.com/aws/aws-sdk-go-v2/credentials v1.16.13 github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7 github.com/casbin/casbin/v2 v2.81.0 diff --git a/go.sum b/go.sum index a160f43..cbeee32 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7Aw github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= -github.com/aws/aws-sdk-go-v2/config v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc= -github.com/aws/aws-sdk-go-v2/config v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8= +github.com/aws/aws-sdk-go-v2/cm v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc= +github.com/aws/aws-sdk-go-v2/cm v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8= github.com/aws/aws-sdk-go-v2/credentials v1.16.13 h1:WLABQ4Cp4vXtXfOWOS3MEZKr6AAYUpMczLhgKtAjQ/8= github.com/aws/aws-sdk-go-v2/credentials v1.16.13/go.mod h1:Qg6x82FXwW0sJHzYruxGiuApNo31UEtJvXVSZAXeWiw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58= diff --git a/protocols/protocols.go b/protocols/protocols.go index ef20dc1..f08853a 100644 --- a/protocols/protocols.go +++ b/protocols/protocols.go @@ -3,15 +3,16 @@ package protocols import ( "context" + "git.lumeweb.com/LumeWeb/portal/config" + "git.lumeweb.com/LumeWeb/portal/protocols/registry" "github.com/samber/lo" - "github.com/spf13/viper" "go.uber.org/fx" ) -func BuildProtocols(config *viper.Viper) fx.Option { +func BuildProtocols(cm *config.Manager) fx.Option { var options []fx.Option - enabledProtocols := config.GetStringSlice("core.protocols") + enabledProtocols := cm.Viper().GetStringSlice("core.protocols") for _, entry := range registry.GetRegistry() { if lo.Contains(enabledProtocols, entry.Key) { options = append(options, entry.Module) @@ -28,7 +29,12 @@ func BuildProtocols(config *viper.Viper) fx.Option { options = append(options, fx.Invoke(func(params initParams) error { for _, protocol := range params.Protocols { - err := protocol.Init() + err := cm.ConfigureProtocol(protocol.Name(), protocol.Config()) + if err != nil { + return err + } + + err = protocol.Init() if err != nil { return err } diff --git a/protocols/registry/registry.go b/protocols/registry/registry.go index 085dad1..eca6276 100644 --- a/protocols/registry/registry.go +++ b/protocols/registry/registry.go @@ -3,6 +3,8 @@ package registry import ( "context" + "git.lumeweb.com/LumeWeb/portal/config" + "go.uber.org/fx" ) @@ -13,6 +15,7 @@ type Protocol interface { Init() error Start(ctx context.Context) error Stop(ctx context.Context) error + Config() config.ProtocolConfig } type ProtocolEntry struct { diff --git a/protocols/s5/config.go b/protocols/s5/config.go new file mode 100644 index 0000000..9fb2b50 --- /dev/null +++ b/protocols/s5/config.go @@ -0,0 +1,26 @@ +package s5 + +import ( + s5config "git.lumeweb.com/LumeWeb/libs5-go/config" + "git.lumeweb.com/LumeWeb/portal/config" +) + +var _ config.ProtocolConfig = (*Config)(nil) + +type Config struct { + s5config.NodeConfig + DbPath string `mapstructure:"db_path"` +} + +func (c Config) Defaults() map[string]interface{} { + + defaults := map[string]interface{}{} + + defaults["p2p.network.peers"] = []string{ + "ss://z2DWuWNZcdSyZLpXFK2uCU3haaWMXrDAgxzv17sDEMHstZb@s5.garden/s5/p2p", + "wss://z2DWuPbL5pweybXnEB618pMnV58ECj2VPDNfVGm3tFqBvjF@s5.ninja/s5/p2p", + } + defaults["db_path"] = "s5.db" + + return defaults +} diff --git a/protocols/s5/s5.go b/protocols/s5/s5.go index 123919a..a42325b 100644 --- a/protocols/s5/s5.go +++ b/protocols/s5/s5.go @@ -7,6 +7,8 @@ import ( "fmt" "time" + "git.lumeweb.com/LumeWeb/portal/config" + "git.lumeweb.com/LumeWeb/portal/metadata" "git.lumeweb.com/LumeWeb/portal/storage" @@ -32,18 +34,19 @@ var ( ) type S5Protocol struct { - config *viper.Viper - logger *zap.Logger - storage storage.StorageService - identity ed25519.PrivateKey - node *s5node.Node - tusHandler *TusHandler - store *S5ProviderStore + portalConfig *config.Manager + config *Config + logger *zap.Logger + storage storage.StorageService + identity ed25519.PrivateKey + node *s5node.Node + tusHandler *TusHandler + store *S5ProviderStore } type S5ProtocolParams struct { fx.In - Config *viper.Viper + PortalConfig *config.Manager Logger *zap.Logger Storage storage.StorageService Identity ed25519.PrivateKey @@ -85,15 +88,15 @@ func NewS5Protocol( params S5ProtocolParams, ) (S5ProtocolResult, error) { proto := &S5Protocol{ - config: params.Config, - logger: params.Logger, - storage: params.Storage, - identity: params.Identity, - tusHandler: params.TusHandler, - store: params.ProviderStore, + portalConfig: params.PortalConfig, + logger: params.Logger, + storage: params.Storage, + identity: params.Identity, + tusHandler: params.TusHandler, + store: params.ProviderStore, } - cfg, err := ConfigureS5Protocol(params) + cfg, err := configureS5Protocol(proto) if err != nil { return S5ProtocolResult{}, err } @@ -106,60 +109,53 @@ func NewS5Protocol( }, nil } -func ConfigureS5Protocol(params S5ProtocolParams) (*s5config.NodeConfig, error) { - cfg := &s5config.NodeConfig{ - P2P: s5config.P2PConfig{ - Network: "", - Peers: s5config.PeersConfig{Initial: []string{}}, - }, - KeyPair: s5ed.New(params.Identity), - DB: nil, - Logger: params.Logger.Named("s5"), - HTTP: s5config.HTTPConfig{}, - } +func configureS5Protocol(proto *S5Protocol) (*s5config.NodeConfig, error) { + cfg := proto.Config().(*Config) + cm := proto.portalConfig + portalCfg := cm.Config() + vpr := cm.Viper() - pconfig := params.Config.Sub("protocol.s5") - - if pconfig == nil { - params.Logger.Fatal("Missing protocol.s5 Config") - } - - err := pconfig.Unmarshal(cfg) + err := cm.ConfigureProtocol(proto.Name(), cfg) if err != nil { return nil, err } - cfg.HTTP.API.Domain = fmt.Sprintf("s5.%s", params.Config.GetString("core.domain")) + cfg.HTTP.API.Domain = fmt.Sprintf("s5.%s", vpr.GetString("core.domain")) - if params.Config.IsSet("core.externalPort") { - cfg.HTTP.API.Port = params.Config.GetUint("core.externalPort") + if portalCfg.Core.ExternalPort != 0 { + cfg.HTTP.API.Port = portalCfg.Core.ExternalPort } else { - cfg.HTTP.API.Port = params.Config.GetUint("core.port") + cfg.HTTP.API.Port = portalCfg.Core.Port } - dbPath := pconfig.GetString("dbPath") - - if dbPath == "" { - params.Logger.Fatal("protocol.s5.dbPath is required") + if cfg.DbPath == "" { + proto.logger.Fatal("protocol.s5.dbPath is required") } _, p, err := ed25519.GenerateKey(nil) if err != nil { - params.Logger.Fatal("Failed to generate key", zap.Error(err)) + proto.logger.Fatal("Failed to generate key", zap.Error(err)) } cfg.KeyPair = s5ed.New(p) - db, err := bolt.Open(dbPath, 0600, nil) + db, err := bolt.Open(cfg.DbPath, 0600, nil) if err != nil { - params.Logger.Fatal("Failed to open db", zap.Error(err)) + proto.logger.Fatal("Failed to open db", zap.Error(err)) } cfg.DB = db - return cfg, nil + return interface{}(cfg).(*s5config.NodeConfig), nil } +func (s *S5Protocol) Config() config.ProtocolConfig { + if s.config == nil { + s.config = &Config{} + } + + return s.config +} func NewS5ProviderStore(params S5ProviderStoreParams) *S5ProviderStore { return &S5ProviderStore{ config: params.Config, diff --git a/protocols/s5/tus.go b/protocols/s5/tus.go index 561541c..909a706 100644 --- a/protocols/s5/tus.go +++ b/protocols/s5/tus.go @@ -11,16 +11,15 @@ import ( "time" "git.lumeweb.com/LumeWeb/portal/api/middleware" + "git.lumeweb.com/LumeWeb/portal/config" "go.uber.org/fx" "git.lumeweb.com/LumeWeb/portal/account" - "github.com/spf13/viper" - "git.lumeweb.com/LumeWeb/portal/metadata" - "github.com/aws/aws-sdk-go-v2/config" + awsConfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/tus/tusd/v2/pkg/s3store" @@ -45,7 +44,7 @@ var ( ) type TusHandler struct { - config *viper.Viper + config *config.Manager db *gorm.DB logger *zap.Logger cron *cron.CronServiceDefault @@ -60,7 +59,7 @@ type TusHandler struct { type TusHandlerParams struct { fx.In - Config *viper.Viper + Config *config.Manager Logger *zap.Logger Db *gorm.DB Cron *cron.CronServiceDefault @@ -127,21 +126,21 @@ func (t *TusHandler) Init() error { customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { if service == s3.ServiceID { return aws.Endpoint{ - URL: t.config.GetString("core.storage.s3.endpoint"), - SigningRegion: t.config.GetString("core.storage.s3.region"), + URL: t.config.Config().Core.Storage.S3.Endpoint, + SigningRegion: t.config.Config().Core.Storage.S3.Region, }, nil } return aws.Endpoint{}, &aws.EndpointNotFoundError{} }) - cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion("us-east-1"), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( - t.config.GetString("core.storage.s3.accessKey"), - t.config.GetString("core.storage.s3.secretKey"), + cfg, err := awsConfig.LoadDefaultConfig(context.TODO(), + awsConfig.WithRegion("us-east-1"), + awsConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + t.config.Config().Core.Storage.S3.AccessKey, + t.config.Config().Core.Storage.S3.SecretKey, "", )), - config.WithEndpointResolverWithOptions(customResolver), + awsConfig.WithEndpointResolverWithOptions(customResolver), ) if err != nil { return err @@ -149,7 +148,7 @@ func (t *TusHandler) Init() error { s3Client := s3.NewFromConfig(cfg) - store := s3store.New(t.config.GetString("core.storage.s3.bufferBucket"), s3Client) + store := s3store.New(t.config.Config().Core.Storage.S3.BufferBucket, s3Client) locker := NewMySQLLocker(t.db, t.logger) @@ -407,7 +406,7 @@ func (t *TusHandler) uploadTask(hash []byte) error { s3InfoId, _ := splitS3Ids(upload.UploadID) _, err = t.s3Client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ - Bucket: aws.String(t.config.GetString("core.storage.s3.bufferBucket")), + Bucket: aws.String(t.config.Config().Core.Storage.S3.BufferBucket), Delete: &s3types.Delete{ Objects: []s3types.ObjectIdentifier{ {