refactor: implement new configuration management system

This commit is contained in:
Derrick Hammer 2024-02-22 02:09:31 -05:00
parent 99b97d9495
commit 7f12ee5b0d
Signed by: pcfreak30
GPG Key ID: C997C339BE476FF2
21 changed files with 296 additions and 148 deletions

View File

@ -40,11 +40,11 @@
* if we have an existing upload, just return it as if successful ([90170e5](https://git.lumeweb.com/LumeWeb/portal/commit/90170e5b81831f3d768291fd37c7c13e32d522fe)) * if we have an existing upload, just return it as if successful ([90170e5](https://git.lumeweb.com/LumeWeb/portal/commit/90170e5b81831f3d768291fd37c7c13e32d522fe))
* iris context.User needs to be embedded in our User struct for type checking to properly work ([1cfc222](https://git.lumeweb.com/LumeWeb/portal/commit/1cfc2223a6df614f26fd0337ced68d92e774589f)) * iris context.User needs to be embedded in our User struct for type checking to properly work ([1cfc222](https://git.lumeweb.com/LumeWeb/portal/commit/1cfc2223a6df614f26fd0337ced68d92e774589f))
* just use the any route ([e100429](https://git.lumeweb.com/LumeWeb/portal/commit/e100429b60e783f6c7c3ddecab7bb9b4dd599726)) * just use the any route ([e100429](https://git.lumeweb.com/LumeWeb/portal/commit/e100429b60e783f6c7c3ddecab7bb9b4dd599726))
* load config before db ([58165e0](https://git.lumeweb.com/LumeWeb/portal/commit/58165e01af9f2b183d654d3d8809cbd1eda0a9bb)) * load awsConfig before db ([58165e0](https://git.lumeweb.com/LumeWeb/portal/commit/58165e01af9f2b183d654d3d8809cbd1eda0a9bb))
* make an attempt to look for the token before adding to db ([f11b285](https://git.lumeweb.com/LumeWeb/portal/commit/f11b285d4e255c1c4c95f6ac15aa904d7a5730e4)) * make an attempt to look for the token before adding to db ([f11b285](https://git.lumeweb.com/LumeWeb/portal/commit/f11b285d4e255c1c4c95f6ac15aa904d7a5730e4))
* missing setting SetTusComposer ([80561f8](https://git.lumeweb.com/LumeWeb/portal/commit/80561f89e92dfa86887ada8361e0046ee6288234)) * missing setting SetTusComposer ([80561f8](https://git.lumeweb.com/LumeWeb/portal/commit/80561f89e92dfa86887ada8361e0046ee6288234))
* newer gorm version causes db rebuilds every boot ([72255eb](https://git.lumeweb.com/LumeWeb/portal/commit/72255eb3c50892aa5f2cfdc4cb1daa5883f0affc)) * newer gorm version causes db rebuilds every boot ([72255eb](https://git.lumeweb.com/LumeWeb/portal/commit/72255eb3c50892aa5f2cfdc4cb1daa5883f0affc))
* only panic if the error is other than a missing config file ([6e0ec8a](https://git.lumeweb.com/LumeWeb/portal/commit/6e0ec8aaf90e86bcb7cb6c8c53f6569e6885e0aa)) * only panic if the error is other than a missing awsConfig file ([6e0ec8a](https://git.lumeweb.com/LumeWeb/portal/commit/6e0ec8aaf90e86bcb7cb6c8c53f6569e6885e0aa))
* output error info ([cfa7ceb](https://git.lumeweb.com/LumeWeb/portal/commit/cfa7ceb2f422a6e594a424315c8eaeffc6572926)) * output error info ([cfa7ceb](https://git.lumeweb.com/LumeWeb/portal/commit/cfa7ceb2f422a6e594a424315c8eaeffc6572926))
* PostPubkeyChallenge should be lowercasing the pubkey for consistency ([d680f06](https://git.lumeweb.com/LumeWeb/portal/commit/d680f0660f910e323356a1169ee13ef2e647a015)) * PostPubkeyChallenge should be lowercasing the pubkey for consistency ([d680f06](https://git.lumeweb.com/LumeWeb/portal/commit/d680f0660f910e323356a1169ee13ef2e647a015))
* PostPubkeyChallenge should be using ChallengeRequest ([36745bb](https://git.lumeweb.com/LumeWeb/portal/commit/36745bb55b1d7cd464b085e410333089504591c1)) * PostPubkeyChallenge should be using ChallengeRequest ([36745bb](https://git.lumeweb.com/LumeWeb/portal/commit/36745bb55b1d7cd464b085e410333089504591c1))

View File

@ -5,6 +5,8 @@ import (
"crypto/ed25519" "crypto/ed25519"
"net/http" "net/http"
"git.lumeweb.com/LumeWeb/portal/config"
"go.uber.org/zap" "go.uber.org/zap"
"github.com/julienschmidt/httprouter" "github.com/julienschmidt/httprouter"
@ -12,7 +14,6 @@ import (
"git.lumeweb.com/LumeWeb/portal/account" "git.lumeweb.com/LumeWeb/portal/account"
"git.lumeweb.com/LumeWeb/portal/api/middleware" "git.lumeweb.com/LumeWeb/portal/api/middleware"
"git.lumeweb.com/LumeWeb/portal/api/registry" "git.lumeweb.com/LumeWeb/portal/api/registry"
"github.com/spf13/viper"
"go.sia.tech/jape" "go.sia.tech/jape"
"go.uber.org/fx" "go.uber.org/fx"
) )
@ -22,7 +23,7 @@ var (
) )
type AccountAPI struct { type AccountAPI struct {
config *viper.Viper config *config.Manager
accounts *account.AccountServiceDefault accounts *account.AccountServiceDefault
identity ed25519.PrivateKey identity ed25519.PrivateKey
logger *zap.Logger logger *zap.Logger
@ -30,7 +31,7 @@ type AccountAPI struct {
type AccountAPIParams struct { type AccountAPIParams struct {
fx.In fx.In
Config *viper.Viper Config *config.Manager
Accounts *account.AccountServiceDefault Accounts *account.AccountServiceDefault
Identity ed25519.PrivateKey Identity ed25519.PrivateKey
Logger *zap.Logger Logger *zap.Logger

View File

@ -4,18 +4,21 @@ import (
"context" "context"
"slices" "slices"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/api/middleware" "git.lumeweb.com/LumeWeb/portal/api/middleware"
"git.lumeweb.com/LumeWeb/portal/api/registry" "git.lumeweb.com/LumeWeb/portal/api/registry"
"github.com/spf13/viper"
"go.uber.org/fx" "go.uber.org/fx"
) )
func BuildApis(config *viper.Viper) fx.Option { var alwaysEnabled = []string{"account"}
func BuildApis(cm *config.Manager) fx.Option {
var options []fx.Option var options []fx.Option
enabledProtocols := config.GetStringSlice("core.protocols") enabledProtocols := cm.Viper().GetStringSlice("core.protocols")
for _, entry := range registry.GetRegistry() { for _, entry := range registry.GetRegistry() {
if slices.Contains(enabledProtocols, entry.Key) { if slices.Contains(enabledProtocols, entry.Key) || slices.Contains(alwaysEnabled, entry.Key) {
options = append(options, entry.Module) options = append(options, entry.Module)
} }
} }
@ -42,7 +45,7 @@ func BuildApis(config *viper.Viper) fx.Option {
if err != nil { if err != nil {
return err return err
} }
middleware.RegisterProtocolSubdomain(config, routes, protocol.Name()) middleware.RegisterProtocolSubdomain(cm, routes, protocol.Name())
} }
return nil return nil

View File

@ -8,11 +8,12 @@ import (
"strconv" "strconv"
"strings" "strings"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/account" "git.lumeweb.com/LumeWeb/portal/account"
"git.lumeweb.com/LumeWeb/portal/api/registry" "git.lumeweb.com/LumeWeb/portal/api/registry"
"github.com/golang-jwt/jwt/v5" "github.com/golang-jwt/jwt/v5"
"github.com/julienschmidt/httprouter" "github.com/julienschmidt/httprouter"
"github.com/spf13/viper"
"go.sia.tech/jape" "go.sia.tech/jape"
) )
@ -64,9 +65,9 @@ func ApplyMiddlewares(handler jape.Handler, middlewares ...interface{}) jape.Han
} }
return handler return handler
} }
func RegisterProtocolSubdomain(config *viper.Viper, mux *httprouter.Router, name string) { func RegisterProtocolSubdomain(config *config.Manager, mux *httprouter.Router, name string) {
router := registry.GetRouter() router := registry.GetRouter()
domain := config.GetString("core.domain") domain := config.Config().Core.Domain
(router)[name+"."+domain] = mux (router)[name+"."+domain] = mux
} }
@ -103,7 +104,7 @@ type AuthMiddlewareOptions struct {
FindToken FindAuthTokenFunc FindToken FindAuthTokenFunc
Purpose account.JWTPurpose Purpose account.JWTPurpose
AuthContextKey string AuthContextKey string
Config *viper.Viper Config *config.Manager
} }
func AuthMiddleware(options AuthMiddlewareOptions) func(http.Handler) http.Handler { func AuthMiddleware(options AuthMiddlewareOptions) func(http.Handler) http.Handler {
@ -114,7 +115,7 @@ func AuthMiddleware(options AuthMiddlewareOptions) func(http.Handler) http.Handl
panic("purpose is missing") panic("purpose is missing")
} }
domain := options.Config.GetString("core.domain") domain := options.Config.Config().Core.Domain
return func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

View File

@ -19,6 +19,8 @@ import (
"strings" "strings"
"time" "time"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/api/swagger" "git.lumeweb.com/LumeWeb/portal/api/swagger"
"git.lumeweb.com/LumeWeb/portal/metadata" "git.lumeweb.com/LumeWeb/portal/metadata"
@ -48,7 +50,6 @@ import (
protoRegistry "git.lumeweb.com/LumeWeb/portal/protocols/registry" protoRegistry "git.lumeweb.com/LumeWeb/portal/protocols/registry"
"git.lumeweb.com/LumeWeb/portal/protocols/s5" "git.lumeweb.com/LumeWeb/portal/protocols/s5"
"github.com/rs/cors" "github.com/rs/cors"
"github.com/spf13/viper"
"go.sia.tech/jape" "go.sia.tech/jape"
"go.uber.org/fx" "go.uber.org/fx"
) )
@ -61,7 +62,7 @@ var (
var swagSpec []byte var swagSpec []byte
type S5API struct { type S5API struct {
config *viper.Viper config *config.Manager
identity ed25519.PrivateKey identity ed25519.PrivateKey
accounts *account.AccountServiceDefault accounts *account.AccountServiceDefault
storage storage.StorageService storage storage.StorageService
@ -75,7 +76,7 @@ type S5API struct {
type APIParams struct { type APIParams struct {
fx.In fx.In
Config *viper.Viper Config *config.Manager
Identity ed25519.PrivateKey Identity ed25519.PrivateKey
Accounts *account.AccountServiceDefault Accounts *account.AccountServiceDefault
Storage storage.StorageService Storage storage.StorageService
@ -355,7 +356,7 @@ func (s *S5API) prepareFileUpload(jc jape.Context) (file io.ReadSeekCloser, s5Er
// Handle multipart form data uploads // Handle multipart form data uploads
if strings.HasPrefix(contentType, "multipart/form-data") { if strings.HasPrefix(contentType, "multipart/form-data") {
if err := r.ParseMultipartForm(s.config.GetInt64("core.post-upload-limit")); err != nil { if err := r.ParseMultipartForm(s.config.Config().Core.PostUploadLimit); err != nil {
return nil, NewS5Error(ErrKeyFileUploadFailed, err) return nil, NewS5Error(ErrKeyFileUploadFailed, err)
} }
@ -745,7 +746,7 @@ func (s *S5API) directoryUpload(jc jape.Context) {
} }
// Parse multipart form with size limit from config // Parse multipart form with size limit from config
if err := jc.Request.ParseMultipartForm(s.config.GetInt64("core.post-upload-limit")); err != nil { if err := jc.Request.ParseMultipartForm(s.config.Config().Core.PostUploadLimit); err != nil {
s.sendErrorResponse(jc, NewS5Error(ErrKeyInvalidOperation, err)) s.sendErrorResponse(jc, NewS5Error(ErrKeyInvalidOperation, err))
return return
} }

View File

@ -4,9 +4,10 @@ import (
"flag" "flag"
"net/http" "net/http"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/account" "git.lumeweb.com/LumeWeb/portal/account"
"git.lumeweb.com/LumeWeb/portal/api" "git.lumeweb.com/LumeWeb/portal/api"
_config "git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/cron" "git.lumeweb.com/LumeWeb/portal/cron"
"git.lumeweb.com/LumeWeb/portal/db" "git.lumeweb.com/LumeWeb/portal/db"
_logger "git.lumeweb.com/LumeWeb/portal/logger" _logger "git.lumeweb.com/LumeWeb/portal/logger"
@ -23,7 +24,7 @@ import (
func main() { func main() {
logger := _logger.NewLogger() logger := _logger.NewLogger()
config, err := _config.NewConfig(logger) cfg, err := config.NewManager(logger)
if err != nil { if err != nil {
logger.Fatal("Failed to load config", zap.Error(err)) logger.Fatal("Failed to load config", zap.Error(err))
@ -48,7 +49,7 @@ func main() {
} }
fx.New( fx.New(
fx.Supply(config), fx.Supply(cfg),
fx.Supply(logger), fx.Supply(logger),
fxLogger, fxLogger,
fx.Invoke(initCheckRequiredConfig), fx.Invoke(initCheckRequiredConfig),
@ -59,8 +60,8 @@ func main() {
cron.Module, cron.Module,
account.Module, account.Module,
metadata.Module, metadata.Module,
protocols.BuildProtocols(config), protocols.BuildProtocols(cfg),
api.BuildApis(config), api.BuildApis(cfg),
fx.Provide(api.NewCasbin), fx.Provide(api.NewCasbin),
fx.Invoke(protocols.SetupLifecycles), fx.Invoke(protocols.SetupLifecycles),
fx.Invoke(api.SetupLifecycles), fx.Invoke(api.SetupLifecycles),

View File

@ -2,7 +2,10 @@ package config
import ( import (
"errors" "errors"
"fmt"
_logger "git.lumeweb.com/LumeWeb/portal/logger" _logger "git.lumeweb.com/LumeWeb/portal/logger"
"github.com/docker/go-units"
"github.com/spf13/viper" "github.com/spf13/viper"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -15,7 +18,106 @@ var (
} }
) )
func NewConfig(logger *zap.Logger) (*viper.Viper, error) { type Config struct {
Core CoreConfig `mapstructure:"core"`
}
type Manager struct {
viper *viper.Viper
root *Config
changes bool
}
func NewManager(logger *zap.Logger) (*Manager, error) {
v, err := newConfig(logger)
if err != nil {
return nil, err
}
var config Config
m := &Manager{
viper: v,
root: &config,
}
m.setDefaults(m.coreDefaults(), "")
err = m.maybeSave()
if err != nil {
return nil, err
}
err = v.Unmarshal(config)
if err != nil {
return nil, err
}
return nil, nil
}
func (m *Manager) ConfigureProtocol(name string, cfg ProtocolConfig) error {
defaults := cfg.Defaults()
m.setDefaults(defaults, fmt.Sprintf("protocol.%s", name))
err := m.maybeSave()
if err != nil {
return err
}
return m.viper.Unmarshal(cfg)
}
func (m *Manager) setDefaults(defaults map[string]interface{}, prefix string) {
for key, value := range defaults {
if prefix != "" {
key = fmt.Sprintf("%s.%s", prefix, key)
}
if m.setDefault(key, value) {
m.changes = true
}
}
}
func (m *Manager) setDefault(key string, value interface{}) bool {
if !m.viper.IsSet(key) {
m.viper.SetDefault(key, value)
return true
}
return false
}
func (m *Manager) maybeSave() error {
if m.changes {
ret := m.viper.WriteConfig()
if ret != nil {
return ret
}
m.changes = false
}
return nil
}
func (m *Manager) coreDefaults() map[string]interface{} {
return map[string]interface{}{
"core.post-upload-limit": units.MiB * 100,
"core.log.level": "info",
"core.db.charset": "utf8mb4",
"core.db.port": 3306,
"core.db.name": "portal",
}
}
func (m *Manager) Config() *Config {
return m.root
}
func (m *Manager) Viper() *viper.Viper {
return m.viper
}
func newConfig(logger *zap.Logger) (*viper.Viper, error) {
if logger == nil { if logger == nil {
logger = _logger.NewFallbackLogger() logger = _logger.NewFallbackLogger()
} }
@ -32,64 +134,19 @@ func NewConfig(logger *zap.Logger) (*viper.Viper, error) {
err := viper.ReadInConfig() err := viper.ReadInConfig()
if err != nil { if err != nil {
if errors.As(err, &viper.ConfigFileNotFoundError{}) { if !errors.Is(err, &viper.ConfigFileNotFoundError{}) {
logger.Info("Config file not found, using default settings.") return nil, err
err := viper.SafeWriteConfig()
if err != nil {
return nil, err
}
err = writeDefaults()
if err != nil {
return nil, err
}
return viper.GetViper(), nil
} }
return nil, err
}
err = writeDefaults() logger.Info("Config file not found, using default settings.")
if err != nil { err := viper.SafeWriteConfig()
return nil, err if err != nil {
return nil, err
}
return viper.GetViper(), nil
} }
return viper.GetViper(), nil return viper.GetViper(), nil
} }
func writeDefaults() error {
defaults := map[string]interface{}{
"core.post-upload-limit": 1024 * 1024 * 1000,
"core.log.level": "info",
"core.db.charset": "utf8mb4",
"core.db.port": 3306,
"core.db.name": "portal",
"protocol.s5.p2p.maxOutgoingPeerFailures": 10,
"protocol.s5.p2p.network": "",
}
changes := false
for key, value := range defaults {
if writeDefault(key, value) {
changes = true
}
}
if changes {
err := viper.WriteConfig()
if err != nil {
return err
}
}
return nil
}
func writeDefault(key string, value interface{}) bool {
if !viper.IsSet(key) {
viper.SetDefault(key, value)
return true
}
return false
}

14
config/core.go Normal file
View File

@ -0,0 +1,14 @@
package config
type CoreConfig struct {
DB DatabaseConfig `mapstructure:"db"`
Domain string `mapstructure:"domain"`
ExternalPort uint `mapstructure:"external_port"`
Identity string `mapstructure:"identity"`
Log LogConfig `mapstructure:"log"`
Port uint `mapstructure:"port"`
PostUploadLimit int64 `mapstructure:"post_upload_limit"`
Sia SiaConfig `mapstructure:"sia"`
Storage StorageConfig `mapstructure:"storage"`
Protocols []string `mapstructure:"protocols"`
}

10
config/database.go Normal file
View File

@ -0,0 +1,10 @@
package config
type DatabaseConfig struct {
Charset string `mapstructure:"charset"`
Host string `mapstructure:"host"`
Name string `mapstructure:"name"`
Password string `mapstructure:"password"`
Port int `mapstructure:"port"`
Username string `mapstructure:"username"`
}

5
config/log.go Normal file
View File

@ -0,0 +1,5 @@
package config
type LogConfig struct {
Level string `mapstructure:"level"`
}

5
config/protocol.go Normal file
View File

@ -0,0 +1,5 @@
package config
type ProtocolConfig interface {
Defaults() map[string]interface{}
}

9
config/s3.go Normal file
View File

@ -0,0 +1,9 @@
package config
type S3Config struct {
BufferBucket string `mapstructure:"buffer_bucket"`
Endpoint string `mapstructure:"endpoint"`
Region string `mapstructure:"region"`
AccessKey string `mapstructure:"access_key"`
SecretKey string `mapstructure:"secret_key"`
}

6
config/sia.go Normal file
View File

@ -0,0 +1,6 @@
package config
type SiaConfig struct {
Key string `mapstructure:"key"`
URL string `mapstructure:"url"`
}

5
config/storage.go Normal file
View File

@ -0,0 +1,5 @@
package config
type StorageConfig struct {
S3 S3Config `mapstructure:"s3"`
}

2
go.mod
View File

@ -8,7 +8,7 @@ require (
git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240201012059-dfeb8b29a8e4 git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240201012059-dfeb8b29a8e4
github.com/AfterShip/email-verifier v1.4.0 github.com/AfterShip/email-verifier v1.4.0
github.com/aws/aws-sdk-go-v2 v1.24.0 github.com/aws/aws-sdk-go-v2 v1.24.0
github.com/aws/aws-sdk-go-v2/config v1.26.2 github.com/aws/aws-sdk-go-v2/cm v1.26.2
github.com/aws/aws-sdk-go-v2/credentials v1.16.13 github.com/aws/aws-sdk-go-v2/credentials v1.16.13
github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7 github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7
github.com/casbin/casbin/v2 v2.81.0 github.com/casbin/casbin/v2 v2.81.0

4
go.sum
View File

@ -24,8 +24,8 @@ github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7Aw
github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
github.com/aws/aws-sdk-go-v2/config v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc= github.com/aws/aws-sdk-go-v2/cm v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc=
github.com/aws/aws-sdk-go-v2/config v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8= github.com/aws/aws-sdk-go-v2/cm v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8=
github.com/aws/aws-sdk-go-v2/credentials v1.16.13 h1:WLABQ4Cp4vXtXfOWOS3MEZKr6AAYUpMczLhgKtAjQ/8= github.com/aws/aws-sdk-go-v2/credentials v1.16.13 h1:WLABQ4Cp4vXtXfOWOS3MEZKr6AAYUpMczLhgKtAjQ/8=
github.com/aws/aws-sdk-go-v2/credentials v1.16.13/go.mod h1:Qg6x82FXwW0sJHzYruxGiuApNo31UEtJvXVSZAXeWiw= github.com/aws/aws-sdk-go-v2/credentials v1.16.13/go.mod h1:Qg6x82FXwW0sJHzYruxGiuApNo31UEtJvXVSZAXeWiw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58=

View File

@ -3,15 +3,16 @@ package protocols
import ( import (
"context" "context"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/protocols/registry" "git.lumeweb.com/LumeWeb/portal/protocols/registry"
"github.com/samber/lo" "github.com/samber/lo"
"github.com/spf13/viper"
"go.uber.org/fx" "go.uber.org/fx"
) )
func BuildProtocols(config *viper.Viper) fx.Option { func BuildProtocols(cm *config.Manager) fx.Option {
var options []fx.Option var options []fx.Option
enabledProtocols := config.GetStringSlice("core.protocols") enabledProtocols := cm.Viper().GetStringSlice("core.protocols")
for _, entry := range registry.GetRegistry() { for _, entry := range registry.GetRegistry() {
if lo.Contains(enabledProtocols, entry.Key) { if lo.Contains(enabledProtocols, entry.Key) {
options = append(options, entry.Module) options = append(options, entry.Module)
@ -28,7 +29,12 @@ func BuildProtocols(config *viper.Viper) fx.Option {
options = append(options, fx.Invoke(func(params initParams) error { options = append(options, fx.Invoke(func(params initParams) error {
for _, protocol := range params.Protocols { for _, protocol := range params.Protocols {
err := protocol.Init() err := cm.ConfigureProtocol(protocol.Name(), protocol.Config())
if err != nil {
return err
}
err = protocol.Init()
if err != nil { if err != nil {
return err return err
} }

View File

@ -3,6 +3,8 @@ package registry
import ( import (
"context" "context"
"git.lumeweb.com/LumeWeb/portal/config"
"go.uber.org/fx" "go.uber.org/fx"
) )
@ -13,6 +15,7 @@ type Protocol interface {
Init() error Init() error
Start(ctx context.Context) error Start(ctx context.Context) error
Stop(ctx context.Context) error Stop(ctx context.Context) error
Config() config.ProtocolConfig
} }
type ProtocolEntry struct { type ProtocolEntry struct {

26
protocols/s5/config.go Normal file
View File

@ -0,0 +1,26 @@
package s5
import (
s5config "git.lumeweb.com/LumeWeb/libs5-go/config"
"git.lumeweb.com/LumeWeb/portal/config"
)
var _ config.ProtocolConfig = (*Config)(nil)
type Config struct {
s5config.NodeConfig
DbPath string `mapstructure:"db_path"`
}
func (c Config) Defaults() map[string]interface{} {
defaults := map[string]interface{}{}
defaults["p2p.network.peers"] = []string{
"ss://z2DWuWNZcdSyZLpXFK2uCU3haaWMXrDAgxzv17sDEMHstZb@s5.garden/s5/p2p",
"wss://z2DWuPbL5pweybXnEB618pMnV58ECj2VPDNfVGm3tFqBvjF@s5.ninja/s5/p2p",
}
defaults["db_path"] = "s5.db"
return defaults
}

View File

@ -7,6 +7,8 @@ import (
"fmt" "fmt"
"time" "time"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/metadata" "git.lumeweb.com/LumeWeb/portal/metadata"
"git.lumeweb.com/LumeWeb/portal/storage" "git.lumeweb.com/LumeWeb/portal/storage"
@ -32,18 +34,19 @@ var (
) )
type S5Protocol struct { type S5Protocol struct {
config *viper.Viper portalConfig *config.Manager
logger *zap.Logger config *Config
storage storage.StorageService logger *zap.Logger
identity ed25519.PrivateKey storage storage.StorageService
node *s5node.Node identity ed25519.PrivateKey
tusHandler *TusHandler node *s5node.Node
store *S5ProviderStore tusHandler *TusHandler
store *S5ProviderStore
} }
type S5ProtocolParams struct { type S5ProtocolParams struct {
fx.In fx.In
Config *viper.Viper PortalConfig *config.Manager
Logger *zap.Logger Logger *zap.Logger
Storage storage.StorageService Storage storage.StorageService
Identity ed25519.PrivateKey Identity ed25519.PrivateKey
@ -85,15 +88,15 @@ func NewS5Protocol(
params S5ProtocolParams, params S5ProtocolParams,
) (S5ProtocolResult, error) { ) (S5ProtocolResult, error) {
proto := &S5Protocol{ proto := &S5Protocol{
config: params.Config, portalConfig: params.PortalConfig,
logger: params.Logger, logger: params.Logger,
storage: params.Storage, storage: params.Storage,
identity: params.Identity, identity: params.Identity,
tusHandler: params.TusHandler, tusHandler: params.TusHandler,
store: params.ProviderStore, store: params.ProviderStore,
} }
cfg, err := ConfigureS5Protocol(params) cfg, err := configureS5Protocol(proto)
if err != nil { if err != nil {
return S5ProtocolResult{}, err return S5ProtocolResult{}, err
} }
@ -106,60 +109,53 @@ func NewS5Protocol(
}, nil }, nil
} }
func ConfigureS5Protocol(params S5ProtocolParams) (*s5config.NodeConfig, error) { func configureS5Protocol(proto *S5Protocol) (*s5config.NodeConfig, error) {
cfg := &s5config.NodeConfig{ cfg := proto.Config().(*Config)
P2P: s5config.P2PConfig{ cm := proto.portalConfig
Network: "", portalCfg := cm.Config()
Peers: s5config.PeersConfig{Initial: []string{}}, vpr := cm.Viper()
},
KeyPair: s5ed.New(params.Identity),
DB: nil,
Logger: params.Logger.Named("s5"),
HTTP: s5config.HTTPConfig{},
}
pconfig := params.Config.Sub("protocol.s5") err := cm.ConfigureProtocol(proto.Name(), cfg)
if pconfig == nil {
params.Logger.Fatal("Missing protocol.s5 Config")
}
err := pconfig.Unmarshal(cfg)
if err != nil { if err != nil {
return nil, err return nil, err
} }
cfg.HTTP.API.Domain = fmt.Sprintf("s5.%s", params.Config.GetString("core.domain")) cfg.HTTP.API.Domain = fmt.Sprintf("s5.%s", vpr.GetString("core.domain"))
if params.Config.IsSet("core.externalPort") { if portalCfg.Core.ExternalPort != 0 {
cfg.HTTP.API.Port = params.Config.GetUint("core.externalPort") cfg.HTTP.API.Port = portalCfg.Core.ExternalPort
} else { } else {
cfg.HTTP.API.Port = params.Config.GetUint("core.port") cfg.HTTP.API.Port = portalCfg.Core.Port
} }
dbPath := pconfig.GetString("dbPath") if cfg.DbPath == "" {
proto.logger.Fatal("protocol.s5.dbPath is required")
if dbPath == "" {
params.Logger.Fatal("protocol.s5.dbPath is required")
} }
_, p, err := ed25519.GenerateKey(nil) _, p, err := ed25519.GenerateKey(nil)
if err != nil { if err != nil {
params.Logger.Fatal("Failed to generate key", zap.Error(err)) proto.logger.Fatal("Failed to generate key", zap.Error(err))
} }
cfg.KeyPair = s5ed.New(p) cfg.KeyPair = s5ed.New(p)
db, err := bolt.Open(dbPath, 0600, nil) db, err := bolt.Open(cfg.DbPath, 0600, nil)
if err != nil { if err != nil {
params.Logger.Fatal("Failed to open db", zap.Error(err)) proto.logger.Fatal("Failed to open db", zap.Error(err))
} }
cfg.DB = db cfg.DB = db
return cfg, nil return interface{}(cfg).(*s5config.NodeConfig), nil
} }
func (s *S5Protocol) Config() config.ProtocolConfig {
if s.config == nil {
s.config = &Config{}
}
return s.config
}
func NewS5ProviderStore(params S5ProviderStoreParams) *S5ProviderStore { func NewS5ProviderStore(params S5ProviderStoreParams) *S5ProviderStore {
return &S5ProviderStore{ return &S5ProviderStore{
config: params.Config, config: params.Config,

View File

@ -11,16 +11,15 @@ import (
"time" "time"
"git.lumeweb.com/LumeWeb/portal/api/middleware" "git.lumeweb.com/LumeWeb/portal/api/middleware"
"git.lumeweb.com/LumeWeb/portal/config"
"go.uber.org/fx" "go.uber.org/fx"
"git.lumeweb.com/LumeWeb/portal/account" "git.lumeweb.com/LumeWeb/portal/account"
"github.com/spf13/viper"
"git.lumeweb.com/LumeWeb/portal/metadata" "git.lumeweb.com/LumeWeb/portal/metadata"
"github.com/aws/aws-sdk-go-v2/config" awsConfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/credentials"
"github.com/tus/tusd/v2/pkg/s3store" "github.com/tus/tusd/v2/pkg/s3store"
@ -45,7 +44,7 @@ var (
) )
type TusHandler struct { type TusHandler struct {
config *viper.Viper config *config.Manager
db *gorm.DB db *gorm.DB
logger *zap.Logger logger *zap.Logger
cron *cron.CronServiceDefault cron *cron.CronServiceDefault
@ -60,7 +59,7 @@ type TusHandler struct {
type TusHandlerParams struct { type TusHandlerParams struct {
fx.In fx.In
Config *viper.Viper Config *config.Manager
Logger *zap.Logger Logger *zap.Logger
Db *gorm.DB Db *gorm.DB
Cron *cron.CronServiceDefault Cron *cron.CronServiceDefault
@ -127,21 +126,21 @@ func (t *TusHandler) Init() error {
customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
if service == s3.ServiceID { if service == s3.ServiceID {
return aws.Endpoint{ return aws.Endpoint{
URL: t.config.GetString("core.storage.s3.endpoint"), URL: t.config.Config().Core.Storage.S3.Endpoint,
SigningRegion: t.config.GetString("core.storage.s3.region"), SigningRegion: t.config.Config().Core.Storage.S3.Region,
}, nil }, nil
} }
return aws.Endpoint{}, &aws.EndpointNotFoundError{} return aws.Endpoint{}, &aws.EndpointNotFoundError{}
}) })
cfg, err := config.LoadDefaultConfig(context.TODO(), cfg, err := awsConfig.LoadDefaultConfig(context.TODO(),
config.WithRegion("us-east-1"), awsConfig.WithRegion("us-east-1"),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( awsConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
t.config.GetString("core.storage.s3.accessKey"), t.config.Config().Core.Storage.S3.AccessKey,
t.config.GetString("core.storage.s3.secretKey"), t.config.Config().Core.Storage.S3.SecretKey,
"", "",
)), )),
config.WithEndpointResolverWithOptions(customResolver), awsConfig.WithEndpointResolverWithOptions(customResolver),
) )
if err != nil { if err != nil {
return err return err
@ -149,7 +148,7 @@ func (t *TusHandler) Init() error {
s3Client := s3.NewFromConfig(cfg) s3Client := s3.NewFromConfig(cfg)
store := s3store.New(t.config.GetString("core.storage.s3.bufferBucket"), s3Client) store := s3store.New(t.config.Config().Core.Storage.S3.BufferBucket, s3Client)
locker := NewMySQLLocker(t.db, t.logger) locker := NewMySQLLocker(t.db, t.logger)
@ -407,7 +406,7 @@ func (t *TusHandler) uploadTask(hash []byte) error {
s3InfoId, _ := splitS3Ids(upload.UploadID) s3InfoId, _ := splitS3Ids(upload.UploadID)
_, err = t.s3Client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ _, err = t.s3Client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
Bucket: aws.String(t.config.GetString("core.storage.s3.bufferBucket")), Bucket: aws.String(t.config.Config().Core.Storage.S3.BufferBucket),
Delete: &s3types.Delete{ Delete: &s3types.Delete{
Objects: []s3types.ObjectIdentifier{ Objects: []s3types.ObjectIdentifier{
{ {