LumeWeb
/
portal
1
0
Fork 0
Code Issues 5 Pull Requests Packages Projects Releases Wiki Activity

refactor: implement new configuration management system

This commit is contained in:
Derrick Hammer 2024-02-22 02:09:31 -05:00
parent 99b97d9495
commit 7f12ee5b0d
Signed by: pcfreak30
GPG Key ID: C997C339BE476FF2
21 changed files with 296 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -40,11 +40,11 @@
* if we have an existing upload, just return it as if successful ([90170e5](https://git.lumeweb.com/LumeWeb/portal/commit/90170e5b81831f3d768291fd37c7c13e32d522fe))
* iris context.User needs to be embedded in our User struct for type checking to properly work ([1cfc222](https://git.lumeweb.com/LumeWeb/portal/commit/1cfc2223a6df614f26fd0337ced68d92e774589f))
* just use the any route ([e100429](https://git.lumeweb.com/LumeWeb/portal/commit/e100429b60e783f6c7c3ddecab7bb9b4dd599726))
* load config before db ([58165e0](https://git.lumeweb.com/LumeWeb/portal/commit/58165e01af9f2b183d654d3d8809cbd1eda0a9bb))
* load awsConfig before db ([58165e0](https://git.lumeweb.com/LumeWeb/portal/commit/58165e01af9f2b183d654d3d8809cbd1eda0a9bb))
* make an attempt to look for the token before adding to db ([f11b285](https://git.lumeweb.com/LumeWeb/portal/commit/f11b285d4e255c1c4c95f6ac15aa904d7a5730e4))
* missing setting SetTusComposer ([80561f8](https://git.lumeweb.com/LumeWeb/portal/commit/80561f89e92dfa86887ada8361e0046ee6288234))
* newer gorm version causes db rebuilds every boot ([72255eb](https://git.lumeweb.com/LumeWeb/portal/commit/72255eb3c50892aa5f2cfdc4cb1daa5883f0affc))
* only panic if the error is other than a missing config file ([6e0ec8a](https://git.lumeweb.com/LumeWeb/portal/commit/6e0ec8aaf90e86bcb7cb6c8c53f6569e6885e0aa))
* only panic if the error is other than a missing awsConfig file ([6e0ec8a](https://git.lumeweb.com/LumeWeb/portal/commit/6e0ec8aaf90e86bcb7cb6c8c53f6569e6885e0aa))
* output error info ([cfa7ceb](https://git.lumeweb.com/LumeWeb/portal/commit/cfa7ceb2f422a6e594a424315c8eaeffc6572926))
* PostPubkeyChallenge should be lowercasing the pubkey for consistency ([d680f06](https://git.lumeweb.com/LumeWeb/portal/commit/d680f0660f910e323356a1169ee13ef2e647a015))
* PostPubkeyChallenge should be using ChallengeRequest ([36745bb](https://git.lumeweb.com/LumeWeb/portal/commit/36745bb55b1d7cd464b085e410333089504591c1))

7
api/account/account.go
View File

@ -5,6 +5,8 @@ import (
"crypto/ed25519"
"net/http"
"git.lumeweb.com/LumeWeb/portal/config"
"go.uber.org/zap"
"github.com/julienschmidt/httprouter"
@ -12,7 +14,6 @@ import (
"git.lumeweb.com/LumeWeb/portal/account"
"git.lumeweb.com/LumeWeb/portal/api/middleware"
"git.lumeweb.com/LumeWeb/portal/api/registry"
"github.com/spf13/viper"
"go.sia.tech/jape"
"go.uber.org/fx"
)
@ -22,7 +23,7 @@ var (
)
type AccountAPI struct {
config *viper.Viper
config *config.Manager
accounts *account.AccountServiceDefault
identity ed25519.PrivateKey
logger *zap.Logger
@ -30,7 +31,7 @@ type AccountAPI struct {
type AccountAPIParams struct {
fx.In
Config *viper.Viper
Config *config.Manager
Accounts *account.AccountServiceDefault
Identity ed25519.PrivateKey
Logger *zap.Logger

13
api/api.go
View File

@ -4,18 +4,21 @@ import (
"context"
"slices"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/api/middleware"
"git.lumeweb.com/LumeWeb/portal/api/registry"
"github.com/spf13/viper"
"go.uber.org/fx"
)
func BuildApis(config *viper.Viper) fx.Option {
var alwaysEnabled = []string{"account"}
func BuildApis(cm *config.Manager) fx.Option {
var options []fx.Option
enabledProtocols := config.GetStringSlice("core.protocols")
enabledProtocols := cm.Viper().GetStringSlice("core.protocols")
for _, entry := range registry.GetRegistry() {
if slices.Contains(enabledProtocols, entry.Key) {
if slices.Contains(enabledProtocols, entry.Key) || slices.Contains(alwaysEnabled, entry.Key) {
options = append(options, entry.Module)
}
}
@ -42,7 +45,7 @@ func BuildApis(config *viper.Viper) fx.Option {
if err != nil {
return err
}
middleware.RegisterProtocolSubdomain(config, routes, protocol.Name())
middleware.RegisterProtocolSubdomain(cm, routes, protocol.Name())
}
return nil

11
api/middleware/middleware.go
View File

@ -8,11 +8,12 @@ import (
"strconv"
"strings"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/account"
"git.lumeweb.com/LumeWeb/portal/api/registry"
"github.com/golang-jwt/jwt/v5"
"github.com/julienschmidt/httprouter"
"github.com/spf13/viper"
"go.sia.tech/jape"
)
@ -64,9 +65,9 @@ func ApplyMiddlewares(handler jape.Handler, middlewares ...interface{}) jape.Han
}
return handler
}
func RegisterProtocolSubdomain(config *viper.Viper, mux *httprouter.Router, name string) {
func RegisterProtocolSubdomain(config *config.Manager, mux *httprouter.Router, name string) {
router := registry.GetRouter()
domain := config.GetString("core.domain")
domain := config.Config().Core.Domain
(router)[name+"."+domain] = mux
}
@ -103,7 +104,7 @@ type AuthMiddlewareOptions struct {
FindToken FindAuthTokenFunc
Purpose account.JWTPurpose
AuthContextKey string
Config *viper.Viper
Config *config.Manager
}
func AuthMiddleware(options AuthMiddlewareOptions) func(http.Handler) http.Handler {
@ -114,7 +115,7 @@ func AuthMiddleware(options AuthMiddlewareOptions) func(http.Handler) http.Handl
panic("purpose is missing")
}
domain := options.Config.GetString("core.domain")
domain := options.Config.Config().Core.Domain
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

11
api/s5/s5.go
View File

@ -19,6 +19,8 @@ import (
"strings"
"time"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/api/swagger"
"git.lumeweb.com/LumeWeb/portal/metadata"
@ -48,7 +50,6 @@ import (
protoRegistry "git.lumeweb.com/LumeWeb/portal/protocols/registry"
"git.lumeweb.com/LumeWeb/portal/protocols/s5"
"github.com/rs/cors"
"github.com/spf13/viper"
"go.sia.tech/jape"
"go.uber.org/fx"
)
@ -61,7 +62,7 @@ var (
var swagSpec []byte
type S5API struct {
config *viper.Viper
config *config.Manager
identity ed25519.PrivateKey
accounts *account.AccountServiceDefault
storage storage.StorageService
@ -75,7 +76,7 @@ type S5API struct {
type APIParams struct {
fx.In
Config *viper.Viper
Config *config.Manager
Identity ed25519.PrivateKey
Accounts *account.AccountServiceDefault
Storage storage.StorageService
@ -355,7 +356,7 @@ func (s *S5API) prepareFileUpload(jc jape.Context) (file io.ReadSeekCloser, s5Er
// Handle multipart form data uploads
if strings.HasPrefix(contentType, "multipart/form-data") {
if err := r.ParseMultipartForm(s.config.GetInt64("core.post-upload-limit")); err != nil {
if err := r.ParseMultipartForm(s.config.Config().Core.PostUploadLimit); err != nil {
return nil, NewS5Error(ErrKeyFileUploadFailed, err)
}
@ -745,7 +746,7 @@ func (s *S5API) directoryUpload(jc jape.Context) {
}
// Parse multipart form with size limit from config
if err := jc.Request.ParseMultipartForm(s.config.GetInt64("core.post-upload-limit")); err != nil {
if err := jc.Request.ParseMultipartForm(s.config.Config().Core.PostUploadLimit); err != nil {
s.sendErrorResponse(jc, NewS5Error(ErrKeyInvalidOperation, err))
return
}

11
cmd/portal/main.go
View File

@ -4,9 +4,10 @@ import (
"flag"
"net/http"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/account"
"git.lumeweb.com/LumeWeb/portal/api"
_config "git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/cron"
"git.lumeweb.com/LumeWeb/portal/db"
_logger "git.lumeweb.com/LumeWeb/portal/logger"
@ -23,7 +24,7 @@ import (
func main() {
logger := _logger.NewLogger()
config, err := _config.NewConfig(logger)
cfg, err := config.NewManager(logger)
if err != nil {
logger.Fatal("Failed to load config", zap.Error(err))
@ -48,7 +49,7 @@ func main() {
}
fx.New(
fx.Supply(config),
fx.Supply(cfg),
fx.Supply(logger),
fxLogger,
fx.Invoke(initCheckRequiredConfig),
@ -59,8 +60,8 @@ func main() {
cron.Module,
account.Module,
metadata.Module,
protocols.BuildProtocols(config),
api.BuildApis(config),
protocols.BuildProtocols(cfg),
api.BuildApis(cfg),
fx.Provide(api.NewCasbin),
fx.Invoke(protocols.SetupLifecycles),
fx.Invoke(api.SetupLifecycles),

169
config/config.go
View File

@ -2,7 +2,10 @@ package config
import (
"errors"
"fmt"
_logger "git.lumeweb.com/LumeWeb/portal/logger"
"github.com/docker/go-units"
"github.com/spf13/viper"
"go.uber.org/zap"
)
@ -15,7 +18,106 @@ var (
}
)
func NewConfig(logger *zap.Logger) (*viper.Viper, error) {
type Config struct {
Core CoreConfig `mapstructure:"core"`
}
type Manager struct {
viper *viper.Viper
root *Config
changes bool
}
func NewManager(logger *zap.Logger) (*Manager, error) {
v, err := newConfig(logger)
if err != nil {
return nil, err
}
var config Config
m := &Manager{
viper: v,
root: &config,
}
m.setDefaults(m.coreDefaults(), "")
err = m.maybeSave()
if err != nil {
return nil, err
}
err = v.Unmarshal(config)
if err != nil {
return nil, err
}
return nil, nil
}
func (m *Manager) ConfigureProtocol(name string, cfg ProtocolConfig) error {
defaults := cfg.Defaults()
m.setDefaults(defaults, fmt.Sprintf("protocol.%s", name))
err := m.maybeSave()
if err != nil {
return err
}
return m.viper.Unmarshal(cfg)
}
func (m *Manager) setDefaults(defaults map[string]interface{}, prefix string) {
for key, value := range defaults {
if prefix != "" {
key = fmt.Sprintf("%s.%s", prefix, key)
}
if m.setDefault(key, value) {
m.changes = true
}
}
}
func (m *Manager) setDefault(key string, value interface{}) bool {
if !m.viper.IsSet(key) {
m.viper.SetDefault(key, value)
return true
}
return false
}
func (m *Manager) maybeSave() error {
if m.changes {
ret := m.viper.WriteConfig()
if ret != nil {
return ret
}
m.changes = false
}
return nil
}
func (m *Manager) coreDefaults() map[string]interface{} {
return map[string]interface{}{
"core.post-upload-limit": units.MiB * 100,
"core.log.level": "info",
"core.db.charset": "utf8mb4",
"core.db.port": 3306,
"core.db.name": "portal",
}
}
func (m *Manager) Config() *Config {
return m.root
}
func (m *Manager) Viper() *viper.Viper {
return m.viper
}
func newConfig(logger *zap.Logger) (*viper.Viper, error) {
if logger == nil {
logger = _logger.NewFallbackLogger()
}
@ -32,64 +134,19 @@ func NewConfig(logger *zap.Logger) (*viper.Viper, error) {
err := viper.ReadInConfig()
if err != nil {
if errors.As(err, &viper.ConfigFileNotFoundError{}) {
logger.Info("Config file not found, using default settings.")
err := viper.SafeWriteConfig()
if err != nil {
return nil, err
}
err = writeDefaults()
if err != nil {
return nil, err
}
return viper.GetViper(), nil
if !errors.Is(err, &viper.ConfigFileNotFoundError{}) {
return nil, err
}
return nil, err
}
err = writeDefaults()
if err != nil {
return nil, err
logger.Info("Config file not found, using default settings.")
err := viper.SafeWriteConfig()
if err != nil {
return nil, err
}
return viper.GetViper(), nil
}
return viper.GetViper(), nil
}
func writeDefaults() error {
defaults := map[string]interface{}{
"core.post-upload-limit": 1024 * 1024 * 1000,
"core.log.level": "info",
"core.db.charset": "utf8mb4",
"core.db.port": 3306,
"core.db.name": "portal",
"protocol.s5.p2p.maxOutgoingPeerFailures": 10,
"protocol.s5.p2p.network": "",
}
changes := false
for key, value := range defaults {
if writeDefault(key, value) {
changes = true
}
}
if changes {
err := viper.WriteConfig()
if err != nil {
return err
}
}
return nil
}
func writeDefault(key string, value interface{}) bool {
if !viper.IsSet(key) {
viper.SetDefault(key, value)
return true
}
return false
}

14
config/core.go Normal file
View File

@ -0,0 +1,14 @@
package config
type CoreConfig struct {
DB DatabaseConfig `mapstructure:"db"`
Domain string `mapstructure:"domain"`
ExternalPort uint `mapstructure:"external_port"`
Identity string `mapstructure:"identity"`
Log LogConfig `mapstructure:"log"`
Port uint `mapstructure:"port"`
PostUploadLimit int64 `mapstructure:"post_upload_limit"`
Sia SiaConfig `mapstructure:"sia"`
Storage StorageConfig `mapstructure:"storage"`
Protocols []string `mapstructure:"protocols"`
}

10
config/database.go Normal file
View File

@ -0,0 +1,10 @@
package config
type DatabaseConfig struct {
Charset string `mapstructure:"charset"`
Host string `mapstructure:"host"`
Name string `mapstructure:"name"`
Password string `mapstructure:"password"`
Port int `mapstructure:"port"`
Username string `mapstructure:"username"`
}

5
config/log.go Normal file
View File

@ -0,0 +1,5 @@
package config
type LogConfig struct {
Level string `mapstructure:"level"`
}

5
config/protocol.go Normal file
View File

@ -0,0 +1,5 @@
package config
type ProtocolConfig interface {
Defaults() map[string]interface{}
}

9
config/s3.go Normal file
View File

@ -0,0 +1,9 @@
package config
type S3Config struct {
BufferBucket string `mapstructure:"buffer_bucket"`
Endpoint string `mapstructure:"endpoint"`
Region string `mapstructure:"region"`
AccessKey string `mapstructure:"access_key"`
SecretKey string `mapstructure:"secret_key"`
}

6
config/sia.go Normal file
View File

@ -0,0 +1,6 @@
package config
type SiaConfig struct {
Key string `mapstructure:"key"`
URL string `mapstructure:"url"`
}

5
config/storage.go Normal file
View File

@ -0,0 +1,5 @@
package config
type StorageConfig struct {
S3 S3Config `mapstructure:"s3"`
}

2
go.mod
View File

@ -8,7 +8,7 @@ require (
git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240201012059-dfeb8b29a8e4
github.com/AfterShip/email-verifier v1.4.0
github.com/aws/aws-sdk-go-v2 v1.24.0
github.com/aws/aws-sdk-go-v2/config v1.26.2
github.com/aws/aws-sdk-go-v2/cm v1.26.2
github.com/aws/aws-sdk-go-v2/credentials v1.16.13
github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7
github.com/casbin/casbin/v2 v2.81.0

4
go.sum
View File

@ -24,8 +24,8 @@ github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7Aw
github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
github.com/aws/aws-sdk-go-v2/config v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc=
github.com/aws/aws-sdk-go-v2/config v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8=
github.com/aws/aws-sdk-go-v2/cm v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc=
github.com/aws/aws-sdk-go-v2/cm v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8=
github.com/aws/aws-sdk-go-v2/credentials v1.16.13 h1:WLABQ4Cp4vXtXfOWOS3MEZKr6AAYUpMczLhgKtAjQ/8=
github.com/aws/aws-sdk-go-v2/credentials v1.16.13/go.mod h1:Qg6x82FXwW0sJHzYruxGiuApNo31UEtJvXVSZAXeWiw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58=

14
protocols/protocols.go
View File

@ -3,15 +3,16 @@ package protocols
import (
"context"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/protocols/registry"
"github.com/samber/lo"
"github.com/spf13/viper"
"go.uber.org/fx"
)
func BuildProtocols(config *viper.Viper) fx.Option {
func BuildProtocols(cm *config.Manager) fx.Option {
var options []fx.Option
enabledProtocols := config.GetStringSlice("core.protocols")
enabledProtocols := cm.Viper().GetStringSlice("core.protocols")
for _, entry := range registry.GetRegistry() {
if lo.Contains(enabledProtocols, entry.Key) {
options = append(options, entry.Module)
@ -28,7 +29,12 @@ func BuildProtocols(config *viper.Viper) fx.Option {
options = append(options, fx.Invoke(func(params initParams) error {
for _, protocol := range params.Protocols {
err := protocol.Init()
err := cm.ConfigureProtocol(protocol.Name(), protocol.Config())
if err != nil {
return err
}
err = protocol.Init()
if err != nil {
return err
}

3
protocols/registry/registry.go
View File

@ -3,6 +3,8 @@ package registry
import (
"context"
"git.lumeweb.com/LumeWeb/portal/config"
"go.uber.org/fx"
)
@ -13,6 +15,7 @@ type Protocol interface {
Init() error
Start(ctx context.Context) error
Stop(ctx context.Context) error
Config() config.ProtocolConfig
}
type ProtocolEntry struct {

26
protocols/s5/config.go Normal file
View File

@ -0,0 +1,26 @@
package s5
import (
s5config "git.lumeweb.com/LumeWeb/libs5-go/config"
"git.lumeweb.com/LumeWeb/portal/config"
)
var _ config.ProtocolConfig = (*Config)(nil)
type Config struct {
s5config.NodeConfig
DbPath string `mapstructure:"db_path"`
}
func (c Config) Defaults() map[string]interface{} {
defaults := map[string]interface{}{}
defaults["p2p.network.peers"] = []string{
"ss://z2DWuWNZcdSyZLpXFK2uCU3haaWMXrDAgxzv17sDEMHstZb@s5.garden/s5/p2p",
"wss://z2DWuPbL5pweybXnEB618pMnV58ECj2VPDNfVGm3tFqBvjF@s5.ninja/s5/p2p",
}
defaults["db_path"] = "s5.db"
return defaults
}

86
protocols/s5/s5.go
View File

@ -7,6 +7,8 @@ import (
"fmt"
"time"
"git.lumeweb.com/LumeWeb/portal/config"
"git.lumeweb.com/LumeWeb/portal/metadata"
"git.lumeweb.com/LumeWeb/portal/storage"
@ -32,18 +34,19 @@ var (
)
type S5Protocol struct {
config *viper.Viper
logger *zap.Logger
storage storage.StorageService
identity ed25519.PrivateKey
node *s5node.Node
tusHandler *TusHandler
store *S5ProviderStore
portalConfig *config.Manager
config *Config
logger *zap.Logger
storage storage.StorageService
identity ed25519.PrivateKey
node *s5node.Node
tusHandler *TusHandler
store *S5ProviderStore
}
type S5ProtocolParams struct {
fx.In
Config *viper.Viper
PortalConfig *config.Manager
Logger *zap.Logger
Storage storage.StorageService
Identity ed25519.PrivateKey
@ -85,15 +88,15 @@ func NewS5Protocol(
params S5ProtocolParams,
) (S5ProtocolResult, error) {
proto := &S5Protocol{
config: params.Config,
logger: params.Logger,
storage: params.Storage,
identity: params.Identity,
tusHandler: params.TusHandler,
store: params.ProviderStore,
portalConfig: params.PortalConfig,
logger: params.Logger,
storage: params.Storage,
identity: params.Identity,
tusHandler: params.TusHandler,
store: params.ProviderStore,
}
cfg, err := ConfigureS5Protocol(params)
cfg, err := configureS5Protocol(proto)
if err != nil {
return S5ProtocolResult{}, err
}
@ -106,60 +109,53 @@ func NewS5Protocol(
}, nil
}
func ConfigureS5Protocol(params S5ProtocolParams) (*s5config.NodeConfig, error) {
cfg := &s5config.NodeConfig{
P2P: s5config.P2PConfig{
Network: "",
Peers: s5config.PeersConfig{Initial: []string{}},
},
KeyPair: s5ed.New(params.Identity),
DB: nil,
Logger: params.Logger.Named("s5"),
HTTP: s5config.HTTPConfig{},
}
func configureS5Protocol(proto *S5Protocol) (*s5config.NodeConfig, error) {
cfg := proto.Config().(*Config)
cm := proto.portalConfig
portalCfg := cm.Config()
vpr := cm.Viper()
pconfig := params.Config.Sub("protocol.s5")
if pconfig == nil {
params.Logger.Fatal("Missing protocol.s5 Config")
}
err := pconfig.Unmarshal(cfg)
err := cm.ConfigureProtocol(proto.Name(), cfg)
if err != nil {
return nil, err
}
cfg.HTTP.API.Domain = fmt.Sprintf("s5.%s", params.Config.GetString("core.domain"))
cfg.HTTP.API.Domain = fmt.Sprintf("s5.%s", vpr.GetString("core.domain"))
if params.Config.IsSet("core.externalPort") {
cfg.HTTP.API.Port = params.Config.GetUint("core.externalPort")
if portalCfg.Core.ExternalPort != 0 {
cfg.HTTP.API.Port = portalCfg.Core.ExternalPort
} else {
cfg.HTTP.API.Port = params.Config.GetUint("core.port")
cfg.HTTP.API.Port = portalCfg.Core.Port
}
dbPath := pconfig.GetString("dbPath")
if dbPath == "" {
params.Logger.Fatal("protocol.s5.dbPath is required")
if cfg.DbPath == "" {
proto.logger.Fatal("protocol.s5.dbPath is required")
}
_, p, err := ed25519.GenerateKey(nil)
if err != nil {
params.Logger.Fatal("Failed to generate key", zap.Error(err))
proto.logger.Fatal("Failed to generate key", zap.Error(err))
}
cfg.KeyPair = s5ed.New(p)
db, err := bolt.Open(dbPath, 0600, nil)
db, err := bolt.Open(cfg.DbPath, 0600, nil)
if err != nil {
params.Logger.Fatal("Failed to open db", zap.Error(err))
proto.logger.Fatal("Failed to open db", zap.Error(err))
}
cfg.DB = db
return cfg, nil
return interface{}(cfg).(*s5config.NodeConfig), nil
}
func (s *S5Protocol) Config() config.ProtocolConfig {
if s.config == nil {
s.config = &Config{}
}
return s.config
}
func NewS5ProviderStore(params S5ProviderStoreParams) *S5ProviderStore {
return &S5ProviderStore{
config: params.Config,

29
protocols/s5/tus.go
View File

@ -11,16 +11,15 @@ import (
"time"
"git.lumeweb.com/LumeWeb/portal/api/middleware"
"git.lumeweb.com/LumeWeb/portal/config"
"go.uber.org/fx"
"git.lumeweb.com/LumeWeb/portal/account"
"github.com/spf13/viper"
"git.lumeweb.com/LumeWeb/portal/metadata"
"github.com/aws/aws-sdk-go-v2/config"
awsConfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/tus/tusd/v2/pkg/s3store"
@ -45,7 +44,7 @@ var (
)
type TusHandler struct {
config *viper.Viper
config *config.Manager
db *gorm.DB
logger *zap.Logger
cron *cron.CronServiceDefault
@ -60,7 +59,7 @@ type TusHandler struct {
type TusHandlerParams struct {
fx.In
Config *viper.Viper
Config *config.Manager
Logger *zap.Logger
Db *gorm.DB
Cron *cron.CronServiceDefault
@ -127,21 +126,21 @@ func (t *TusHandler) Init() error {
customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
if service == s3.ServiceID {
return aws.Endpoint{
URL: t.config.GetString("core.storage.s3.endpoint"),
SigningRegion: t.config.GetString("core.storage.s3.region"),
URL: t.config.Config().Core.Storage.S3.Endpoint,
SigningRegion: t.config.Config().Core.Storage.S3.Region,
}, nil
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithRegion("us-east-1"),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
t.config.GetString("core.storage.s3.accessKey"),
t.config.GetString("core.storage.s3.secretKey"),
cfg, err := awsConfig.LoadDefaultConfig(context.TODO(),
awsConfig.WithRegion("us-east-1"),
awsConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
t.config.Config().Core.Storage.S3.AccessKey,
t.config.Config().Core.Storage.S3.SecretKey,
"",
)),
config.WithEndpointResolverWithOptions(customResolver),
awsConfig.WithEndpointResolverWithOptions(customResolver),
)
if err != nil {
return err
@ -149,7 +148,7 @@ func (t *TusHandler) Init() error {
s3Client := s3.NewFromConfig(cfg)
store := s3store.New(t.config.GetString("core.storage.s3.bufferBucket"), s3Client)
store := s3store.New(t.config.Config().Core.Storage.S3.BufferBucket, s3Client)
locker := NewMySQLLocker(t.db, t.logger)
@ -407,7 +406,7 @@ func (t *TusHandler) uploadTask(hash []byte) error {
s3InfoId, _ := splitS3Ids(upload.UploadID)
_, err = t.s3Client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
Bucket: aws.String(t.config.GetString("core.storage.s3.bufferBucket")),
Bucket: aws.String(t.config.Config().Core.Storage.S3.BufferBucket),
Delete: &s3types.Delete{
Objects: []s3types.ObjectIdentifier{
{