From b5b0ed64b6f2766feaa1c6f2d9bf109952f1af9b Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Sat, 24 Feb 2024 08:19:27 -0500 Subject: [PATCH] feat: add database cache support with both memory and redis modes --- config/config.go | 2 +- config/database.go | 70 ++++++++++++++++++++++++++++++++++++++++++---- db/cache_memory.go | 48 +++++++++++++++++++++++++++++++ db/cache_redis.go | 70 ++++++++++++++++++++++++++++++++++++++++++++++ db/db.go | 64 ++++++++++++++++++++++++++++++++++++++++++ go.mod | 5 +++- go.sum | 12 ++++++-- 7 files changed, 261 insertions(+), 10 deletions(-) create mode 100644 db/cache_memory.go create mode 100644 db/cache_redis.go diff --git a/config/config.go b/config/config.go index 6d51b39..3a210c6 100644 --- a/config/config.go +++ b/config/config.go @@ -47,7 +47,7 @@ func NewManager(logger *zap.Logger) (*Manager, error) { return nil, err } - err = v.Unmarshal(&config) + err = v.Unmarshal(&config, viper.DecodeHook(cacheConfigHook)) if err != nil { return nil, err } diff --git a/config/database.go b/config/database.go index 68983aa..47f1ce1 100644 --- a/config/database.go +++ b/config/database.go @@ -1,10 +1,68 @@ package config +import ( + "reflect" + + "github.com/mitchellh/mapstructure" +) + type DatabaseConfig struct { - Charset string `mapstructure:"charset"` - Host string `mapstructure:"host"` - Name string `mapstructure:"name"` - Password string `mapstructure:"password"` - Port int `mapstructure:"port"` - Username string `mapstructure:"username"` + Charset string `mapstructure:"charset"` + Host string `mapstructure:"host"` + Name string `mapstructure:"name"` + Password string `mapstructure:"password"` + Port int `mapstructure:"port"` + Username string `mapstructure:"username"` + Cache *CacheConfig `mapstructure:"cache"` +} + +type CacheConfig struct { + Mode string `mapstructure:"mode"` + Options interface{} `mapstructure:"options"` +} + +type RedisConfig struct { + Address string `mapstructure:"address"` + Password string `mapstructure:"password"` + DB int `mapstructure:"db"` +} + +type MemoryConfig struct { +} + +func cacheConfigHook() mapstructure.DecodeHookFuncType { + return func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) { + // This hook is designed to operate on the options field within the CacheConfig + if f.Kind() != reflect.Map || t != reflect.TypeOf(&CacheConfig{}) { + return data, nil + } + + var cacheConfig CacheConfig + if err := mapstructure.Decode(data, &cacheConfig); err != nil { + return nil, err + } + + // Assuming the input data map includes "mode" and "options" + switch cacheConfig.Mode { + case "redis": + var redisOptions RedisConfig + if opts, ok := cacheConfig.Options.(map[string]interface{}); ok && opts != nil { + if err := mapstructure.Decode(opts, &redisOptions); err != nil { + return nil, err + } + cacheConfig.Options = redisOptions + } + case "memory": + // For "memory", you might simply use an empty MemoryConfig, + // or decode options similarly if there are any specific to memory caching. + cacheConfig.Options = MemoryConfig{} + case "false": + // If "false", ensure no options are set, or set to a nil or similar neutral value. + cacheConfig.Options = nil + default: + cacheConfig.Options = nil + } + + return cacheConfig, nil + } } diff --git a/db/cache_memory.go b/db/cache_memory.go new file mode 100644 index 0000000..cc2009f --- /dev/null +++ b/db/cache_memory.go @@ -0,0 +1,48 @@ +package db + +import ( + "context" + "sync" + + "github.com/go-gorm/caches/v4" +) + +type memoryCacher struct { + store *sync.Map +} + +func (c *memoryCacher) init() { + if c.store == nil { + c.store = &sync.Map{} + } +} + +func (c *memoryCacher) Get(ctx context.Context, key string, q *caches.Query[any]) (*caches.Query[any], error) { + c.init() + val, ok := c.store.Load(key) + if !ok { + return nil, nil + } + + if err := q.Unmarshal(val.([]byte)); err != nil { + return nil, err + } + + return q, nil +} + +func (c *memoryCacher) Store(ctx context.Context, key string, val *caches.Query[any]) error { + c.init() + res, err := val.Marshal() + if err != nil { + return err + } + + c.store.Store(key, res) + return nil +} + +func (c *memoryCacher) Invalidate(ctx context.Context) error { + c.store = &sync.Map{} + return nil +} diff --git a/db/cache_redis.go b/db/cache_redis.go new file mode 100644 index 0000000..ed9fbd4 --- /dev/null +++ b/db/cache_redis.go @@ -0,0 +1,70 @@ +package db + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/go-gorm/caches/v4" + "github.com/redis/go-redis/v9" +) + +type redisCacher struct { + rdb *redis.Client +} + +func (c *redisCacher) Get(ctx context.Context, key string, q *caches.Query[any]) (*caches.Query[any], error) { + res, err := c.rdb.Get(ctx, key).Result() + if errors.Is(err, redis.Nil) { + return nil, nil + } + + if err != nil { + return nil, err + } + + if err := q.Unmarshal([]byte(res)); err != nil { + return nil, err + } + + return q, nil +} + +func (c *redisCacher) Store(ctx context.Context, key string, val *caches.Query[any]) error { + res, err := val.Marshal() + if err != nil { + return err + } + + c.rdb.Set(ctx, key, res, 300*time.Second) // Set proper cache time + return nil +} + +func (c *redisCacher) Invalidate(ctx context.Context) error { + var ( + cursor uint64 + keys []string + ) + for { + var ( + k []string + err error + ) + k, cursor, err = c.rdb.Scan(ctx, cursor, fmt.Sprintf("%s*", caches.IdentifierPrefix), 0).Result() + if err != nil { + return err + } + keys = append(keys, k...) + if cursor == 0 { + break + } + } + + if len(keys) > 0 { + if _, err := c.rdb.Del(ctx, keys...).Result(); err != nil { + return err + } + } + return nil +} diff --git a/db/db.go b/db/db.go index 2eadfd4..3826a12 100644 --- a/db/db.go +++ b/db/db.go @@ -4,9 +4,14 @@ import ( "context" "fmt" + "github.com/redis/go-redis/v9" + + "go.uber.org/zap" + "git.lumeweb.com/LumeWeb/portal/config" "git.lumeweb.com/LumeWeb/portal/db/models" + "github.com/go-gorm/caches/v4" "go.uber.org/fx" "gorm.io/driver/mysql" "gorm.io/gorm" @@ -15,6 +20,7 @@ import ( type DatabaseParams struct { fx.In Config *config.Manager + Logger *zap.Logger } var Module = fx.Module("db", @@ -38,6 +44,17 @@ func NewDatabase(lc fx.Lifecycle, params DatabaseParams) *gorm.DB { panic(err) } + cacher := getCacher(params.Config, params.Logger) + if cacher != nil { + cache := &caches.Caches{Conf: &caches.Config{ + Cacher: cacher, + }} + err := db.Use(cache) + if err != nil { + return nil + } + } + lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { return db.AutoMigrate( @@ -57,3 +74,50 @@ func NewDatabase(lc fx.Lifecycle, params DatabaseParams) *gorm.DB { return db } + +func getCacheMode(cm *config.Manager, logger *zap.Logger) string { + + if cm.Config().Core.DB.Cache == nil { + return "none" + } + + switch cm.Config().Core.DB.Cache.Mode { + case "", "none": + return "none" + case "memory": + return "memory" + case "redis": + return "redis" + default: + logger.Fatal("invalid cache mode", zap.String("mode", cm.Config().Core.DB.Cache.Mode)) + } + + return "none" +} + +func getCacher(cm *config.Manager, logger *zap.Logger) caches.Cacher { + mode := getCacheMode(cm, logger) + + switch mode { + case "none": + return nil + + case "memory": + return &memoryCacher{} + case "redis": + rcfg, ok := cm.Config().Core.DB.Cache.Options.(config.RedisConfig) + if !ok { + logger.Fatal("invalid redis config") + return nil + } + return &redisCacher{ + redis.NewClient(&redis.Options{ + Addr: rcfg.Address, + Password: rcfg.Password, + DB: rcfg.DB, + }), + } + } + + return nil +} diff --git a/go.mod b/go.mod index 70e2839..a49019a 100644 --- a/go.mod +++ b/go.mod @@ -13,12 +13,15 @@ require ( github.com/docker/go-units v0.5.0 github.com/getkin/kin-openapi v0.118.0 github.com/go-co-op/gocron/v2 v2.2.4 + github.com/go-gorm/caches/v4 v4.0.0 github.com/go-resty/resty/v2 v2.11.0 github.com/golang-jwt/jwt/v5 v5.2.0 github.com/google/uuid v1.6.0 github.com/hashicorp/go-plugin v1.6.0 github.com/julienschmidt/httprouter v1.3.0 + github.com/mitchellh/mapstructure v1.5.0 github.com/pquerna/otp v1.4.0 + github.com/redis/go-redis/v9 v9.5.1 github.com/rs/cors v1.10.1 github.com/samber/lo v1.39.0 github.com/spf13/viper v1.18.2 @@ -60,6 +63,7 @@ require ( github.com/casbin/govaluate v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fatih/color v1.14.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect @@ -85,7 +89,6 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/mr-tron/base58 v1.1.0 // indirect github.com/multiformats/go-base32 v0.0.3 // indirect diff --git a/go.sum b/go.sum index b7dc9d2..4632205 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240201012059-dfeb8b29a8e4 h1:yUv0uhdPvE2pnYdhNMk3r3rs7tTP+frRLtycZozZCG8= -git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240201012059-dfeb8b29a8e4/go.mod h1:1fftK3db+qKZhPxijPSlORciLf5y5t8Ozok59ifx6T8= git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240223122333-b0c459785222 h1:CJEtKAJO7d5IC/z6n+S9WxK7PpsczEKr3CDhNSl5D8I= git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240223122333-b0c459785222/go.mod h1:1fftK3db+qKZhPxijPSlORciLf5y5t8Ozok59ifx6T8= github.com/Acconut/go-httptest-recorder v1.0.0 h1:TAv2dfnqp/l+SUvIaMAUK4GeN4+wqb6KZsFFFTGhoJg= @@ -66,6 +64,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/casbin/casbin/v2 v2.82.0 h1:2CgvunqQQoepcbGRnMc9vEcDhuqh3B5yWKoj+kKSxf8= @@ -89,6 +91,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf h1:K5VXW9LjmJv/xhjvQcNWTdk4WOSyreil6YaubuCPeRY= github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf/go.mod h1:bXVurdTuvOiJu7NHALemFe0JMvC2UmwYHW+7fcZaZ2M= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -109,6 +113,8 @@ github.com/getkin/kin-openapi v0.118.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BH github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-co-op/gocron/v2 v2.2.4 h1:fL6a8/U+BJQ9UbaeqKxua8wY02w4ftKZsxPzLSNOCKk= github.com/go-co-op/gocron/v2 v2.2.4/go.mod h1:igssOwzZkfcnu3m2kwnCf/mYj4SmhP9ecSgmYjCOHkk= +github.com/go-gorm/caches/v4 v4.0.0 h1:3nfNy1ya6f9s0RjpJ6lFMOfeyOzQjPoaXuLMmagFL8k= +github.com/go-gorm/caches/v4 v4.0.0/go.mod h1:Ms8LnWVoW4GkTofpDzFH8OfDGNTjLxQDyxBmRN67Ujw= github.com/go-gormigrate/gormigrate/v2 v2.1.1 h1:eGS0WTFRV30r103lU8JNXY27KbviRnqqIDobW3EV3iY= github.com/go-gormigrate/gormigrate/v2 v2.1.1/go.mod h1:L7nJ620PFDKei9QOhJzqA8kRCk+E3UbV2f5gv+1ndLc= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -290,6 +296,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= +github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=