feat: add database cache support with both memory and redis modes
This commit is contained in:
parent
995b227d7e
commit
b5b0ed64b6
GPG Key ID:
C997C339BE476FF2
|
|
@ -47,7 +47,7 @@ func NewManager(logger *zap.Logger) (*Manager, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
err = v.Unmarshal(&config)
|
||||
err = v.Unmarshal(&config, viper.DecodeHook(cacheConfigHook))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,68 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
|
||||
"github.com/mitchellh/mapstructure"
|
||||
)
|
||||
|
||||
type DatabaseConfig struct {
|
||||
Charset string `mapstructure:"charset"`
|
||||
Host string `mapstructure:"host"`
|
||||
Name string `mapstructure:"name"`
|
||||
Password string `mapstructure:"password"`
|
||||
Port int `mapstructure:"port"`
|
||||
Username string `mapstructure:"username"`
|
||||
Charset string `mapstructure:"charset"`
|
||||
Host string `mapstructure:"host"`
|
||||
Name string `mapstructure:"name"`
|
||||
Password string `mapstructure:"password"`
|
||||
Port int `mapstructure:"port"`
|
||||
Username string `mapstructure:"username"`
|
||||
Cache *CacheConfig `mapstructure:"cache"`
|
||||
}
|
||||
|
||||
type CacheConfig struct {
|
||||
Mode string `mapstructure:"mode"`
|
||||
Options interface{} `mapstructure:"options"`
|
||||
}
|
||||
|
||||
type RedisConfig struct {
|
||||
Address string `mapstructure:"address"`
|
||||
Password string `mapstructure:"password"`
|
||||
DB int `mapstructure:"db"`
|
||||
}
|
||||
|
||||
type MemoryConfig struct {
|
||||
}
|
||||
|
||||
func cacheConfigHook() mapstructure.DecodeHookFuncType {
|
||||
return func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) {
|
||||
// This hook is designed to operate on the options field within the CacheConfig
|
||||
if f.Kind() != reflect.Map || t != reflect.TypeOf(&CacheConfig{}) {
|
||||
return data, nil
|
||||
}
|
||||
|
||||
var cacheConfig CacheConfig
|
||||
if err := mapstructure.Decode(data, &cacheConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Assuming the input data map includes "mode" and "options"
|
||||
switch cacheConfig.Mode {
|
||||
case "redis":
|
||||
var redisOptions RedisConfig
|
||||
if opts, ok := cacheConfig.Options.(map[string]interface{}); ok && opts != nil {
|
||||
if err := mapstructure.Decode(opts, &redisOptions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cacheConfig.Options = redisOptions
|
||||
}
|
||||
case "memory":
|
||||
// For "memory", you might simply use an empty MemoryConfig,
|
||||
// or decode options similarly if there are any specific to memory caching.
|
||||
cacheConfig.Options = MemoryConfig{}
|
||||
case "false":
|
||||
// If "false", ensure no options are set, or set to a nil or similar neutral value.
|
||||
cacheConfig.Options = nil
|
||||
default:
|
||||
cacheConfig.Options = nil
|
||||
}
|
||||
|
||||
return cacheConfig, nil
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/go-gorm/caches/v4"
|
||||
)
|
||||
|
||||
type memoryCacher struct {
|
||||
store *sync.Map
|
||||
}
|
||||
|
||||
func (c *memoryCacher) init() {
|
||||
if c.store == nil {
|
||||
c.store = &sync.Map{}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *memoryCacher) Get(ctx context.Context, key string, q *caches.Query[any]) (*caches.Query[any], error) {
|
||||
c.init()
|
||||
val, ok := c.store.Load(key)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err := q.Unmarshal(val.([]byte)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return q, nil
|
||||
}
|
||||
|
||||
func (c *memoryCacher) Store(ctx context.Context, key string, val *caches.Query[any]) error {
|
||||
c.init()
|
||||
res, err := val.Marshal()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.store.Store(key, res)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *memoryCacher) Invalidate(ctx context.Context) error {
|
||||
c.store = &sync.Map{}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/go-gorm/caches/v4"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
type redisCacher struct {
|
||||
rdb *redis.Client
|
||||
}
|
||||
|
||||
func (c *redisCacher) Get(ctx context.Context, key string, q *caches.Query[any]) (*caches.Query[any], error) {
|
||||
res, err := c.rdb.Get(ctx, key).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := q.Unmarshal([]byte(res)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return q, nil
|
||||
}
|
||||
|
||||
func (c *redisCacher) Store(ctx context.Context, key string, val *caches.Query[any]) error {
|
||||
res, err := val.Marshal()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.rdb.Set(ctx, key, res, 300*time.Second) // Set proper cache time
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *redisCacher) Invalidate(ctx context.Context) error {
|
||||
var (
|
||||
cursor uint64
|
||||
keys []string
|
||||
)
|
||||
for {
|
||||
var (
|
||||
k []string
|
||||
err error
|
||||
)
|
||||
k, cursor, err = c.rdb.Scan(ctx, cursor, fmt.Sprintf("%s*", caches.IdentifierPrefix), 0).Result()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keys = append(keys, k...)
|
||||
if cursor == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(keys) > 0 {
|
||||
if _, err := c.rdb.Del(ctx, keys...).Result(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
64
db/db.go
64
db/db.go
|
|
@ -4,9 +4,14 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"git.lumeweb.com/LumeWeb/portal/config"
|
||||
|
||||
"git.lumeweb.com/LumeWeb/portal/db/models"
|
||||
"github.com/go-gorm/caches/v4"
|
||||
"go.uber.org/fx"
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/gorm"
|
||||
|
|
@ -15,6 +20,7 @@ import (
|
|||
type DatabaseParams struct {
|
||||
fx.In
|
||||
Config *config.Manager
|
||||
Logger *zap.Logger
|
||||
}
|
||||
|
||||
var Module = fx.Module("db",
|
||||
|
|
@ -38,6 +44,17 @@ func NewDatabase(lc fx.Lifecycle, params DatabaseParams) *gorm.DB {
|
|||
panic(err)
|
||||
}
|
||||
|
||||
cacher := getCacher(params.Config, params.Logger)
|
||||
if cacher != nil {
|
||||
cache := &caches.Caches{Conf: &caches.Config{
|
||||
Cacher: cacher,
|
||||
}}
|
||||
err := db.Use(cache)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
return db.AutoMigrate(
|
||||
|
|
@ -57,3 +74,50 @@ func NewDatabase(lc fx.Lifecycle, params DatabaseParams) *gorm.DB {
|
|||
|
||||
return db
|
||||
}
|
||||
|
||||
func getCacheMode(cm *config.Manager, logger *zap.Logger) string {
|
||||
|
||||
if cm.Config().Core.DB.Cache == nil {
|
||||
return "none"
|
||||
}
|
||||
|
||||
switch cm.Config().Core.DB.Cache.Mode {
|
||||
case "", "none":
|
||||
return "none"
|
||||
case "memory":
|
||||
return "memory"
|
||||
case "redis":
|
||||
return "redis"
|
||||
default:
|
||||
logger.Fatal("invalid cache mode", zap.String("mode", cm.Config().Core.DB.Cache.Mode))
|
||||
}
|
||||
|
||||
return "none"
|
||||
}
|
||||
|
||||
func getCacher(cm *config.Manager, logger *zap.Logger) caches.Cacher {
|
||||
mode := getCacheMode(cm, logger)
|
||||
|
||||
switch mode {
|
||||
case "none":
|
||||
return nil
|
||||
|
||||
case "memory":
|
||||
return &memoryCacher{}
|
||||
case "redis":
|
||||
rcfg, ok := cm.Config().Core.DB.Cache.Options.(config.RedisConfig)
|
||||
if !ok {
|
||||
logger.Fatal("invalid redis config")
|
||||
return nil
|
||||
}
|
||||
return &redisCacher{
|
||||
redis.NewClient(&redis.Options{
|
||||
Addr: rcfg.Address,
|
||||
Password: rcfg.Password,
|
||||
DB: rcfg.DB,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
5
go.mod
5
go.mod
|
|
@ -13,12 +13,15 @@ require (
|
|||
github.com/docker/go-units v0.5.0
|
||||
github.com/getkin/kin-openapi v0.118.0
|
||||
github.com/go-co-op/gocron/v2 v2.2.4
|
||||
github.com/go-gorm/caches/v4 v4.0.0
|
||||
github.com/go-resty/resty/v2 v2.11.0
|
||||
github.com/golang-jwt/jwt/v5 v5.2.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/hashicorp/go-plugin v1.6.0
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
github.com/pquerna/otp v1.4.0
|
||||
github.com/redis/go-redis/v9 v9.5.1
|
||||
github.com/rs/cors v1.10.1
|
||||
github.com/samber/lo v1.39.0
|
||||
github.com/spf13/viper v1.18.2
|
||||
|
|
@ -60,6 +63,7 @@ require (
|
|||
github.com/casbin/govaluate v1.1.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/fatih/color v1.14.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
|
|
@ -85,7 +89,6 @@ require (
|
|||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
|
||||
github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
|
||||
github.com/mr-tron/base58 v1.1.0 // indirect
|
||||
github.com/multiformats/go-base32 v0.0.3 // indirect
|
||||
|
|
|
|||
12
go.sum
12
go.sum
|
|
@ -1,6 +1,4 @@
|
|||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240201012059-dfeb8b29a8e4 h1:yUv0uhdPvE2pnYdhNMk3r3rs7tTP+frRLtycZozZCG8=
|
||||
git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240201012059-dfeb8b29a8e4/go.mod h1:1fftK3db+qKZhPxijPSlORciLf5y5t8Ozok59ifx6T8=
|
||||
git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240223122333-b0c459785222 h1:CJEtKAJO7d5IC/z6n+S9WxK7PpsczEKr3CDhNSl5D8I=
|
||||
git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240223122333-b0c459785222/go.mod h1:1fftK3db+qKZhPxijPSlORciLf5y5t8Ozok59ifx6T8=
|
||||
github.com/Acconut/go-httptest-recorder v1.0.0 h1:TAv2dfnqp/l+SUvIaMAUK4GeN4+wqb6KZsFFFTGhoJg=
|
||||
|
|
@ -66,6 +64,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
|||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA=
|
||||
github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8=
|
||||
github.com/casbin/casbin/v2 v2.82.0 h1:2CgvunqQQoepcbGRnMc9vEcDhuqh3B5yWKoj+kKSxf8=
|
||||
|
|
@ -89,6 +91,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
|
|||
github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf h1:K5VXW9LjmJv/xhjvQcNWTdk4WOSyreil6YaubuCPeRY=
|
||||
github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf/go.mod h1:bXVurdTuvOiJu7NHALemFe0JMvC2UmwYHW+7fcZaZ2M=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
|
|
@ -109,6 +113,8 @@ github.com/getkin/kin-openapi v0.118.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BH
|
|||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/go-co-op/gocron/v2 v2.2.4 h1:fL6a8/U+BJQ9UbaeqKxua8wY02w4ftKZsxPzLSNOCKk=
|
||||
github.com/go-co-op/gocron/v2 v2.2.4/go.mod h1:igssOwzZkfcnu3m2kwnCf/mYj4SmhP9ecSgmYjCOHkk=
|
||||
github.com/go-gorm/caches/v4 v4.0.0 h1:3nfNy1ya6f9s0RjpJ6lFMOfeyOzQjPoaXuLMmagFL8k=
|
||||
github.com/go-gorm/caches/v4 v4.0.0/go.mod h1:Ms8LnWVoW4GkTofpDzFH8OfDGNTjLxQDyxBmRN67Ujw=
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.1 h1:eGS0WTFRV30r103lU8JNXY27KbviRnqqIDobW3EV3iY=
|
||||
github.com/go-gormigrate/gormigrate/v2 v2.1.1/go.mod h1:L7nJ620PFDKei9QOhJzqA8kRCk+E3UbV2f5gv+1ndLc=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
|
|
@ -290,6 +296,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
|
|||
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
|
||||
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
|
||||
github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
|
|
|
|||
Loading…
Reference in New Issue