feat: add database cache support with both memory and redis modes

config/config.go

@@ -47,7 +47,7 @@ func NewManager(logger *zap.Logger) (*Manager, error) {
 		return nil, err
 	}
 
-	err = v.Unmarshal(&config)
+	err = v.Unmarshal(&config, viper.DecodeHook(cacheConfigHook))
 	if err != nil {
 		return nil, err
 	}

config/database.go

@@ -1,5 +1,11 @@
 package config
 
+import (
+	"reflect"
+
+	"github.com/mitchellh/mapstructure"
+)
+
 type DatabaseConfig struct {
 	Charset  string       `mapstructure:"charset"`
 	Host     string       `mapstructure:"host"`
@@ -7,4 +13,56 @@ type DatabaseConfig struct {
 	Password string       `mapstructure:"password"`
 	Port     int          `mapstructure:"port"`
 	Username string       `mapstructure:"username"`
+	Cache    *CacheConfig `mapstructure:"cache"`
+}
+
+type CacheConfig struct {
+	Mode    string      `mapstructure:"mode"`
+	Options interface{} `mapstructure:"options"`
+}
+
+type RedisConfig struct {
+	Address  string `mapstructure:"address"`
+	Password string `mapstructure:"password"`
+	DB       int    `mapstructure:"db"`
+}
+
+type MemoryConfig struct {
+}
+
+func cacheConfigHook() mapstructure.DecodeHookFuncType {
+	return func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) {
+		// This hook is designed to operate on the options field within the CacheConfig
+		if f.Kind() != reflect.Map || t != reflect.TypeOf(&CacheConfig{}) {
+			return data, nil
+		}
+
+		var cacheConfig CacheConfig
+		if err := mapstructure.Decode(data, &cacheConfig); err != nil {
+			return nil, err
+		}
+
+		// Assuming the input data map includes "mode" and "options"
+		switch cacheConfig.Mode {
+		case "redis":
+			var redisOptions RedisConfig
+			if opts, ok := cacheConfig.Options.(map[string]interface{}); ok && opts != nil {
+				if err := mapstructure.Decode(opts, &redisOptions); err != nil {
+					return nil, err
+				}
+				cacheConfig.Options = redisOptions
+			}
+		case "memory":
+			// For "memory", you might simply use an empty MemoryConfig,
+			// or decode options similarly if there are any specific to memory caching.
+			cacheConfig.Options = MemoryConfig{}
+		case "false":
+			// If "false", ensure no options are set, or set to a nil or similar neutral value.
+			cacheConfig.Options = nil
+		default:
+			cacheConfig.Options = nil
+		}
+
+		return cacheConfig, nil
+	}
 }

db/cache_memory.go

@@ -0,0 +1,48 @@
+package db
+
+import (
+	"context"
+	"sync"
+
+	"github.com/go-gorm/caches/v4"
+)
+
+type memoryCacher struct {
+	store *sync.Map
+}
+
+func (c *memoryCacher) init() {
+	if c.store == nil {
+		c.store = &sync.Map{}
+	}
+}
+
+func (c *memoryCacher) Get(ctx context.Context, key string, q *caches.Query[any]) (*caches.Query[any], error) {
+	c.init()
+	val, ok := c.store.Load(key)
+	if !ok {
+		return nil, nil
+	}
+
+	if err := q.Unmarshal(val.([]byte)); err != nil {
+		return nil, err
+	}
+
+	return q, nil
+}
+
+func (c *memoryCacher) Store(ctx context.Context, key string, val *caches.Query[any]) error {
+	c.init()
+	res, err := val.Marshal()
+	if err != nil {
+		return err
+	}
+
+	c.store.Store(key, res)
+	return nil
+}
+
+func (c *memoryCacher) Invalidate(ctx context.Context) error {
+	c.store = &sync.Map{}
+	return nil
+}

db/cache_redis.go

@@ -0,0 +1,70 @@
+package db
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/go-gorm/caches/v4"
+	"github.com/redis/go-redis/v9"
+)
+
+type redisCacher struct {
+	rdb *redis.Client
+}
+
+func (c *redisCacher) Get(ctx context.Context, key string, q *caches.Query[any]) (*caches.Query[any], error) {
+	res, err := c.rdb.Get(ctx, key).Result()
+	if errors.Is(err, redis.Nil) {
+		return nil, nil
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	if err := q.Unmarshal([]byte(res)); err != nil {
+		return nil, err
+	}
+
+	return q, nil
+}
+
+func (c *redisCacher) Store(ctx context.Context, key string, val *caches.Query[any]) error {
+	res, err := val.Marshal()
+	if err != nil {
+		return err
+	}
+
+	c.rdb.Set(ctx, key, res, 300*time.Second) // Set proper cache time
+	return nil
+}
+
+func (c *redisCacher) Invalidate(ctx context.Context) error {
+	var (
+		cursor uint64
+		keys   []string
+	)
+	for {
+		var (
+			k   []string
+			err error
+		)
+		k, cursor, err = c.rdb.Scan(ctx, cursor, fmt.Sprintf("%s*", caches.IdentifierPrefix), 0).Result()
+		if err != nil {
+			return err
+		}
+		keys = append(keys, k...)
+		if cursor == 0 {
+			break
+		}
+	}
+
+	if len(keys) > 0 {
+		if _, err := c.rdb.Del(ctx, keys...).Result(); err != nil {
+			return err
+		}
+	}
+	return nil
+}

db/db.go

@@ -4,9 +4,14 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/redis/go-redis/v9"
+
+	"go.uber.org/zap"
+
 	"git.lumeweb.com/LumeWeb/portal/config"
 
 	"git.lumeweb.com/LumeWeb/portal/db/models"
+	"github.com/go-gorm/caches/v4"
 	"go.uber.org/fx"
 	"gorm.io/driver/mysql"
 	"gorm.io/gorm"
@@ -15,6 +20,7 @@ import (
 type DatabaseParams struct {
 	fx.In
 	Config *config.Manager
+	Logger *zap.Logger
 }
 
 var Module = fx.Module("db",
@@ -38,6 +44,17 @@ func NewDatabase(lc fx.Lifecycle, params DatabaseParams) *gorm.DB {
 		panic(err)
 	}
 
+	cacher := getCacher(params.Config, params.Logger)
+	if cacher != nil {
+		cache := &caches.Caches{Conf: &caches.Config{
+			Cacher: cacher,
+		}}
+		err := db.Use(cache)
+		if err != nil {
+			return nil
+		}
+	}
+
 	lc.Append(fx.Hook{
 		OnStart: func(ctx context.Context) error {
 			return db.AutoMigrate(
@@ -57,3 +74,50 @@ func NewDatabase(lc fx.Lifecycle, params DatabaseParams) *gorm.DB {
 
 	return db
 }
+
+func getCacheMode(cm *config.Manager, logger *zap.Logger) string {
+
+	if cm.Config().Core.DB.Cache == nil {
+		return "none"
+	}
+
+	switch cm.Config().Core.DB.Cache.Mode {
+	case "", "none":
+		return "none"
+	case "memory":
+		return "memory"
+	case "redis":
+		return "redis"
+	default:
+		logger.Fatal("invalid cache mode", zap.String("mode", cm.Config().Core.DB.Cache.Mode))
+	}
+
+	return "none"
+}
+
+func getCacher(cm *config.Manager, logger *zap.Logger) caches.Cacher {
+	mode := getCacheMode(cm, logger)
+
+	switch mode {
+	case "none":
+		return nil
+
+	case "memory":
+		return &memoryCacher{}
+	case "redis":
+		rcfg, ok := cm.Config().Core.DB.Cache.Options.(config.RedisConfig)
+		if !ok {
+			logger.Fatal("invalid redis config")
+			return nil
+		}
+		return &redisCacher{
+			redis.NewClient(&redis.Options{
+				Addr:     rcfg.Address,
+				Password: rcfg.Password,
+				DB:       rcfg.DB,
+			}),
+		}
+	}
+
+	return nil
+}

go.mod

@@ -13,12 +13,15 @@ require (
 	github.com/docker/go-units v0.5.0
 	github.com/getkin/kin-openapi v0.118.0
 	github.com/go-co-op/gocron/v2 v2.2.4
+	github.com/go-gorm/caches/v4 v4.0.0
 	github.com/go-resty/resty/v2 v2.11.0
 	github.com/golang-jwt/jwt/v5 v5.2.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-plugin v1.6.0
 	github.com/julienschmidt/httprouter v1.3.0
+	github.com/mitchellh/mapstructure v1.5.0
 	github.com/pquerna/otp v1.4.0
+	github.com/redis/go-redis/v9 v9.5.1
 	github.com/rs/cors v1.10.1
 	github.com/samber/lo v1.39.0
 	github.com/spf13/viper v1.18.2
@@ -60,6 +63,7 @@ require (
 	github.com/casbin/govaluate v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fatih/color v1.14.1 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
@@ -85,7 +89,6 @@ require (
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/mr-tron/base58 v1.1.0 // indirect
 	github.com/multiformats/go-base32 v0.0.3 // indirect

go.sum

@@ -1,6 +1,4 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240201012059-dfeb8b29a8e4 h1:yUv0uhdPvE2pnYdhNMk3r3rs7tTP+frRLtycZozZCG8=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240201012059-dfeb8b29a8e4/go.mod h1:1fftK3db+qKZhPxijPSlORciLf5y5t8Ozok59ifx6T8=
 git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240223122333-b0c459785222 h1:CJEtKAJO7d5IC/z6n+S9WxK7PpsczEKr3CDhNSl5D8I=
 git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240223122333-b0c459785222/go.mod h1:1fftK3db+qKZhPxijPSlORciLf5y5t8Ozok59ifx6T8=
 github.com/Acconut/go-httptest-recorder v1.0.0 h1:TAv2dfnqp/l+SUvIaMAUK4GeN4+wqb6KZsFFFTGhoJg=
@@ -66,6 +64,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI=
 github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
 github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA=
 github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8=
 github.com/casbin/casbin/v2 v2.82.0 h1:2CgvunqQQoepcbGRnMc9vEcDhuqh3B5yWKoj+kKSxf8=
@@ -89,6 +91,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
 github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf h1:K5VXW9LjmJv/xhjvQcNWTdk4WOSyreil6YaubuCPeRY=
 github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf/go.mod h1:bXVurdTuvOiJu7NHALemFe0JMvC2UmwYHW+7fcZaZ2M=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -109,6 +113,8 @@ github.com/getkin/kin-openapi v0.118.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BH
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-co-op/gocron/v2 v2.2.4 h1:fL6a8/U+BJQ9UbaeqKxua8wY02w4ftKZsxPzLSNOCKk=
 github.com/go-co-op/gocron/v2 v2.2.4/go.mod h1:igssOwzZkfcnu3m2kwnCf/mYj4SmhP9ecSgmYjCOHkk=
+github.com/go-gorm/caches/v4 v4.0.0 h1:3nfNy1ya6f9s0RjpJ6lFMOfeyOzQjPoaXuLMmagFL8k=
+github.com/go-gorm/caches/v4 v4.0.0/go.mod h1:Ms8LnWVoW4GkTofpDzFH8OfDGNTjLxQDyxBmRN67Ujw=
 github.com/go-gormigrate/gormigrate/v2 v2.1.1 h1:eGS0WTFRV30r103lU8JNXY27KbviRnqqIDobW3EV3iY=
 github.com/go-gormigrate/gormigrate/v2 v2.1.1/go.mod h1:L7nJ620PFDKei9QOhJzqA8kRCk+E3UbV2f5gv+1ndLc=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -290,6 +296,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
+github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=