feat: initial database support and models

cmd/portal/init.go

@@ -62,6 +62,10 @@ func initLogger(p interfaces.Portal) error {
 	return nil
 }
 
+func initDatabase(p interfaces.Portal) error {
+	return p.Database().Init(p)
+}
+
 func initProtocols(p interfaces.Portal) error {
 	return protocols.Init(p.ProtocolRegistry())
 }
@@ -108,6 +112,7 @@ func getInitList() []initFunc {
 		initIdentity,
 		initCheckRequiredConfig,
 		initLogger,
+		initDatabase,
 		initProtocols,
 		initStorage,
 		initAPI,

cmd/portal/portal.go

@@ -3,6 +3,7 @@ package main
 import (
 	"crypto/ed25519"
 	"git.lumeweb.com/LumeWeb/portal/api"
+	"git.lumeweb.com/LumeWeb/portal/db"
 	"git.lumeweb.com/LumeWeb/portal/interfaces"
 	"git.lumeweb.com/LumeWeb/portal/protocols"
 	"git.lumeweb.com/LumeWeb/portal/storage"
@@ -23,6 +24,12 @@ type PortalImpl struct {
 	logger           *zap.Logger
 	identity         ed25519.PrivateKey
 	storage          interfaces.StorageService
+	database         interfaces.Database
+}
+
+func (p *PortalImpl) Database() interfaces.Database {
+	//TODO implement me
+	panic("implement me")
 }
 
 func NewPortal() interfaces.Portal {
@@ -31,10 +38,13 @@ func NewPortal() interfaces.Portal {
 		apiRegistry:      api.NewRegistry(),
 		protocolRegistry: protocols.NewProtocolRegistry(),
 		storage:          nil,
+		database:         nil,
 	}
 
 	storageServ := storage.NewStorageService(portal)
+	database := db.NewDatabase(portal)
 	portal.storage = storageServ
+	portal.database = database
 
 	return portal
 }

cmd/portal/start.go

@@ -15,8 +15,13 @@ func initProtocolRegistry(p interfaces.Portal) error {
 	return nil
 }
 
+func startDatabase(p interfaces.Portal) error {
+	return p.Database().Start()
+}
+
 func getStartList() []startFunc {
 	return []startFunc{
 		initProtocolRegistry,
+		startDatabase,
 	}
 }

config/config.go

@@ -46,6 +46,8 @@ func writeDefaults() error {
 	defaults := map[string]interface{}{
 		"core.post-upload-limit":                  1024 * 1024 * 1000,
 		"core.log.level":                          "info",
+		"core.db.charset":                         "utf8mb4",
+		"core.db.port":                            "3306",
 		"protocol.s5.p2p.maxOutgoingPeerFailures": 10,
 		"protocol.s5.p2p.network":                 "",
 	}

db/db.go

@@ -0,0 +1,62 @@
+package db
+
+import (
+	"fmt"
+	"git.lumeweb.com/LumeWeb/portal/db/models"
+	"git.lumeweb.com/LumeWeb/portal/interfaces"
+	"github.com/spf13/viper"
+	"go.uber.org/zap"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+)
+
+var (
+	_ interfaces.Database = (*DatabaseImpl)(nil)
+)
+
+type DatabaseImpl struct {
+	DB     *gorm.DB
+	portal interfaces.Portal
+}
+
+func NewDatabase(p interfaces.Portal) interfaces.Database {
+	return &DatabaseImpl{
+		portal: p,
+	}
+}
+
+// Init initializes the database connection
+func (d *DatabaseImpl) Init(p interfaces.Portal) error {
+	// Retrieve DB config from Viper
+	username := viper.GetString("core.db.username")
+	password := viper.GetString("core.db.password")
+	host := viper.GetString("core.db.host")
+	port := viper.GetString("core.db.port")
+	dbname := viper.GetString("core.db.dbname")
+	charset := viper.GetString("core.db.charset")
+
+	// Construct DSN
+	dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=%s&parseTime=True&loc=Local", username, password, host, port, dbname, charset)
+
+	// Open DB connection
+	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
+	if err != nil {
+		p.Logger().Error("Failed to connect to database", zap.Error(err))
+	}
+	d.DB = db
+
+	return nil
+}
+
+// Start performs any additional setup
+func (d *DatabaseImpl) Start() error {
+	return d.DB.AutoMigrate(
+		&models.APIKey{},
+		&models.Blocklist{},
+		&models.Download{},
+		&models.Pin{},
+		&models.PublicKey{},
+		&models.Upload{},
+		&models.User{},
+	)
+}

db/models/api_key.go

@@ -0,0 +1,10 @@
+package models
+
+import "gorm.io/gorm"
+
+type APIKey struct {
+	gorm.Model
+	UserID uint
+	Key    string
+	User   User
+}

db/models/blocklist.go

@@ -0,0 +1,13 @@
+package models
+
+import (
+	"gorm.io/gorm"
+	"time"
+)
+
+type Blocklist struct {
+	gorm.Model
+	IP        string
+	Reason    string
+	BlockedAt time.Time
+}

db/models/download.go

@@ -0,0 +1,16 @@
+package models
+
+import (
+	"gorm.io/gorm"
+	"time"
+)
+
+type Download struct {
+	gorm.Model
+	UserID       uint
+	User         User
+	UploadID     uint
+	Upload       Upload
+	DownloadedAt time.Time
+	IP           string
+}

db/models/pin.go

@@ -0,0 +1,11 @@
+package models
+
+import "gorm.io/gorm"
+
+type Pin struct {
+	gorm.Model
+	UploadID       uint
+	Upload         Upload
+	PinnedByUserID uint
+	User           User
+}

db/models/public_key.go

@@ -0,0 +1,10 @@
+package models
+
+import "gorm.io/gorm"
+
+type PublicKey struct {
+	gorm.Model
+	UserID uint
+	Key    string
+	User   User
+}

db/models/upload.go

@@ -0,0 +1,12 @@
+package models
+
+import "gorm.io/gorm"
+
+type Upload struct {
+	gorm.Model
+	UserID       uint
+	CID          string
+	ProtocolType string
+	User         User
+	UploaderIP   string
+}

db/models/user.go

@@ -0,0 +1,19 @@
+package models
+
+import (
+	"gorm.io/gorm"
+	"time"
+)
+
+type User struct {
+	gorm.Model
+	Username     string
+	Email        string `gorm:"unique"`
+	PasswordHash string
+	Role         string
+	PublicKeys   []PublicKey
+	APIKeys      []APIKey
+	Uploads      []Upload
+	LastLogin    *time.Time
+	LastLoginIP  string
+}

go.mod

@@ -4,19 +4,23 @@ go 1.20
 
 require (
 	git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240116003411-13ca22d80e1c
+	github.com/go-resty/resty/v2 v2.11.0
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/spf13/viper v1.18.2
 	go.etcd.io/bbolt v1.3.8
 	go.sia.tech/core v0.1.12
+	go.sia.tech/jape v0.11.1
 	go.uber.org/zap v1.26.0
+	gorm.io/driver/mysql v1.5.2
 	gorm.io/gorm v1.25.5
+	lukechampine.com/blake3 v1.2.1
 )
 
 require (
 	github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-resty/resty/v2 v2.11.0 // indirect
+	github.com/go-sql-driver/mysql v1.7.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
@@ -38,7 +42,6 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
-	go.sia.tech/jape v0.11.1 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect
@@ -48,7 +51,6 @@ require (
 	golang.org/x/tools v0.17.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	lukechampine.com/blake3 v1.2.1 // indirect
 	lukechampine.com/frand v1.4.2 // indirect
 	nhooyr.io/websocket v1.8.10 // indirect
 )

go.sum

@@ -1,39 +1,3 @@
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240112125539-f38c02adb920 h1:5vNHb37MQtIibnku37M1TB+VcTxSHB/8dEkYV5vw/nc=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240112125539-f38c02adb920/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240112201024-36f087dc8357 h1:MpiUDBduDDZArPi8Nl/N7bsaTQv5wzWA++wOkTS4FNY=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240112201024-36f087dc8357/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240113162201-3d41119f7451 h1:AnbIFGrPpoCS53nJw1fFD2HszOPaTH8005xLo0XZouc=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240113162201-3d41119f7451/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115015344-38e330e02b2d h1:qZRKBBkLxG37HdQrMu8bvFzXWfwtsef5xB6/ND3f3+E=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115015344-38e330e02b2d/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115030657-d79455c68c4b h1:KObWrO/RehW7eFKWD1CZ8pWJLFFOfotOHnui21sdZeo=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115030657-d79455c68c4b/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115155431-883f50b19861 h1:gbVGg5qobgWRtjaXwDcIFVAzpj+DRnuYoNBq+i6ZTiQ=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115155431-883f50b19861/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115161511-fc10a265a772 h1:P4Qw97ZcdoLUIeBpwDaKM7/FgFrnR49z3Xw4o2uA1vI=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115161511-fc10a265a772/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115163838-ab53dbdf08ca h1:clldADmpy5pC8a+/7yHVKx5qXAeznhOJShKFZ1WUnq8=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115163838-ab53dbdf08ca/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115164525-bbb407b0e10f h1:hWJA0nYnKLv6qDQnSf+F06Qd6aVTc2FuJ2t+dUcO7Hw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115164525-bbb407b0e10f/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115171317-def05376f5cb h1:JLwWU0bFVjPT5VnlPr4M84q7D0F0B7bd7biftPyYKzg=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115171317-def05376f5cb/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115175846-743ba71e4b63 h1:Q9BOefL7PW/4klvcKWBljR+vkV9DeU0bfV6CSAN0m3Q=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115175846-743ba71e4b63/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115180349-3ddf2595b920 h1:XD47Uz5rKoZmGfz7O+b8nt2j46QXL7s10AKKDuaDpiU=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115180349-3ddf2595b920/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115181015-d76bfc6daf3b h1:ZA1ysYnGyn2qOAz2bV8KODWeald+asifJRmEnwbVfVg=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115181015-d76bfc6daf3b/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115181946-5e8083133518 h1:AQ2zb6VVvBUijUO02+9/7DTmO1oxrgAaxn35nQpYsJc=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115181946-5e8083133518/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115183749-c2ab3b4651dc h1:FXiVjETHXPsE/HOD2fD5J54uUE7zWQyPkZ80jEtG+as=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115183749-c2ab3b4651dc/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115184041-7d34ac37dbd9 h1:+HOAMx1EcM2GAm1CZRNGi7PrU1Now1lXskciGLBg8E0=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115184041-7d34ac37dbd9/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115185004-1b6925c296af h1:JXwgNIfbuSUJbQ+CaSE8aVrWyhRYFTwxAOzcIXDj3FM=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115185004-1b6925c296af/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115191327-dced32ab21a7 h1:awLw1VZd9irjJEwfXkJgn3gVunQ5QkjAsC5vIAN2XOY=
-git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240115191327-dced32ab21a7/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
 git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240116003411-13ca22d80e1c h1:qG16pbgMd7mdXRXZRyveMl8p9MsWlnqEm21pSW9AUfM=
 git.lumeweb.com/LumeWeb/libs5-go v0.0.0-20240116003411-13ca22d80e1c/go.mod h1:CMtoCT4WlAWzJtNer2MEW170i14jeKhSjxbYQ5DIGqw=
 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
@@ -48,6 +12,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
 github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
+github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@@ -125,8 +91,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
 golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
-golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE=
-golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
 golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o=
 golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -183,6 +147,9 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.5.2 h1:QC2HRskSE75wBuOxe0+iCkyJZ+RqpudsQtqkp+IMuXs=
+gorm.io/driver/mysql v1.5.2/go.mod h1:pQLhh1Ut/WUAySdTHwBpBv6+JKcj+ua4ZFx1QQTBzb8=
+gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
 gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
 gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
 lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI=

interfaces/database.go

@@ -0,0 +1,6 @@
+package interfaces
+
+type Database interface {
+	Init(p Portal) error
+	Start() error
+}

interfaces/portal.go

@@ -19,4 +19,5 @@ type Portal interface {
 	Storage() StorageService
 	SetIdentity(identity ed25519.PrivateKey)
 	SetLogger(logger *zap.Logger)
+	Database() Database
 }