feat: wip initial storage location support

This commit is contained in:
Derrick Hammer 2024-01-06 09:45:00 -05:00
parent f45e297791
commit 4959270f51
Signed by: pcfreak30
GPG Key ID: C997C339BE476FF2
3 changed files with 262 additions and 83 deletions

98
node.go
View File

@ -65,7 +65,6 @@ func (n *Node) Services() *S5Services {
return nil
}
func (n *Node) Start() error {
n.started = true
return nil
@ -75,28 +74,30 @@ func (n *Node) Stop() error {
n.started = false
return nil
}
func (n *Node) GetCachedStorageLocations(hash Multihash, types []int) (map[NodeId]*StorageLocation, error) {
locations := make(map[NodeId]*StorageLocation)
*/
func (n *Node) GetCachedStorageLocations(hash *encoding.Multihash, types []int) (map[encoding.NodeIdCode]*StorageLocation, error) {
locations := make(map[encoding.NodeIdCode]*StorageLocation)
mapFromDB, err := n.readStorageLocationsFromDB(hash)
locationMap, err := n.readStorageLocationsFromDB(hash)
if err != nil {
return nil, err
}
if len(mapFromDB) == 0 {
return make(map[NodeId]*StorageLocation), nil
if len(locationMap) == 0 {
return make(map[encoding.NodeIdCode]*StorageLocation), nil
}
ts := time.Now().Unix()
for _, t := range types {
nodeMap, ok := mapFromDB[t]
nodeMap, ok := (locationMap)[t]
if !ok {
continue
}
for key, value := range nodeMap {
if len(value) < 4 {
continue // or handle error
continue
}
expiry, ok := value[3].(int64)
@ -106,12 +107,12 @@ func (n *Node) GetCachedStorageLocations(hash Multihash, types []int) (map[NodeI
addresses, ok := value[1].([]string)
if !ok {
continue // or handle error
continue
}
storageLocation := NewStorageLocation(t, addresses, expiry)
if len(value) > 4 {
if providerMessage, ok := value[4].(string); ok {
if providerMessage, ok := value[4].([]byte); ok {
storageLocation.ProviderMessage = providerMessage
}
}
@ -119,103 +120,60 @@ func (n *Node) GetCachedStorageLocations(hash Multihash, types []int) (map[NodeI
locations[NodeId(key)] = storageLocation
}
}
return locations, nil
}
func (n *Node) ReadStorageLocationsFromDB(hash Multihash) (map[int]map[NodeId]map[int]interface{}, error) {
locations := make(map[int]map[NodeId]map[int]interface{})
func (n *Node) readStorageLocationsFromDB(hash *encoding.Multihash) (storageLocationMap, error) {
locationMap := newStorageLocationMap()
bytes, err := n.config.CacheDb.Get(StringifyHash(hash)) // Assume StringifyHash and CacheDb.Get are implemented
if err != nil {
return locations, nil
}
bytes := n.cacheBucket.Get(hash.FullBytes())
if bytes == nil {
return locations, nil
return locationMap, nil
}
unpacker := NewUnpacker(bytes) // Assume NewUnpacker is implemented to handle the unpacking
mapLength, err := unpacker.UnpackMapLength()
err := msgpack.Unmarshal(bytes, locationMap)
if err != nil {
return nil, err
}
for i := 0; i < mapLength; i++ {
t, err := unpacker.UnpackInt()
if err != nil {
continue // or handle error
return locationMap, nil
}
innerMap := make(map[NodeId]map[int]interface{})
locations[t] = innerMap
innerMapLength, err := unpacker.UnpackMapLength()
if err != nil {
continue // or handle error
}
for j := 0; j < innerMapLength; j++ {
nodeIdBytes, err := unpacker.UnpackBinary()
if err != nil {
continue // or handle error
}
nodeId := NodeId(nodeIdBytes)
// Assuming unpacker.UnpackMap() returns a map[string]interface{} and is implemented
unpackedMap, err := unpacker.UnpackMap()
if err != nil {
continue // or handle error
}
convertedMap := make(map[int]interface{})
for key, value := range unpackedMap {
intKey, err := strconv.Atoi(key)
if err != nil {
continue // or handle error
}
convertedMap[intKey] = value
}
innerMap[nodeId] = convertedMap
}
}
return locations, nil
}
func (n *Node) AddStorageLocation(hash Multihash, nodeId NodeId, location StorageLocation, message []byte, config S5Config) error {
func (n *Node) AddStorageLocation(hash *encoding.Multihash, nodeId *encoding.NodeId, location *StorageLocation, message []byte, config *NodeConfig) error {
// Read existing storage locations
mapFromDB, err := n.ReadStorageLocationsFromDB(hash)
locationDb, err := n.readStorageLocationsFromDB(hash)
if err != nil {
return err
}
// Get or create the inner map for the specific type
innerMap, exists := mapFromDB[location.Type]
innerMap, exists := locationDb[location.Type]
if !exists {
innerMap = make(map[NodeId]map[int]interface{})
mapFromDB[location.Type] = innerMap
innerMap = make(nodeStorage, 1)
innerMap[nodeId.HashCode()] = make(nodeDetailsStorage, 1)
}
// Create location map with new data
locationMap := make(map[int]interface{})
locationMap := make(map[int]interface{}, 3)
locationMap[1] = location.Parts
// locationMap[2] = location.BinaryParts // Uncomment if BinaryParts is a field of StorageLocation
locationMap[3] = location.Expiry
locationMap[4] = message
// Update the inner map with the new location
innerMap[nodeId] = locationMap
innerMap[nodeId.HashCode()] = locationMap
locationDb[location.Type] = innerMap
// Serialize the updated map and store it in the database
packedBytes, err := NewPacker().Pack(mapFromDB) // Assuming NewPacker and Pack are implemented
packedBytes, err := msgpack.Marshal(locationDb)
if err != nil {
return err
}
err = config.CacheDb.Put(StringifyHash(hash), packedBytes) // Assume CacheDb.Put and StringifyHash are implemented
err = n.cacheBucket.Put(hash.FullBytes(), packedBytes)
if err != nil {
return err
}
return nil
}
} /*
func (n *Node) DownloadBytesByHash(hash Multihash) ([]byte, error) {
dlUriProvider := NewStorageLocationProvider(n, hash, []int{storageLocationTypeFull, storageLocationTypeFile})

View File

@ -0,0 +1,109 @@
package protocol
import (
"crypto/ed25519"
"fmt"
libs5_go "git.lumeweb.com/LumeWeb/libs5-go"
"git.lumeweb.com/LumeWeb/libs5-go/encoding"
"git.lumeweb.com/LumeWeb/libs5-go/net"
"git.lumeweb.com/LumeWeb/libs5-go/types"
"git.lumeweb.com/LumeWeb/libs5-go/utils"
"github.com/emirpasic/gods/sets/hashset"
"github.com/vmihailenco/msgpack/v5"
"go.uber.org/zap"
)
var _ IncomingMessageTyped = (*StorageLocation)(nil)
type StorageLocation struct {
raw []byte
hash *encoding.Multihash
kind int
expiry int64
parts []string
publicKey []byte
signature []byte
IncomingMessageTypedImpl
IncomingMessageHandler
}
func (s *StorageLocation) DecodeMessage(dec *msgpack.Decoder) error {
data, err := dec.DecodeRaw()
if err != nil {
return err
}
s.raw = data
return nil
}
func (s *StorageLocation) HandleMessage(node *libs5_go.Node, peer *net.Peer, verifyId bool) error {
hash := encoding.NewMultihash(s.raw[1:34]) // Replace NewMultihash with appropriate function
fmt.Println("Hash:", hash)
typeOfData := s.raw[34]
expiry := utils.DecodeEndian(s.raw[35:39])
partCount := s.raw[39]
parts := []string{}
cursor := 40
for i := 0; i < int(partCount); i++ {
length := utils.DecodeEndian(s.raw[cursor : cursor+2])
cursor += 2
part := string(s.raw[cursor : cursor+int(length)])
parts = append(parts, part)
cursor += int(length)
}
publicKey := s.raw[cursor : cursor+33]
signature := s.raw[cursor+33:]
if types.HashType(publicKey[0]) != types.HashTypeEd25519 { // Replace CID_HASH_TYPES_ED25519 with actual constant
return fmt.Errorf("Unsupported public key type %d", publicKey[0])
}
if !ed25519.Verify(publicKey[1:], s.raw[:cursor], signature) {
return fmt.Errorf("Signature verification failed")
}
nodeId := encoding.NewNodeId(publicKey)
// Assuming `node` is an instance of your Node structure
err := node.AddStorageLocation(hash, nodeId, libs5_go.NewStorageLocation(int(typeOfData), parts, int64(expiry)), s.raw, node.Config()) // Implement AddStorageLocation
if err != nil {
return fmt.Errorf("Failed to add storage location: %s", err)
}
var list *hashset.Set
listVal, ok := node.HashQueryRoutingTable().Get(hash.HashCode()) // Implement HashQueryRoutingTable method
if !ok {
list = hashset.New()
}
list = listVal.(*hashset.Set)
for _, peerIdVal := range list.Values() {
peerId := peerIdVal.(*encoding.NodeId)
if peerId.Equals(nodeId) || peerId.Equals(peer) {
continue
}
if peerVal, ok := node.Services().P2P().Peers().Get(peerId.HashCode()); ok {
foundPeer := peerVal.(net.Peer)
err := foundPeer.SendMessage(s.raw)
if err != nil {
node.Logger().Error("Failed to send message", zap.Error(err))
continue
}
}
node.HashQueryRoutingTable().Remove(hash.HashCode())
}
return nil
}

112
storage.go Normal file
View File

@ -0,0 +1,112 @@
package libs5_go
import (
"fmt"
"git.lumeweb.com/LumeWeb/libs5-go/encoding"
"github.com/vmihailenco/msgpack/v5"
"strconv"
"time"
)
var (
_ msgpack.CustomDecoder = (*storageLocationMap)(nil)
)
type StorageLocation struct {
Type int
Parts []string
BinaryParts [][]byte
Expiry int64
ProviderMessage []byte
}
func NewStorageLocation(Type int, Parts []string, Expiry int64) *StorageLocation {
return &StorageLocation{
Type: Type,
Parts: Parts,
Expiry: Expiry,
}
}
func (s *StorageLocation) BytesURL() string {
return s.Parts[0]
}
func (s *StorageLocation) OutboardBytesURL() string {
if len(s.Parts) == 1 {
return s.Parts[0] + ".obao"
}
return s.Parts[1]
}
func (s *StorageLocation) String() string {
expiryDate := time.Unix(s.Expiry, 0)
return "StorageLocation(" + strconv.Itoa(s.Type) + ", " + fmt.Sprint(s.Parts) + ", expiry: " + expiryDate.Format(time.RFC3339) + ")"
}
type SignedStorageLocation struct {
NodeID encoding.NodeId
Location StorageLocation
}
func NewSignedStorageLocation(NodeID encoding.NodeId, Location StorageLocation) *SignedStorageLocation {
return &SignedStorageLocation{
NodeID: NodeID,
Location: Location,
}
}
func (ssl *SignedStorageLocation) String() string {
nodeString, _ := ssl.NodeID.ToString()
if nodeString == "" {
nodeString = "failed to decode node id"
}
return "SignedStorageLocation(" + ssl.Location.String() + ", " + nodeString + ")"
}
type storageLocationMap map[int]nodeStorage
type nodeStorage map[encoding.NodeIdCode]nodeDetailsStorage
type nodeDetailsStorage map[int]interface{}
func (s *storageLocationMap) DecodeMsgpack(dec *msgpack.Decoder) error {
temp, err := dec.DecodeUntypedMap()
if err != nil {
return err
}
if *s == nil {
*s = make(map[int]nodeStorage)
}
tempMap, ok := interface{}(temp).(storageLocationMap)
if !ok {
return fmt.Errorf("unexpected data format from msgpack decoding")
}
*s = tempMap
return nil
}
func (s storageLocationMap) EncodeMsgpack(enc *msgpack.Encoder) error {
// Create a temporary map to hold the encoded data
tempMap := make(map[int]map[encoding.NodeIdCode]map[int]interface{})
// Populate the temporary map with data from storageLocationMap
for storageKey, nodeStorages := range s {
tempNodeStorages := make(map[encoding.NodeIdCode]map[int]interface{})
for nodeId, nodeDetails := range nodeStorages {
tempNodeStorages[nodeId] = nodeDetails
}
tempMap[storageKey] = tempNodeStorages
}
// Encode the temporary map using MessagePack
return enc.Encode(tempMap)
}
func newStorageLocationMap() storageLocationMap {
return storageLocationMap{}
}