refactor: switch to using structs.SetImpl

This reverts commit 4004dd98c9.

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 LumeWeb
+Copyright (c) 2024 Hammer Technologies LLC
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

build/build.go

@@ -0,0 +1,3 @@
+package build
+
+var Version = "development"

config/config.go

@@ -0,0 +1,35 @@
+package config
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/db"
+	"git.lumeweb.com/LumeWeb/libs5-go/ed25519"
+	"go.uber.org/zap"
+)
+
+type NodeConfig struct {
+	P2P     P2PConfig `mapstructure:"p2p"`
+	KeyPair *ed25519.KeyPairEd25519
+	DB      db.KVStore
+	Logger  *zap.Logger
+	HTTP    HTTPConfig `mapstructure:"http"`
+}
+type P2PConfig struct {
+	Network                 string      `mapstructure:"network"`
+	Peers                   PeersConfig `mapstructure:"peers"`
+	MaxOutgoingPeerFailures uint        `mapstructure:"max_outgoing_peer_failures"`
+	MaxConnectionAttempts   uint        `mapstructure:"max_connection_attempts"`
+}
+
+type PeersConfig struct {
+	Initial   []string `mapstructure:"initial"`
+	Blocklist []string `mapstructure:"blocklist"`
+}
+
+type HTTPAPIConfig struct {
+	Domain string `mapstructure:"domain"`
+	Port   uint   `mapstructure:"port"`
+}
+
+type HTTPConfig struct {
+	API HTTPAPIConfig `mapstructure:"api"`
+}

db/bboltdb.go

@@ -0,0 +1,133 @@
+package db
+
+import (
+	"errors"
+	"go.etcd.io/bbolt"
+)
+
+var _ KVStore = (*BboltDBKVStore)(nil)
+
+type BboltDBKVStore struct {
+	db         *bbolt.DB
+	bucket     *bbolt.Bucket
+	bucketName string
+	root       bool
+	dbPath     string
+}
+
+func (b *BboltDBKVStore) Open() error {
+	if b.root && b.db == nil {
+		db, err := bbolt.Open(b.dbPath, 0666, nil)
+		if err != nil {
+			return err
+		}
+		b.db = db
+	}
+
+	if len(b.bucketName) > 0 {
+		err := b.db.Update(func(txn *bbolt.Tx) error {
+			var bucket *bbolt.Bucket
+			var err error
+
+			if b.bucket == nil {
+				bucket, err = txn.CreateBucketIfNotExists([]byte(b.bucketName))
+				if err != nil {
+					return err
+				}
+			} else {
+				bucket, err = b.bucket.CreateBucketIfNotExists([]byte(b.bucketName))
+				if err != nil {
+					return err
+				}
+			}
+
+			b.bucket = bucket
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (b *BboltDBKVStore) Close() error {
+	if b.root && b.db != nil {
+		err := b.db.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (b *BboltDBKVStore) Get(key []byte) ([]byte, error) {
+	if b.root {
+		return nil, errors.New("Cannot get from root")
+	}
+
+	var val []byte
+	err := b.db.View(func(txn *bbolt.Tx) error {
+		bucket := txn.Bucket([]byte(b.bucketName))
+		val = bucket.Get(key)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return val, nil
+}
+
+func (b *BboltDBKVStore) Put(key []byte, value []byte) error {
+
+	if b.root {
+		return errors.New("Cannot put from root")
+	}
+
+	err := b.db.Update(func(txn *bbolt.Tx) error {
+		bucket := txn.Bucket([]byte(b.bucketName))
+		err := bucket.Put(key, value)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	return err
+}
+
+func (b *BboltDBKVStore) Delete(key []byte) error {
+	if b.root {
+		return errors.New("Cannot delete from root")
+	}
+
+	err := b.db.Update(func(txn *bbolt.Tx) error {
+		bucket := txn.Bucket([]byte(b.bucketName))
+		err := bucket.Delete(key)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	return err
+}
+
+func (b *BboltDBKVStore) Bucket(prefix string) (KVStore, error) {
+	return &BboltDBKVStore{
+		db:         b.db,
+		bucket:     b.bucket,
+		bucketName: prefix,
+		root:       false,
+	}, nil
+}
+
+func NewBboltDBKVStore(dbPath string) *BboltDBKVStore {
+	return &BboltDBKVStore{
+		dbPath: dbPath,
+		root:   true,
+	}
+}

db/db.go

@@ -0,0 +1,10 @@
+package db
+
+type KVStore interface {
+	Open() error
+	Close() error
+	Get(key []byte) ([]byte, error)
+	Put(key []byte, value []byte) error
+	Delete(key []byte) error
+	Bucket(prefix string) (KVStore, error)
+}

ed25519/ed25519.go

@@ -0,0 +1,29 @@
+package ed25519
+
+import (
+	"crypto/ed25519"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"git.lumeweb.com/LumeWeb/libs5-go/utils"
+)
+
+type KeyPairEd25519 struct {
+	Bytes []byte
+}
+
+func New(bytes []byte) *KeyPairEd25519 {
+	return &KeyPairEd25519{Bytes: bytes}
+}
+
+func (kp *KeyPairEd25519) PublicKey() []byte {
+	return utils.ConcatBytes([]byte{byte(types.HashTypeEd25519)}, kp.PublicKeyRaw())
+}
+
+func (kp *KeyPairEd25519) PublicKeyRaw() []byte {
+	publicKey := ed25519.PrivateKey(kp.Bytes).Public()
+
+	return publicKey.(ed25519.PublicKey)
+}
+
+func (kp *KeyPairEd25519) ExtractBytes() []byte {
+	return kp.Bytes
+}

encoding/cid.go

@@ -0,0 +1,301 @@
+package encoding
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"git.lumeweb.com/LumeWeb/libs5-go/internal/bases"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"git.lumeweb.com/LumeWeb/libs5-go/utils"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+var (
+	errEmptyBytes       = errors.New("empty bytes")
+	errInvalidInputType = errors.New("invalid input type for bytes")
+)
+
+type CID struct {
+	Multibase
+	Type types.CIDType
+	Hash Multihash
+	Size uint64
+}
+
+var _ json.Marshaler = (*CID)(nil)
+var _ json.Unmarshaler = (*CID)(nil)
+var _ msgpack.CustomEncoder = (*CID)(nil)
+var _ msgpack.CustomDecoder = (*CID)(nil)
+
+func NewCID(Type types.CIDType, Hash Multihash, Size uint64) *CID {
+	c := &CID{
+		Type: Type,
+		Hash: Hash,
+		Size: Size,
+	}
+	m := NewMultibase(c)
+	c.Multibase = m
+
+	return c
+}
+
+func (cid *CID) getPrefixBytes() []byte {
+	return []byte{byte(cid.Type)}
+}
+
+func (cid *CID) ToBytes() []byte {
+	if cid.Type == types.CIDTypeBridge {
+		return cid.Hash.fullBytes
+	} else if cid.Type == types.CIDTypeRaw {
+		sizeBytes := utils.EncodeEndian(cid.Size, 8)
+
+		for len(sizeBytes) > 0 && sizeBytes[len(sizeBytes)-1] == 0 {
+			sizeBytes = sizeBytes[:len(sizeBytes)-1]
+		}
+		if len(sizeBytes) == 0 {
+			sizeBytes = []byte{0}
+		}
+
+		return utils.ConcatBytes(cid.getPrefixBytes(), cid.Hash.fullBytes, sizeBytes)
+	}
+
+	return utils.ConcatBytes(cid.getPrefixBytes(), cid.Hash.fullBytes)
+}
+
+func CIDFromString(cid string) (*CID, error) {
+	decodedBytes, err := MultibaseDecodeString(cid)
+	if err != nil {
+		return nil, err
+	}
+
+	cidInstance, err := initCID(decodedBytes)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return cidInstance, nil
+}
+
+func CIDFromRegistry(bytes []byte) (*CID, error) {
+	if len(bytes) == 0 {
+		return nil, errEmptyBytes
+	}
+
+	registryType := types.RegistryType(bytes[0])
+	if _, exists := types.RegistryTypeMap[registryType]; !exists {
+		return nil, fmt.Errorf("invalid registry type %d", bytes[0])
+	}
+
+	bytes = bytes[1:]
+
+	cidInstance, err := initCID(bytes)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return cidInstance, nil
+}
+
+func CIDFromBytes(bytes []byte) (*CID, error) {
+	return initCID(bytes)
+}
+
+func CIDFromHash(bytes interface{}, size uint64, cidType types.CIDType, hashType types.HashType) (*CID, error) {
+	var (
+		byteSlice []byte
+		err       error
+	)
+
+	switch v := bytes.(type) {
+	case string:
+		byteSlice, err = hex.DecodeString(v)
+		if err != nil {
+			return nil, err
+		}
+	case []byte:
+		byteSlice = v
+	default:
+		return nil, errInvalidInputType
+	}
+
+	if _, exists := types.CIDTypeMap[cidType]; !exists {
+		return nil, fmt.Errorf("invalid hash type %d", cidType)
+	}
+
+	return NewCID(cidType, *MultihashFromBytes(byteSlice, hashType), size), nil
+}
+
+func CIDVerify(bytes interface{}) bool {
+	var (
+		byteSlice []byte
+		err       error
+	)
+
+	switch v := bytes.(type) {
+	case string:
+		byteSlice, err = MultibaseDecodeString(v) // Assuming MultibaseDecodeString function is defined
+		if err != nil {
+			return false
+		}
+	case []byte:
+		byteSlice = v
+	default:
+		return false
+	}
+
+	_, err = initCID(byteSlice)
+	return err == nil
+}
+
+func (cid *CID) CopyWith(newType int, newSize uint64) (*CID, error) {
+	if newType == 0 {
+		newType = int(cid.Type)
+	}
+
+	if _, exists := types.CIDTypeMap[types.CIDType(newType)]; !exists {
+		return nil, fmt.Errorf("invalid cid type %d", newType)
+	}
+
+	return NewCID(types.CIDType(newType), cid.Hash, newSize), nil
+}
+
+func (cid *CID) ToRegistryEntry() []byte {
+	registryType := types.RegistryTypeCID
+	cidBytes := cid.ToBytes()
+	return utils.ConcatBytes([]byte{byte(registryType)}, cidBytes)
+}
+
+func (cid *CID) ToRegistryCID() ([]byte, error) {
+	registryCIDType := types.CIDTypeResolver
+	copiedCID, err := cid.CopyWith(int(registryCIDType), cid.Size)
+	if err != nil {
+		return nil, err
+	}
+	return copiedCID.ToBytes(), nil
+}
+
+func (cid *CID) ToString() (string, error) {
+	if cid.Type == types.CIDTypeBridge {
+		return cid.Hash.ToString()
+	}
+
+	return bases.ToBase58BTC(cid.ToBytes())
+}
+
+func (cid *CID) Equals(other *CID) bool {
+	return bytes.Equal(cid.ToBytes(), other.ToBytes())
+}
+
+func (cid *CID) HashCode() int {
+	fullBytes := cid.ToBytes()
+	if len(fullBytes) < 4 {
+		return 0
+	}
+
+	return int(fullBytes[0]) +
+		int(fullBytes[1])<<8 +
+		int(fullBytes[2])<<16 +
+		int(fullBytes[3])<<24
+}
+
+func (b CID) MarshalJSON() ([]byte, error) {
+	url, err := b.ToBase64Url()
+	if err != nil {
+		return nil, err
+	}
+
+	return json.Marshal(url)
+}
+
+func (cid *CID) UnmarshalJSON(data []byte) error {
+	decData, err := UnmarshalBase64UrlJSON(data)
+
+	if err != nil {
+		return err
+	}
+
+	decodedCid, err := CIDFromBytes(decData)
+
+	if err != nil {
+		return err
+	}
+
+	*cid = *decodedCid
+	return nil
+}
+
+func (cid CID) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return enc.EncodeBytes(cid.ToBytes())
+}
+
+func (cid *CID) DecodeMsgpack(dec *msgpack.Decoder) error {
+	return decodeMsgpackCID(cid, dec)
+}
+
+func CIDFromRegistryPublicKey(pubkey interface{}) (*CID, error) {
+	return CIDFromHash(pubkey, 0, types.CIDTypeResolver, types.HashTypeEd25519)
+}
+
+func decodeMsgpackCID(cid interface{}, dec *msgpack.Decoder) error {
+	byt, err := dec.DecodeBytes()
+	if err != nil {
+		return err
+	}
+
+	switch v := cid.(type) {
+	case *CID:
+		cidInstance, err := CIDFromBytes(byt)
+		if err != nil {
+			return err
+		}
+
+		*v = *cidInstance
+	case *EncryptedCID:
+		cidInstance, err := EncryptedCIDFromBytes(byt)
+		if err != nil {
+			return err
+		}
+
+		*v = *cidInstance
+	default:
+		return errors.New("Unsupported type")
+	}
+
+	return nil
+}
+
+func initCID(bytes []byte) (*CID, error) {
+	if len(bytes) == 0 {
+		return nil, errEmptyBytes
+	}
+
+	cidType := types.CIDType(bytes[0])
+	if cidType == types.CIDTypeBridge {
+		hash := NewMultihash(bytes[1:35])
+		return NewCID(cidType, *hash, 0), nil
+	}
+
+	if len(bytes) < 34 {
+		return nil, errors.New("invalid cid")
+	}
+
+	hashBytes := bytes[1:34]
+	hash := NewMultihash(hashBytes)
+
+	var size uint64
+	if len(bytes) > 34 {
+		sizeBytes := bytes[34:]
+		sizeValue := utils.DecodeEndian(sizeBytes)
+		size = sizeValue
+	}
+
+	if _, exists := types.CIDTypeMap[cidType]; !exists {
+		return nil, fmt.Errorf("invalid cid type %d", cidType)
+	}
+
+	return NewCID(cidType, *hash, size), nil
+}

encoding/cid_test.go

@@ -0,0 +1,474 @@
+package encoding
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding/testdata"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"reflect"
+	"testing"
+)
+
+func TestCID_CopyWith(t *testing.T) {
+	type fields struct {
+		Multibase Multibase
+		Type      types.CIDType
+		Hash      Multihash
+		Size      uint32
+	}
+	type args struct {
+		newType int
+		newSize uint32
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    *CID
+		wantErr bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cid := &CID{
+				Multibase: tt.fields.Multibase,
+				Type:      tt.fields.Type,
+				Hash:      tt.fields.Hash,
+				Size:      tt.fields.Size,
+			}
+			got, err := cid.CopyWith(tt.args.newType, tt.args.newSize)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("CopyWith() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("CopyWith() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCID_Equals(t *testing.T) {
+	type fields struct {
+		Multibase Multibase
+		Type      types.CIDType
+		Hash      Multihash
+		Size      uint32
+	}
+	type args struct {
+		other *CID
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cid := &CID{
+				Multibase: tt.fields.Multibase,
+				Type:      tt.fields.Type,
+				Hash:      tt.fields.Hash,
+				Size:      tt.fields.Size,
+			}
+			if got := cid.Equals(tt.args.other); got != tt.want {
+				t.Errorf("Equals() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCID_HashCode(t *testing.T) {
+	type fields struct {
+		Multibase Multibase
+		Type      types.CIDType
+		Hash      Multihash
+		Size      uint32
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   int
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cid := &CID{
+				Multibase: tt.fields.Multibase,
+				Type:      tt.fields.Type,
+				Hash:      tt.fields.Hash,
+				Size:      tt.fields.Size,
+			}
+			if got := cid.HashCode(); got != tt.want {
+				t.Errorf("HashCode() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+/*func TestCID_ToBytes(t *testing.T) {
+	CIDFromHash(testdata.RawBase58CID)
+
+	println(len(testdata.RawCIDBytes))
+	println(utils.DecodeEndian(testdata.RawCIDBytes[35:]))
+	return
+	type fields struct {
+		kind types.CIDType
+		Hash Multihash
+		Size uint32
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   []byte
+	}{
+				{
+				name: "Bridge CID",
+				fields: fields{
+					kind: types.CIDTypeBridge,
+					Hash: NewMultibase(), // Replace with a valid hash value
+				},
+				want: , // Replace with the expected byte output for Bridge CID
+			},
+		{
+			name: "Raw CID with Non-Zero Size",
+			fields: fields{
+				kind: types.CIDTypeRaw,
+				Hash: *NewMultibase(testdata.RawCIDBytes[1:34]),
+				Size: utils.DecodeEndian(testdata.RawCIDBytes[34:]),
+			},
+			want: testdata.RawCIDBytes,
+		},
+			{
+				name: "Raw CID with Zero Size",
+				fields: fields{
+					kind: types.CIDTypeRaw,
+					Hash: yourHashValue, // Replace with a valid hash value
+					Size: 0,             // Zero size
+				},
+				want: yourExpectedBytesForRawCIDWithZeroSize, // Replace with the expected byte output
+			},
+			{
+				name: "Default CID",
+				fields: fields{
+					kind: types.CIDTypeDefault,
+					Hash: yourHashValue, // Replace with a valid hash value
+				},
+				want: yourExpectedBytesForDefaultCID, // Replace with the expected byte output for Default CID
+			},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cid := &CID{
+				kind: tt.fields.kind,
+				Hash: tt.fields.Hash,
+				Size: tt.fields.Size,
+			}
+			m := NewMultibase(cid)
+			cid.Multibase = m
+			if got := cid.ToBytes(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ToBytes() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}*/
+
+func TestCID_ToRegistryCID(t *testing.T) {
+	type fields struct {
+		Multibase Multibase
+		Type      types.CIDType
+		Hash      Multihash
+		Size      uint32
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		want    []byte
+		wantErr bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cid := &CID{
+				Multibase: tt.fields.Multibase,
+				Type:      tt.fields.Type,
+				Hash:      tt.fields.Hash,
+				Size:      tt.fields.Size,
+			}
+			got, err := cid.ToRegistryCID()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ToRegistryCID() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ToRegistryCID() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCID_ToRegistryEntry(t *testing.T) {
+	type fields struct {
+		Multibase Multibase
+		Type      types.CIDType
+		Hash      Multihash
+		Size      uint32
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   []byte
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cid := &CID{
+				Multibase: tt.fields.Multibase,
+				Type:      tt.fields.Type,
+				Hash:      tt.fields.Hash,
+				Size:      tt.fields.Size,
+			}
+			if got := cid.ToRegistryEntry(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ToRegistryEntry() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCID_ToString(t *testing.T) {
+	type fields struct {
+		Multibase Multibase
+		Type      types.CIDType
+		Hash      Multihash
+		Size      uint32
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		want    string
+		wantErr bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cid := &CID{
+				Multibase: tt.fields.Multibase,
+				Type:      tt.fields.Type,
+				Hash:      tt.fields.Hash,
+				Size:      tt.fields.Size,
+			}
+			got, err := cid.ToString()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ToString() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ToString() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCID_getPrefixBytes(t *testing.T) {
+	type fields struct {
+		Multibase Multibase
+		Type      types.CIDType
+		Hash      Multihash
+		Size      uint32
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   []byte
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cid := &CID{
+				Multibase: tt.fields.Multibase,
+				Type:      tt.fields.Type,
+				Hash:      tt.fields.Hash,
+				Size:      tt.fields.Size,
+			}
+			if got := cid.getPrefixBytes(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("getPrefixBytes() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestDecode(t *testing.T) {
+	type args struct {
+		cid string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *CID
+		wantErr bool
+	}{
+		/*        {
+		          name:    "Valid Bridge CID",
+		          args:args  {cid: ""},
+		          want:    nil,
+		          wantErr: false,
+		      },*/
+
+		{
+			name:    "Valid Raw Base 58 CID",
+			args:    args{cid: testdata.RawBase58CID},
+			want:    NewCID(types.CIDTypeRaw, *NewMultihash(testdata.RawCIDBytes[1:34]), testdata.RawCIDSize),
+			wantErr: false,
+		},
+		{
+			name:    "Valid Media 58 CID",
+			args:    args{cid: testdata.MediaBase58CID},
+			want:    NewCID(types.CIDTypeMetadataMedia, *NewMultihash(testdata.MediaCIDBytes[1:34]), testdata.MediaCIDSize),
+			wantErr: false,
+		},
+		{
+			name:    "Valid Resolver CID",
+			args:    args{cid: testdata.ResolverBase58CID},
+			want:    NewCID(types.CIDTypeResolver, *NewMultihash(testdata.ResolverCIDBytes[1:34]), testdata.ResolverCIDSize),
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := CIDFromString(tt.args.cid)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("DecodeCID() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("DecodeCID() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestFromBytes(t *testing.T) {
+	type args struct {
+		bytes []byte
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *CID
+		wantErr bool
+	}{
+		{
+			name:    "Valid Raw Base 58 CID",
+			args:    args{bytes: testdata.RawCIDBytes},
+			want:    NewCID(types.CIDTypeRaw, *NewMultihash(testdata.RawCIDBytes[1:34]), testdata.RawCIDSize),
+			wantErr: false,
+		},
+		{
+			name:    "Valid Media 58 CID",
+			args:    args{bytes: testdata.MediaCIDBytes},
+			want:    NewCID(types.CIDTypeMetadataMedia, *NewMultihash(testdata.MediaCIDBytes[1:34]), testdata.MediaCIDSize),
+			wantErr: false,
+		},
+		{
+			name:    "Valid Resolver CID",
+			args:    args{bytes: testdata.ResolverCIDBytes},
+			want:    NewCID(types.CIDTypeResolver, *NewMultihash(testdata.ResolverCIDBytes[1:34]), testdata.ResolverCIDSize),
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := CIDFromBytes(tt.args.bytes)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("CIDFromBytes() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("CIDFromBytes() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestFromHash(t *testing.T) {
+	type args struct {
+		bytes   interface{}
+		size    uint32
+		cidType types.CIDType
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *CID
+		wantErr bool
+	}{
+		{
+			name:    "Valid Raw Base 58 CID",
+			args:    args{bytes: testdata.RawCIDBytes[1:34], size: testdata.RawCIDSize, cidType: types.CIDTypeRaw},
+			want:    NewCID(types.CIDTypeRaw, *NewMultihash(testdata.RawCIDBytes[1:34]), testdata.RawCIDSize),
+			wantErr: false,
+		},
+		{
+			name:    "Valid Media 58 CID",
+			args:    args{bytes: testdata.MediaCIDBytes[1:34], size: testdata.MediaCIDSize, cidType: types.CIDTypeMetadataMedia},
+			want:    NewCID(types.CIDTypeMetadataMedia, *NewMultihash(testdata.MediaCIDBytes[1:34]), testdata.MediaCIDSize),
+			wantErr: false,
+		},
+		{
+			name:    "Valid Resolver CID",
+			args:    args{bytes: testdata.ResolverCIDBytes[1:34], size: testdata.ResolverCIDSize, cidType: types.CIDTypeResolver},
+			want:    NewCID(types.CIDTypeResolver, *NewMultihash(testdata.ResolverCIDBytes[1:34]), testdata.ResolverCIDSize),
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := CIDFromHash(tt.args.bytes, tt.args.size, tt.args.cidType)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("CIDFromHash() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("CIDFromHash() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestFromRegistry(t *testing.T) {
+	type args struct {
+		bytes []byte
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *CID
+		wantErr bool
+	}{
+		{
+			name:    "Valid Resolver Data",
+			args:    args{bytes: testdata.ResolverDataBytes},
+			want:    NewCID(types.CIDTypeMetadataWebapp, *NewMultihash(testdata.ResolverDataBytes[2:35]), testdata.ResolverDataSize),
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := CIDFromRegistry(tt.args.bytes)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("CIDFromRegistry() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("CIDFromRegistry() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

encoding/encoding.go

@@ -0,0 +1,30 @@
+package encoding
+
+import "encoding/base64"
+
+func UnmarshalBase64UrlJSON(data []byte) ([]byte, error) {
+	strData := string(data)
+	if len(strData) >= 2 && strData[0] == '"' && strData[len(strData)-1] == '"' {
+		strData = strData[1 : len(strData)-1]
+	}
+
+	if strData == "null" {
+		return nil, nil
+	}
+
+	decodedData, err := MultibaseDecodeString(strData)
+	if err != nil {
+		if err != ErrMultibaseEncodingNotSupported {
+			return nil, err
+		}
+	} else {
+		return decodedData, nil
+	}
+
+	decodedData, err = base64.RawURLEncoding.DecodeString(strData)
+	if err != nil {
+		return nil, err
+	}
+
+	return decodedData, nil
+}

encoding/encrypted_cid.go

@@ -0,0 +1,117 @@
+package encoding
+
+import (
+	"encoding/json"
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"git.lumeweb.com/LumeWeb/libs5-go/utils"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+type EncryptedCID struct {
+	Multibase
+	encryptedBlobHash   Multihash
+	OriginalCID         CID
+	encryptionAlgorithm byte
+	padding             uint64
+	chunkSizeAsPowerOf2 int
+	encryptionKey       []byte
+}
+
+var _ msgpack.CustomEncoder = (*EncryptedCID)(nil)
+var _ msgpack.CustomDecoder = (*EncryptedCID)(nil)
+var _ json.Marshaler = (*EncryptedCID)(nil)
+var _ json.Unmarshaler = (*EncryptedCID)(nil)
+
+func NewEncryptedCID(encryptedBlobHash Multihash, originalCID CID, encryptionKey []byte, padding uint64, chunkSizeAsPowerOf2 int, encryptionAlgorithm byte) *EncryptedCID {
+	e := &EncryptedCID{
+		encryptedBlobHash:   encryptedBlobHash,
+		OriginalCID:         originalCID,
+		encryptionKey:       encryptionKey,
+		padding:             padding,
+		chunkSizeAsPowerOf2: chunkSizeAsPowerOf2,
+		encryptionAlgorithm: encryptionAlgorithm,
+	}
+
+	m := NewMultibase(e)
+	e.Multibase = m
+
+	return e
+}
+
+func DecodeEncryptedCID(cid string) (*EncryptedCID, error) {
+	data, err := MultibaseDecodeString(cid)
+	if err != nil {
+		return nil, err
+	}
+	return EncryptedCIDFromBytes(data)
+}
+
+func EncryptedCIDFromBytes(data []byte) (*EncryptedCID, error) {
+	if types.CIDType(data[0]) != types.CIDTypeEncryptedStatic {
+		return nil, errors.New("Invalid CID type")
+	}
+
+	cid, err := CIDFromBytes(data[72:])
+	if err != nil {
+		return nil, err
+	}
+
+	encryptedBlobHash := NewMultihash(data[3:36])
+	encryptionKey := data[36:68]
+	padding := utils.DecodeEndian(data[68:72])
+	chunkSizeAsPowerOf2 := int(data[2])
+	encryptionAlgorithm := data[1]
+
+	return NewEncryptedCID(*encryptedBlobHash, *cid, encryptionKey, padding, chunkSizeAsPowerOf2, encryptionAlgorithm), nil
+}
+
+func (c *EncryptedCID) ChunkSize() int {
+	return 1 << uint(c.chunkSizeAsPowerOf2)
+}
+
+func (c *EncryptedCID) ToBytes() []byte {
+	data := []byte{
+		byte(types.CIDTypeEncryptedStatic),
+		c.encryptionAlgorithm,
+		byte(c.chunkSizeAsPowerOf2),
+	}
+	data = append(data, c.encryptedBlobHash.fullBytes...)
+	data = append(data, c.encryptionKey...)
+	data = append(data, utils.EncodeEndian(c.padding, 4)...)
+	data = append(data, c.OriginalCID.ToBytes()...)
+	return data
+}
+func (c EncryptedCID) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return enc.EncodeBytes(c.ToBytes())
+}
+
+func (c *EncryptedCID) DecodeMsgpack(dec *msgpack.Decoder) error {
+	return decodeMsgpackCID(c, dec)
+}
+func (c EncryptedCID) MarshalJSON() ([]byte, error) {
+	str, err := c.ToString()
+
+	if err != nil {
+		return nil, err
+	}
+	// Delegate to the MarshalJSON method of the Encoder
+	return json.Marshal(str)
+}
+
+func (c *EncryptedCID) UnmarshalJSON(data []byte) error {
+	decData, err := UnmarshalBase64UrlJSON(data)
+
+	if err != nil {
+		return err
+	}
+
+	decodedCid, err := EncryptedCIDFromBytes(decData)
+
+	if err != nil {
+		return err
+	}
+
+	*c = *decodedCid
+	return nil
+}

encoding/multibase.go

@@ -0,0 +1,72 @@
+package encoding
+
+import (
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/internal/bases"
+	"github.com/multiformats/go-multibase"
+)
+
+var (
+	ErrMultibaseEncodingNotSupported = errors.New("multibase encoding not supported")
+	errMultibaseDecodeZeroLength     = errors.New("cannot decode multibase for zero length string")
+)
+
+type Encoder interface {
+	ToBytes() []byte
+}
+
+type multibaseImpl struct {
+	Multibase
+	Encoder Encoder
+}
+
+type Multibase interface {
+	ToHex() (string, error)
+	ToBase32() (string, error)
+	ToBase64Url() (string, error)
+	ToBase58() (string, error)
+	ToString() (string, error)
+}
+
+var _ Multibase = (*multibaseImpl)(nil)
+
+func NewMultibase(encoder Encoder) Multibase {
+	return &multibaseImpl{Encoder: encoder}
+}
+
+func MultibaseDecodeString(data string) (bytes []byte, err error) {
+	if len(data) == 0 {
+		return nil, errMultibaseDecodeZeroLength
+	}
+
+	switch data[0] {
+	case 'z', 'f', 'u', 'b':
+		_, bytes, err = multibase.Decode(data)
+	case ':':
+		bytes = []byte(data)
+	default:
+		err = ErrMultibaseEncodingNotSupported
+	}
+
+	return bytes, err
+}
+
+func (m *multibaseImpl) ToHex() (string, error) {
+	return bases.ToHex(m.Encoder.ToBytes())
+}
+
+func (m *multibaseImpl) ToBase32() (string, error) {
+	return bases.ToBase32(m.Encoder.ToBytes())
+}
+
+func (m *multibaseImpl) ToBase64Url() (string, error) {
+	return bases.ToBase64Url(m.Encoder.ToBytes())
+}
+
+func (m *multibaseImpl) ToBase58() (string, error) {
+	return bases.ToBase58BTC(m.Encoder.ToBytes())
+}
+
+func (m *multibaseImpl) ToString() (string, error) {
+	return m.ToBase58()
+}

encoding/multibase_test.go

@@ -0,0 +1,257 @@
+package encoding
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding/testdata"
+	"reflect"
+	"testing"
+)
+
+type encoder struct {
+	Multibase
+	data []byte
+}
+
+func (e *encoder) ToBytes() []byte {
+	return e.data
+}
+
+func newEncoder(data []byte) encoder {
+	e := &encoder{data: data}
+	m := NewMultibase(e)
+	e.Multibase = m
+
+	return *e
+}
+
+func TestDecodeString(t *testing.T) {
+	type args struct {
+		data string
+	}
+	tests := []struct {
+		name      string
+		args      args
+		wantBytes []byte
+		wantErr   bool
+	}{
+		{
+			name:      "TestValidMultibase_z",
+			args:      args{data: testdata.MediaBase58CID},
+			wantBytes: testdata.MediaCIDBytes, // Adjust this based on the expected output of multibase.CIDFromString("zabc")
+			wantErr:   false,
+		},
+		{
+			name:      "TestValidMultibase_f",
+			args:      args{data: testdata.MediaBase16CID},
+			wantBytes: testdata.MediaCIDBytes, // Adjust this based on the expected output of multibase.CIDFromString("fxyz")
+			wantErr:   false,
+		},
+		{
+			name:      "TestValidMultibase_u",
+			args:      args{data: testdata.MediaBase64CID},
+			wantBytes: testdata.MediaCIDBytes, // Adjust this based on the expected output of multibase.CIDFromString("uhello")
+			wantErr:   false,
+		},
+		{
+			name:      "TestValidMultibase_b",
+			args:      args{data: testdata.MediaBase32CID},
+			wantBytes: testdata.MediaCIDBytes, // Adjust this based on the expected output of multibase.CIDFromString("bworld")
+			wantErr:   false,
+		},
+		/*		{
+				name:      "TestColonPrefix",
+				args:      args{data: ":data"},
+				wantBytes: []byte(":data"),
+				wantErr:   false,
+			},*/
+		{
+			name:      "TestUnsupportedPrefix",
+			args:      args{data: "xunsupported"},
+			wantBytes: nil,
+			wantErr:   true,
+		},
+		{
+			name:      "TestEmptyInput",
+			args:      args{data: ""},
+			wantBytes: nil,
+			wantErr:   true,
+		}, /*
+			{
+				name:      "TestColonOnlyInput",
+				args:      args{data: ":"},
+				wantBytes: []byte(":"),
+				wantErr:   false,
+			},*/
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotBytes, err := MultibaseDecodeString(tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("MultibaseDecodeString() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotBytes, tt.wantBytes) {
+				t.Errorf("MultibaseDecodeString() gotBytes = %v, want %v", gotBytes, tt.wantBytes)
+			}
+		})
+	}
+}
+
+func TestMultibase_ToBase32(t *testing.T) {
+
+	tests := []struct {
+		name    string
+		encoder encoder
+		want    string
+		wantErr bool
+	}{
+		{
+			name:    "Is Raw CID",
+			encoder: newEncoder(testdata.RawCIDBytes),
+			want:    testdata.RawBase32CID,
+			wantErr: false,
+		}, {
+			name:    "Is Media CID",
+			encoder: newEncoder(testdata.MediaCIDBytes),
+			want:    testdata.MediaBase32CID,
+			wantErr: false,
+		}, {
+			name:    "Is Resolver CID",
+			encoder: newEncoder(testdata.ResolverCIDBytes),
+			want:    testdata.ResolverBase32CID,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.encoder.ToBase32()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ToBase32() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ToBase32() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestMultibase_ToBase58(t *testing.T) {
+	tests := []struct {
+		name    string
+		encoder encoder
+		want    string
+		wantErr bool
+	}{
+		{
+			name:    "Is Raw CID",
+			encoder: newEncoder(testdata.RawCIDBytes),
+			want:    testdata.RawBase58CID,
+			wantErr: false,
+		}, {
+			name:    "Is Media CID",
+			encoder: newEncoder(testdata.MediaCIDBytes),
+			want:    testdata.MediaBase58CID,
+			wantErr: false,
+		},
+		{
+			name:    "Is Resolver CID",
+			encoder: newEncoder(testdata.ResolverCIDBytes),
+			want:    testdata.ResolverBase58CID,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.encoder.ToBase58()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ToBase58() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ToBase58() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestMultibase_ToBase64Url(t *testing.T) {
+	tests := []struct {
+		name    string
+		encoder encoder
+		want    string
+		wantErr bool
+	}{
+		{
+			name:    "Is Raw CID",
+			encoder: newEncoder(testdata.RawCIDBytes),
+			want:    testdata.RawBase64CID,
+			wantErr: false,
+		}, {
+			name:    "Is Media CID",
+			encoder: newEncoder(testdata.MediaCIDBytes),
+			want:    testdata.MediaBase64CID,
+			wantErr: false,
+		},
+		{
+			name:    "Is Resolver CID",
+			encoder: newEncoder(testdata.ResolverCIDBytes),
+			want:    testdata.ResolverBase64CID,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.encoder.ToBase64Url()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ToBase64Url() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ToBase64Url() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestMultibase_ToHex(t *testing.T) {
+	tests := []struct {
+		name    string
+		encoder encoder
+		want    string
+		wantErr bool
+	}{
+		{
+			name:    "Is Raw CID",
+			encoder: newEncoder(testdata.RawCIDBytes),
+			want:    testdata.RawBase16CID,
+			wantErr: false,
+		}, {
+			name:    "Is Media CID",
+			encoder: newEncoder(testdata.MediaCIDBytes),
+			want:    testdata.MediaBase16CID,
+			wantErr: false,
+		},
+		{
+			name:    "Is Resolver CID",
+			encoder: newEncoder(testdata.ResolverCIDBytes),
+			want:    testdata.ResolverBase16CID,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.encoder.ToHex()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ToHex() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ToHex() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestMultibase_ToString(t *testing.T) {
+	TestMultibase_ToBase58(t)
+}

encoding/multihash.go

@@ -0,0 +1,89 @@
+package encoding
+
+import (
+	"bytes"
+	"encoding/base32"
+	"encoding/base64"
+	"encoding/json"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"git.lumeweb.com/LumeWeb/libs5-go/utils"
+)
+
+type MultihashCode = int
+
+type Multihash struct {
+	fullBytes []byte
+}
+
+func (m *Multihash) FullBytes() []byte {
+	return m.fullBytes
+}
+
+var _ json.Marshaler = (*Multihash)(nil)
+var _ json.Unmarshaler = (*Multihash)(nil)
+
+func NewMultihash(fullBytes []byte) *Multihash {
+	return &Multihash{fullBytes: fullBytes}
+}
+
+func (m *Multihash) FunctionType() types.HashType {
+	return types.HashType(m.fullBytes[0])
+}
+
+func (m *Multihash) HashBytes() []byte {
+	return m.fullBytes[1:]
+}
+
+func MultihashFromBytes(bytes []byte, kind types.HashType) *Multihash {
+	return NewMultihash(append([]byte{byte(kind)}, bytes...))
+}
+
+func MultihashFromBase64Url(hash string) (*Multihash, error) {
+	ret, err := base64.RawURLEncoding.DecodeString(hash)
+	if err != nil {
+		return nil, err
+	}
+	return NewMultihash(ret), nil
+}
+
+func (m *Multihash) ToBase64Url() (string, error) {
+	return base64.RawURLEncoding.EncodeToString(m.fullBytes), nil
+}
+
+func (m *Multihash) ToBase32() (string, error) {
+	return base32.StdEncoding.EncodeToString(m.fullBytes), nil
+}
+
+func (m *Multihash) ToString() (string, error) {
+	if m.FunctionType() == types.HashType(types.CIDTypeBridge) {
+		return string(m.fullBytes), nil // Assumes the bytes are valid UTF-8
+	}
+	return m.ToBase64Url()
+}
+
+func (m *Multihash) Equals(other *Multihash) bool {
+	return bytes.Equal(m.fullBytes, other.fullBytes)
+}
+
+func (m *Multihash) HashCode() MultihashCode {
+	return utils.HashCode(m.fullBytes[:4])
+}
+
+func (b *Multihash) UnmarshalJSON(data []byte) error {
+	decodedData, err := MultihashFromBase64Url(string(data))
+	if err != nil {
+		return err
+	}
+
+	b.fullBytes = decodedData.fullBytes
+	return nil
+}
+func (b Multihash) MarshalJSON() ([]byte, error) {
+	url, err := b.ToBase64Url()
+	if err != nil {
+		return nil, err
+	}
+
+	return []byte(url), nil
+
+}

encoding/multihash_test.go

@@ -0,0 +1,211 @@
+package encoding
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding/testdata"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestFromBase64Url(t *testing.T) {
+	type args struct {
+		hash string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *Multihash
+		wantErr bool
+	}{
+		{
+			name:    "Valid Base64 URL Encoded String",
+			args:    args{hash: testdata.MediaBase64CID},
+			want:    &Multihash{fullBytes: testdata.MediaCIDBytes},
+			wantErr: false,
+		},
+		{
+			name:    "Invalid Base64 URL String",
+			args:    args{hash: "@@invalid@@"},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "Empty String",
+			args:    args{hash: ""},
+			want:    nil,
+			wantErr: true, // or false
+		},
+		{
+			name:    "Non-URL Base64 Encoded String",
+			args:    args{hash: "aGVsbG8gd29ybGQ="},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "String Not Representing a Multihash",
+			args:    args{hash: "cGxhaW50ZXh0"},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "Long String",
+			args:    args{hash: "uYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh"},
+			want:    &Multihash{fullBytes: []byte(strings.Repeat("a", 750))},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := MultihashFromBase64Url(tt.args.hash)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("MultihashFromBase64Url() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("MultihashFromBase64Url() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestMultihash_FunctionType(t *testing.T) {
+	type fields struct {
+		FullBytes []byte
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   types.HashType
+	}{
+		{
+			name: "Is Raw CID",
+			fields: fields{
+				FullBytes: testdata.RawCIDBytes[1:34],
+			},
+			want: types.HashTypeBlake3,
+		}, {
+			name: "Is Resolver CID",
+			fields: fields{
+				FullBytes: testdata.ResolverCIDBytes[1:34],
+			},
+			want: types.HashTypeEd25519,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := &Multihash{
+				fullBytes: tt.fields.FullBytes,
+			}
+			if got := m.FunctionType(); got != tt.want {
+				t.Errorf("FunctionType() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestMultihash_ToBase32(t *testing.T) {
+	type fields struct {
+		FullBytes []byte
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "Is Raw CID",
+			fields: fields{
+				FullBytes: testdata.RawCIDBytes,
+			},
+			want: "beyprdl3g2it54ua2ian3a5wybrnva6kr7kzkpv6wbfvgjsb2aepw6yc6t4eq",
+		}, {
+			name: "Is Media CID",
+			fields: fields{
+				FullBytes: testdata.MediaCIDBytes,
+			},
+			want: "byupv6i7z5g6unmx2btc2ihsrrivnox6gqnwfgiwkpuw36d6q4dl35hi",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := &Multihash{
+				fullBytes: tt.fields.FullBytes,
+			}
+			got, err := m.ToBase32()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ToBase32() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ToBase32() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestMultihash_ToBase64Url(t *testing.T) {
+	type fields struct {
+		FullBytes []byte
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "Is Raw CID",
+			fields: fields{
+				FullBytes: testdata.RawCIDBytes,
+			},
+			want: "uJh8Rr2bSJ95QGkAbsHbYDFtQeVH6sqfX1glqZMg6AR9vYF6fCQ",
+		}, {
+			name: "Is Media CID",
+			fields: fields{
+				FullBytes: testdata.MediaCIDBytes,
+			},
+			want: "uxR9fI_npvUay-gzFpB5RiirXX8aDbFMiyn0tvw_Q4Ne-nQ",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := &Multihash{
+				fullBytes: tt.fields.FullBytes,
+			}
+			got, err := m.ToBase64Url()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ToBase64Url() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ToBase64Url() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestNewMultihash(t *testing.T) {
+	type args struct {
+		fullBytes []byte
+	}
+	tests := []struct {
+		name string
+		args args
+		want *Multihash
+	}{
+		{
+			name: "Valid Base64 URL Encoded String",
+			args: args{fullBytes: testdata.RawCIDBytes},
+			want: &Multihash{fullBytes: testdata.RawCIDBytes},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := NewMultihash(tt.args.fullBytes); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("NewMultihash() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

encoding/nodeid.go

@@ -0,0 +1,63 @@
+package encoding
+
+import (
+	"bytes"
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/internal/bases"
+	"git.lumeweb.com/LumeWeb/libs5-go/utils"
+	"github.com/multiformats/go-multibase"
+)
+
+var (
+	errorNotBase58BTC = errors.New("not a base58btc string")
+)
+
+type NodeIdCode = int
+
+type NodeId struct {
+	bytes []byte
+}
+
+func (nodeId *NodeId) Bytes() []byte {
+	return nodeId.bytes
+}
+
+func NewNodeId(bytes []byte) *NodeId {
+	return &NodeId{bytes: bytes}
+}
+
+func DecodeNodeId(nodeId string) (*NodeId, error) {
+	encoding, ret, err := multibase.Decode(nodeId)
+	if err != nil {
+		return nil, err
+	}
+
+	if encoding != multibase.Base58BTC {
+		return nil, errorNotBase58BTC
+	}
+
+	return NewNodeId(ret), nil
+}
+
+func (nodeId *NodeId) Equals(other interface{}) bool {
+	if otherNodeId, ok := other.(*NodeId); ok {
+		return bytes.Equal(nodeId.bytes, otherNodeId.bytes)
+	}
+	return false
+}
+
+func (nodeId *NodeId) HashCode() int {
+	return utils.HashCode(nodeId.bytes[:4])
+}
+
+func (nodeId *NodeId) ToBase58() (string, error) {
+	return bases.ToBase58BTC(nodeId.bytes)
+}
+
+func (nodeId *NodeId) ToString() (string, error) {
+	return nodeId.ToBase58()
+}
+
+func (nodeId *NodeId) Raw() []byte {
+	return nodeId.bytes
+}

encoding/testdata/cid.go

@@ -0,0 +1,29 @@
+package testdata
+
+import "encoding/hex"
+
+var (
+	RawCIDBytes         = []byte{0x26, 0x1f, 0x11, 0xaf, 0x66, 0xd2, 0x27, 0xde, 0x50, 0x1a, 0x40, 0x1b, 0xb0, 0x76, 0xd8, 0x0c, 0x5b, 0x50, 0x79, 0x51, 0xfa, 0xb2, 0xa7, 0xd7, 0xd6, 0x09, 0x6a, 0x64, 0xc8, 0x3a, 0x01, 0x1f, 0x6f, 0x60, 0x5e, 0x9f, 0x09}
+	RawBase16CID        = "f261f11af66d227de501a401bb076d80c5b507951fab2a7d7d6096a64c83a011f6f605e9f09"
+	RawBase32CID        = "beyprdl3g2it54ua2ian3a5wybrnva6kr7kzkpv6wbfvgjsb2aepw6yc6t4eq"
+	RawBase58CID        = "z2H6yKf4s6awVkoiVJ4ARCZWLzX6eBhSaCkkqcjUCtmvqKcM4c5W"
+	RawBase64CID        = "uJh8Rr2bSJ95QGkAbsHbYDFtQeVH6sqfX1glqZMg6AR9vYF6fCQ"
+	RawCIDSize   uint32 = 630622
+
+	MediaCIDBytes         = []byte{0xc5, 0x1f, 0x5f, 0x23, 0xf9, 0xe9, 0xbd, 0x46, 0xb2, 0xfa, 0x0c, 0xc5, 0xa4, 0x1e, 0x51, 0x8a, 0x2a, 0xd7, 0x5f, 0xc6, 0x83, 0x6c, 0x53, 0x22, 0xca, 0x7d, 0x2d, 0xbf, 0x0f, 0xd0, 0xe0, 0xd7, 0xbe, 0x9d}
+	MediaBase16CID        = "fc51f5f23f9e9bd46b2fa0cc5a41e518a2ad75fc6836c5322ca7d2dbf0fd0e0d7be9d"
+	MediaBase32CID        = "byupv6i7z5g6unmx2btc2ihsrrivnox6gqnwfgiwkpuw36d6q4dl35hi"
+	MediaBase58CID        = "z5TTkenVbffNSgTcU4pkBcN2H1ZYctwLyQeLNEdr48tEpZHv"
+	MediaBase64CID        = "uxR9fI_npvUay-gzFpB5RiirXX8aDbFMiyn0tvw_Q4Ne-nQ"
+	MediaCIDSize   uint32 = 0
+
+	ResolverCIDBytes            = []byte{0x25, 0xed, 0x2f, 0x66, 0xbf, 0xfa, 0xd8, 0x19, 0xa6, 0xbf, 0x22, 0x1d, 0x26, 0xee, 0x0f, 0xfe, 0x75, 0xe4, 0x8d, 0x15, 0x4f, 0x13, 0x76, 0x1e, 0xaa, 0xe5, 0x75, 0x89, 0x6f, 0x17, 0xdb, 0xda, 0x5f, 0xd3}
+	ResolverBase16CID           = "f25ed2f66bffad819a6bf221d26ee0ffe75e48d154f13761eaae575896f17dbda5fd3"
+	ResolverBase32CID           = "bexws6zv77lmbtjv7eiosn3qp7z26jdivj4jxmhvk4v2ys3yx3pnf7uy"
+	ResolverBase58CID           = "zrjENDT9Doeok7pHUaojsYh5j3U1zKMudwTqZYNUftd8WCA"
+	ResolverBase64CID           = "uJe0vZr_62BmmvyIdJu4P_nXkjRVPE3YequV1iW8X29pf0w"
+	ResolverCIDSize      uint32 = 0
+	ResolverData                = "5a591f6f05b96e1684cab18b410d7510a08392bf4b529a954e94636d6c9a5fc639cacd"
+	ResolverDataBytes, _        = hex.DecodeString(ResolverData)
+	ResolverDataSize     uint32 = 0
+)

fx/fx.go

@@ -0,0 +1,75 @@
+package fx
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/config"
+	"git.lumeweb.com/LumeWeb/libs5-go/db"
+	"git.lumeweb.com/LumeWeb/libs5-go/node"
+	"git.lumeweb.com/LumeWeb/libs5-go/service"
+	_default "git.lumeweb.com/LumeWeb/libs5-go/service/default"
+	"go.uber.org/fx"
+	"go.uber.org/zap"
+)
+
+var Module = fx.Module("libs5",
+	fx.Provide(newP2P),
+	fx.Provide(newRegistry),
+	fx.Provide(newHTTP),
+	fx.Provide(newStorage),
+	fx.Provide(newServices),
+	fx.Provide(node.NewNode),
+)
+
+type ServiceParams struct {
+	fx.In
+	Logger *zap.Logger
+	Config *config.NodeConfig
+	Db     db.KVStore
+}
+
+type ServicesParams struct {
+	fx.In
+	P2P      service.P2PService
+	Registry service.RegistryService
+	HTTP     service.HTTPService
+	Storage  service.StorageService
+}
+
+func newP2P(params ServiceParams) service.P2PService {
+	return _default.NewP2P(service.ServiceParams{
+		Logger: params.Logger,
+		Config: params.Config,
+		Db:     params.Db,
+	})
+}
+
+func newRegistry(params ServiceParams) service.RegistryService {
+	return _default.NewRegistry(service.ServiceParams{
+		Logger: params.Logger,
+		Config: params.Config,
+		Db:     params.Db,
+	})
+}
+func newHTTP(params ServiceParams) service.HTTPService {
+	return _default.NewHTTP(service.ServiceParams{
+		Logger: params.Logger,
+		Config: params.Config,
+		Db:     params.Db,
+	})
+}
+
+func newStorage(params ServiceParams) service.StorageService {
+	return _default.NewStorage(service.ServiceParams{
+		Logger: params.Logger,
+		Config: params.Config,
+		Db:     params.Db,
+	})
+}
+
+func newServices(params ServicesParams) service.Services {
+	return node.NewServices(node.ServicesParams{
+		P2P:      params.P2P,
+		Registry: params.Registry,
+		HTTP:     params.HTTP,
+		Storage:  params.Storage,
+	})
+}

go.mod

@@ -0,0 +1,39 @@
+module git.lumeweb.com/LumeWeb/libs5-go
+
+go 1.21.6
+
+require (
+	github.com/ddo/rq v0.0.0-20190828174524-b3daa55fcaba
+	github.com/emirpasic/gods v1.18.1
+	github.com/golang/mock v1.6.0
+	github.com/google/go-cmp v0.6.0
+	github.com/julienschmidt/httprouter v1.3.0
+	github.com/multiformats/go-multibase v0.2.0
+	github.com/olebedev/emitter v0.0.0-20230411050614-349169dec2ba
+	github.com/samber/lo v1.39.0
+	github.com/stretchr/testify v1.8.1
+	github.com/vmihailenco/msgpack/v5 v5.4.1
+	go.etcd.io/bbolt v1.3.8
+	go.sia.tech/jape v0.11.1
+	go.uber.org/fx v1.20.1
+	go.uber.org/zap v1.26.0
+	lukechampine.com/blake3 v1.2.1
+	nhooyr.io/websocket v1.7.1
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/mr-tron/base58 v1.1.0 // indirect
+	github.com/multiformats/go-base32 v0.0.3 // indirect
+	github.com/multiformats/go-base36 v0.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	go.uber.org/dig v1.17.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -0,0 +1,115 @@
+github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
+github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/ddo/rq v0.0.0-20190828174524-b3daa55fcaba h1:EMLQSxP68m4ddJpmMT+LizYO0AjrROprPXjll2CARj0=
+github.com/ddo/rq v0.0.0-20190828174524-b3daa55fcaba/go.mod h1:XIayI7kdKklkc7yyWDBYMJLbK/AO4AchQUxdoSFcn+k=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mr-tron/base58 v1.1.0 h1:Y51FGVJ91WBqCEabAi5OPUz38eAx8DakuAm5svLcsfQ=
+github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
+github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
+github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
+github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
+github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
+github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
+github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
+github.com/olebedev/emitter v0.0.0-20230411050614-349169dec2ba h1:/Q5vvLs180BFH7u+Nakdrr1B9O9RAxVaIurFQy0c8QQ=
+github.com/olebedev/emitter v0.0.0-20230411050614-349169dec2ba/go.mod h1:eT2/Pcsim3XBjbvldGiJBvvgiqZkAFyiOJJsDKXs/ts=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
+github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
+github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
+github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
+github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
+go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.sia.tech/jape v0.11.1 h1:M7IP+byXL7xOqzxcHUQuXW+q3sYMkYzmMlMw+q8ZZw0=
+go.sia.tech/jape v0.11.1/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI=
+go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU=
+go.uber.org/fx v1.20.1 h1:zVwVQGS8zYvhh9Xxcu4w1M6ESyeMzebzj2NbSayZ4Mk=
+go.uber.org/fx v1.20.1/go.mod h1:iSYNbHf2y55acNCwCXKx7LbWb5WG1Bnue5RDXz1OREg=
+go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
+go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
+go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM=
+golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/s5net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/s5net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/s5net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI=
+lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
+nhooyr.io/websocket v1.7.1 h1:W8X03PI0a2fiKb3Srck2gpmWr8UwJqDGndXw8bb93yM=
+nhooyr.io/websocket v1.7.1/go.mod h1:FyTYp9aYEPchTiPpXj2mOOnHJ49S35YStWZCjotwizg=

internal/bases/bases.go

@@ -0,0 +1,30 @@
+package bases
+
+import "github.com/multiformats/go-multibase"
+
+func ToBase64Url(data []byte) (string, error) {
+	return ToBase(data, "base64url")
+}
+
+func ToBase58BTC(data []byte) (string, error) {
+	return ToBase(data, "base58btc")
+}
+
+func ToBase32(data []byte) (string, error) {
+	return ToBase(data, "base32")
+}
+
+func ToHex(data []byte) (string, error) {
+	return ToBase(data, "base16")
+}
+
+func ToBase(data []byte, base string) (string, error) {
+	baseEncoder, _ := multibase.EncoderByName(base)
+
+	ret, err := multibase.Encode(baseEncoder.Encoding(), data)
+	if err != nil {
+		return "", err
+	}
+
+	return ret, nil
+}

metadata/directory.go

@@ -0,0 +1,96 @@
+package metadata
+
+import (
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/serialize"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+type DirectoryMetadata struct {
+	Details       DirectoryMetadataDetails `json:"details"`
+	Directories   directoryReferenceMap    `json:"directories"`
+	Files         fileReferenceMap         `json:"files"`
+	ExtraMetadata ExtraMetadata            `json:"extraMetadata"`
+	BaseMetadata
+}
+
+var _ SerializableMetadata = (*DirectoryMetadata)(nil)
+
+func NewEmptyDirectoryMetadata() *DirectoryMetadata {
+	return &DirectoryMetadata{}
+}
+
+func NewDirectoryMetadata(details DirectoryMetadataDetails, directories directoryReferenceMap, files fileReferenceMap, extraMetadata ExtraMetadata) *DirectoryMetadata {
+	dirMetadata := &DirectoryMetadata{
+		Details:       details,
+		Directories:   directories,
+		Files:         files,
+		ExtraMetadata: extraMetadata,
+	}
+
+	dirMetadata.Type = "directory"
+	return dirMetadata
+}
+func (dm *DirectoryMetadata) EncodeMsgpack(enc *msgpack.Encoder) error {
+	err := serialize.InitMarshaller(enc, types.MetadataTypeDirectory)
+	if err != nil {
+		return err
+	}
+
+	items := make([]interface{}, 4)
+
+	items[0] = dm.Details
+	items[1] = dm.Directories
+	items[2] = dm.Files
+	items[3] = dm.ExtraMetadata.Data
+
+	return enc.Encode(items)
+}
+
+func (dm *DirectoryMetadata) DecodeMsgpack(dec *msgpack.Decoder) error {
+	_, err := serialize.InitUnmarshaller(dec, types.MetadataTypeDirectory)
+	if err != nil {
+		return err
+	}
+	val, err := dec.DecodeArrayLen()
+
+	if err != nil {
+		return err
+	}
+
+	if val != 4 {
+		return errors.New(" Corrupted metadata")
+	}
+
+	for i := 0; i < val; i++ {
+		switch i {
+		case 0:
+			err = dec.Decode(&dm.Details)
+			if err != nil {
+				return err
+			}
+		case 1:
+			err = dec.Decode(&dm.Directories)
+			if err != nil {
+				return err
+			}
+
+		case 2:
+			err = dec.Decode(&dm.Files)
+			if err != nil {
+				return err
+			}
+		case 3:
+			intMap, err := decodeIntMap(dec)
+			if err != nil {
+				return err
+			}
+			dm.ExtraMetadata.Data = intMap
+		}
+	}
+
+	dm.Type = "directory"
+
+	return nil
+}

metadata/directory_details.go

@@ -0,0 +1,78 @@
+package metadata
+
+import "github.com/vmihailenco/msgpack/v5"
+
+type DirectoryMetadataDetails struct {
+	Data map[int]interface{}
+}
+
+func NewDirectoryMetadataDetails(data map[int]interface{}) *DirectoryMetadataDetails {
+	return &DirectoryMetadataDetails{
+		Data: data,
+	}
+}
+
+func (dmd *DirectoryMetadataDetails) IsShared() bool {
+	_, exists := dmd.Data[3]
+	return exists
+}
+
+func (dmd *DirectoryMetadataDetails) IsSharedReadOnly() bool {
+	value, exists := dmd.Data[3].([]interface{})
+	if !exists {
+		return false
+	}
+	return len(value) > 1 && value[1] == true
+}
+
+func (dmd *DirectoryMetadataDetails) IsSharedReadWrite() bool {
+	value, exists := dmd.Data[3].([]interface{})
+	if !exists {
+		return false
+	}
+	return len(value) > 2 && value[2] == true
+}
+
+func (dmd *DirectoryMetadataDetails) SetShared(value bool, write bool) {
+	if dmd.Data == nil {
+		dmd.Data = make(map[int]interface{})
+	}
+	sharedValue, exists := dmd.Data[3].([]interface{})
+	if !exists {
+		sharedValue = make([]interface{}, 3)
+		dmd.Data[3] = sharedValue
+	}
+	if write {
+		sharedValue[2] = value
+	} else {
+		sharedValue[1] = value
+	}
+}
+func (dmd *DirectoryMetadataDetails) DecodeMsgpack(dec *msgpack.Decoder) error {
+	mapLen, err := dec.DecodeMapLen()
+
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < mapLen; i++ {
+		key, err := dec.DecodeInt8()
+		if err != nil {
+			return err
+		}
+		value, err := dec.DecodeInterface()
+		if err != nil {
+			return err
+		}
+		dmd.Data[int(key)] = value
+	}
+
+	return nil
+}
+
+func (dmd DirectoryMetadataDetails) EncodeMsgpack(enc *msgpack.Encoder) error {
+	if dmd.Data == nil {
+		dmd.Data = make(map[int]interface{})
+	}
+	return enc.Encode(dmd.Data)
+}

metadata/directory_map.go

@@ -0,0 +1,285 @@
+package metadata
+
+import (
+	"encoding/json"
+	"fmt"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"github.com/emirpasic/gods/maps/linkedhashmap"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+type directoryReferenceMap struct {
+	linkedhashmap.Map
+}
+
+func (drm directoryReferenceMap) Items() map[string]*DirectoryReference {
+	files := make(map[string]*DirectoryReference)
+	iter := drm.Iterator()
+	for iter.Next() {
+		files[iter.Key().(string)] = iter.Value().(*DirectoryReference)
+	}
+	return files
+}
+
+func (drm directoryReferenceMap) Get(key string) *DirectoryReference {
+	ret, found := drm.Map.Get(key)
+
+	if !found {
+		return nil
+	}
+
+	return ret.(*DirectoryReference)
+}
+
+func (drm directoryReferenceMap) Has(key string) bool {
+	_, found := drm.Map.Get(key)
+	return found
+}
+
+type fileReferenceMap struct {
+	linkedhashmap.Map
+}
+
+func (drm fileReferenceMap) Items() map[string]*FileReference {
+	files := make(map[string]*FileReference)
+	iter := drm.Iterator()
+	for iter.Next() {
+		files[iter.Key().(string)] = iter.Value().(*FileReference)
+	}
+	return files
+}
+
+func (drm fileReferenceMap) Get(key string) *FileReference {
+	ret, found := drm.Map.Get(key)
+
+	if !found {
+		return nil
+	}
+
+	return ret.(*FileReference)
+}
+
+func (drm fileReferenceMap) Has(key string) bool {
+	_, found := drm.Map.Get(key)
+	return found
+
+}
+
+type fileReferenceSerializationMap struct {
+	linkedhashmap.Map
+}
+type directoryReferenceSerializationMap struct {
+	linkedhashmap.Map
+}
+
+type fileVersionSerializationMap struct {
+	linkedhashmap.Map
+}
+
+type fileVersionThumbnailSerializationMap struct {
+	linkedhashmap.Map
+}
+
+type unmarshalNewInstanceFunc func() interface{}
+
+var _ SerializableMetadata = (*directoryReferenceMap)(nil)
+var _ SerializableMetadata = (*fileReferenceMap)(nil)
+var _ msgpack.CustomEncoder = (*directoryReferenceSerializationMap)(nil)
+var _ msgpack.CustomEncoder = (*fileVersionSerializationMap)(nil)
+var _ msgpack.CustomEncoder = (*fileReferenceSerializationMap)(nil)
+
+func unmarshalMapMsgpack(dec *msgpack.Decoder, m *linkedhashmap.Map, placeholder interface{}, intMap bool) error {
+	*m = *linkedhashmap.New()
+
+	l, err := dec.DecodeMapLen()
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < l; i++ {
+		var key interface{}
+		if intMap {
+			intKey, err := dec.DecodeInt()
+			if err != nil {
+				return err
+			}
+			key = intKey
+		} else {
+			strKey, err := dec.DecodeString()
+			if err != nil {
+				return err
+			}
+			key = strKey
+		}
+
+		switch placeholder.(type) {
+		case *DirectoryReference:
+			var value DirectoryReference
+			if err := dec.Decode(&value); err != nil {
+				return err
+			}
+			m.Put(key, &value)
+
+		case *FileReference:
+			var file FileReference
+			if err := dec.Decode(&file); err != nil {
+				return err
+			}
+			m.Put(key, &file)
+
+		default:
+			return fmt.Errorf("unsupported type for decoding")
+		}
+	}
+
+	return nil
+}
+
+func marshallMapMsgpack(enc *msgpack.Encoder, m *linkedhashmap.Map) error {
+	// First, encode the length of the map
+	if err := enc.EncodeMapLen(m.Size()); err != nil {
+		return err
+	}
+
+	iter := m.Iterator()
+	for iter.Next() {
+		key := iter.Key()
+
+		// Determine the type of the key and encode it
+		switch k := key.(type) {
+		case string:
+			if err := enc.EncodeString(k); err != nil {
+				return err
+			}
+		case int:
+			if err := enc.EncodeInt(int64(k)); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("unsupported key type for encoding")
+		}
+
+		value := iter.Value()
+		switch v := value.(type) {
+		case *FileReference:
+			if err := enc.Encode(&v); err != nil {
+				return err
+			}
+		case *DirectoryReference:
+			if err := enc.Encode(&v); err != nil {
+				return err
+			}
+		case string:
+			if err := enc.EncodeString(v); err != nil {
+				return err
+			}
+		case int:
+			if err := enc.EncodeInt(int64(v)); err != nil {
+				return err
+			}
+
+		case uint64:
+			if err := enc.EncodeInt(int64(v)); err != nil {
+				return err
+			}
+		case Base64UrlBinary:
+			if err := enc.Encode(&v); err != nil {
+				return err
+			}
+		case FileVersion:
+			if err := enc.Encode(&v); err != nil {
+				return err
+			}
+		case *FileVersion:
+			if err := enc.Encode(&v); err != nil {
+				return err
+			}
+		case *encoding.CID:
+			if err := enc.Encode(&v); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("unsupported type for encoding")
+		}
+	}
+
+	return nil
+}
+
+func unmarshalMapJson(bytes []byte, m *linkedhashmap.Map, newInstance unmarshalNewInstanceFunc) error {
+	*m = *linkedhashmap.New()
+	err := m.FromJSON(bytes)
+	if err != nil {
+		return err
+	}
+
+	iter := m.Iterator()
+	for iter.Next() {
+		key := iter.Key()
+		val := iter.Value()
+
+		instance := newInstance()
+
+		data, err := json.Marshal(val)
+		if err != nil {
+			return err
+		}
+
+		err = json.Unmarshal(data, instance)
+		if err != nil {
+			return err
+		}
+
+		// kind switch to handle different types
+		switch v := instance.(type) {
+		case *DirectoryReference:
+			m.Put(key, *v)
+		case *FileReference:
+			m.Put(key, *v)
+		default:
+			return fmt.Errorf("unhandled type: %T", v)
+		}
+	}
+
+	return nil
+}
+func (drm directoryReferenceMap) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return marshallMapMsgpack(enc, &drm.Map)
+}
+
+func (drm *directoryReferenceMap) DecodeMsgpack(dec *msgpack.Decoder) error {
+	return unmarshalMapMsgpack(dec, &drm.Map, &DirectoryReference{}, false)
+}
+
+func (frm fileReferenceMap) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return marshallMapMsgpack(enc, &frm.Map)
+}
+
+func (frm *fileReferenceMap) DecodeMsgpack(dec *msgpack.Decoder) error {
+	return unmarshalMapMsgpack(dec, &frm.Map, &FileReference{}, false)
+}
+
+func (frm *fileReferenceMap) UnmarshalJSON(bytes []byte) error {
+	createFileInstance := func() interface{} { return &FileReference{} }
+	return unmarshalMapJson(bytes, &frm.Map, createFileInstance)
+}
+
+func (drm *directoryReferenceMap) UnmarshalJSON(bytes []byte) error {
+	createDirInstance := func() interface{} { return &DirectoryReference{} }
+	return unmarshalMapJson(bytes, &drm.Map, createDirInstance)
+}
+
+func (frm directoryReferenceSerializationMap) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return marshallMapMsgpack(enc, &frm.Map)
+}
+
+func (frt fileReferenceSerializationMap) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return marshallMapMsgpack(enc, &frt.Map)
+}
+
+func (fvs fileVersionSerializationMap) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return marshallMapMsgpack(enc, &fvs.Map)
+}
+func (fvts fileVersionThumbnailSerializationMap) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return marshallMapMsgpack(enc, &fvts.Map)
+}

metadata/directory_reference.go

@@ -0,0 +1,90 @@
+package metadata
+
+import (
+	"github.com/emirpasic/gods/maps/linkedhashmap"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+var _ SerializableMetadata = (*DirectoryReference)(nil)
+
+type DirectoryReference struct {
+	Created           uint64                 `json:"created"`
+	Name              string                 `json:"name"`
+	EncryptedWriteKey Base64UrlBinary        `json:"encryptedWriteKey,string"`
+	PublicKey         Base64UrlBinary        `json:"publicKey,string"`
+	EncryptionKey     *Base64UrlBinary       `json:"encryptionKey,string"`
+	Ext               map[string]interface{} `json:"ext"`
+	URI               string                 `json:"uri,omitempty"`
+	Key               string                 `json:"key,omitempty"`
+	Size              int64                  `json:"size"`
+}
+
+func NewDirectoryReference(created uint64, name string, encryptedWriteKey, publicKey, encryptionKey Base64UrlBinary, ext map[string]interface{}) *DirectoryReference {
+	return &DirectoryReference{
+		Created:           created,
+		Name:              name,
+		EncryptedWriteKey: encryptedWriteKey,
+		PublicKey:         publicKey,
+		EncryptionKey:     &encryptionKey,
+		Ext:               ext,
+		URI:               "",
+		Key:               "",
+		Size:              0,
+	}
+}
+
+func (dr *DirectoryReference) EncodeMsgpack(enc *msgpack.Encoder) error {
+	dmap := &directoryReferenceSerializationMap{*linkedhashmap.New()}
+
+	dmap.Put(1, dr.Name)
+	dmap.Put(2, dr.Created)
+	dmap.Put(3, dr.PublicKey)
+	dmap.Put(4, dr.EncryptedWriteKey)
+
+	if dr.EncryptionKey != nil {
+		dmap.Put(5, dr.EncryptionKey)
+	}
+
+	if dr.Ext != nil {
+		dmap.Put(6, dr.Ext)
+	}
+
+	return enc.Encode(dmap)
+}
+
+func (dr *DirectoryReference) DecodeMsgpack(dec *msgpack.Decoder) error {
+	var (
+		err error
+		l   int
+	)
+	if l, err = dec.DecodeMapLen(); err != nil {
+		return err
+	}
+
+	for i := 0; i < l; i++ {
+		key, err := dec.DecodeInt8()
+		if err != nil {
+			return err
+		}
+		value, err := dec.DecodeInterface()
+		if err != nil {
+			return err
+		}
+		switch key {
+		case int8(1):
+			dr.Name = value.(string)
+		case int8(2):
+			dr.Created = value.(uint64)
+		case int8(3):
+			dr.PublicKey = value.([]byte)
+		case int8(4):
+			dr.EncryptedWriteKey = value.([]byte)
+		case int8(5):
+			dr.EncryptionKey = value.(*Base64UrlBinary)
+		case int8(6):
+			dr.Ext = value.(map[string]interface{})
+		}
+	}
+
+	return nil
+}

metadata/directory_test.go

@@ -0,0 +1,211 @@
+package metadata
+
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/emirpasic/gods/maps/linkedhashmap"
+	cmp "github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/assert"
+	"github.com/vmihailenco/msgpack/v5"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func isEqual(sizeFunc1, sizeFunc2 func() int, iteratorFunc1, iteratorFunc2 func() linkedhashmap.Iterator) bool {
+	if sizeFunc1() != sizeFunc2() {
+		return false
+	}
+
+	iter1 := iteratorFunc1()
+	iter2 := iteratorFunc2()
+
+	for iter1.Next() {
+		iter2.Next()
+		if iter1.Key() != iter2.Key() {
+			return false
+		}
+		if !cmp.Equal(iter1.Value(), iter2.Value()) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (frm fileReferenceMap) Equal(other fileReferenceMap) bool {
+	return isEqual(frm.Size, other.Size, frm.Iterator, other.Iterator)
+}
+
+func (frm FileHistoryMap) Equal(other FileHistoryMap) bool {
+	return isEqual(frm.Size, other.Size, frm.Iterator, other.Iterator)
+}
+
+func (drm directoryReferenceMap) Equal(other directoryReferenceMap) bool {
+	return isEqual(drm.Size, other.Size, drm.Iterator, other.Iterator)
+}
+func (ext ExtMap) Equal(other ExtMap) bool {
+	return isEqual(ext.Size, other.Size, ext.Iterator, other.Iterator)
+}
+func (fr FileReference) Equal(other FileReference) bool {
+	return fr.File.CID().Equals(other.File.CID())
+}
+
+func readFile(filename string) []byte {
+	filePath := filepath.Join("testdata", filename)
+	data, err := os.ReadFile(filePath)
+	if err != nil {
+		panic(err)
+	}
+	return data
+}
+
+func getDirectoryMeta() *DirectoryMetadata {
+	data := getDirectoryMetaContent()
+
+	var dir DirectoryMetadata
+
+	err := json.Unmarshal(data, &dir)
+	if err != nil {
+		panic(err)
+	}
+
+	return &dir
+}
+
+func getDirectoryMetaContent() []byte {
+	data := readFile("directory.json")
+
+	return data
+}
+
+func TestDirectoryMetadata_DecodeJSON(t *testing.T) {
+	tests := []struct {
+		name    string
+		wantErr bool
+	}{
+		{
+			name:    "Decode",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			jsonDm := getDirectoryMeta()
+			dm := &DirectoryMetadata{}
+
+			if err := msgpack.Unmarshal(readFile("directory.bin"), dm); (err != nil) != tt.wantErr {
+				t.Errorf("DecodeMsgpack() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			if !cmp.Equal(jsonDm, dm) {
+				t.Errorf("DecodeMsgpack() error = %v, wantErr %v", "msgpack does not match json", tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestDirectoryMetadata_DecodeMsgpack(t *testing.T) {
+	tests := []struct {
+		name    string
+		wantErr bool
+	}{
+		{
+			name:    "Decode",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			jsonDm := getDirectoryMeta()
+			dm := &DirectoryMetadata{}
+
+			if err := msgpack.Unmarshal(readFile("directory.bin"), dm); (err != nil) != tt.wantErr {
+				t.Errorf("DecodeMsgpack() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			if !cmp.Equal(jsonDm, dm) {
+				t.Errorf("DecodeMsgpack() error = %v, wantErr %v", "msgpack does not match json", tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestDirectoryMetadata_EncodeMsgpack(t *testing.T) {
+	tests := []struct {
+		name    string
+		wantErr bool
+	}{
+		{
+			name:    "Encode",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dm := &DirectoryMetadata{}
+
+			good := readFile("directory.bin")
+
+			if err := msgpack.Unmarshal(good, dm); (err != nil) != tt.wantErr {
+				t.Errorf("DecodeMsgpack() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			out, err := msgpack.Marshal(dm)
+
+			if (err != nil) != tt.wantErr {
+				t.Errorf("EncodeMsgpack() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			if !cmp.Equal(good, out) {
+				t.Errorf("EncodeMsgpack() error = %v, wantErr %v", "msgpack does not match sample", tt.wantErr)
+			}
+
+			dm2 := &DirectoryMetadata{}
+
+			if err := msgpack.Unmarshal(out, dm2); (err != nil) != tt.wantErr {
+				t.Errorf("DecodeMsgpack() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			if !cmp.Equal(dm, dm2) {
+				t.Errorf("EncodeMsgpack() error = %v, wantErr %v", "msgpack deser verification does not match", tt.wantErr)
+			}
+		})
+	}
+}
+func TestDirectoryMetadata_EncodeJSON(t *testing.T) {
+	tests := []struct {
+		name    string
+		wantErr bool
+	}{
+		{
+			name:    "Encode",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			jsonDm := getDirectoryMetaContent()
+			dm := &DirectoryMetadata{}
+
+			if err := json.Unmarshal(jsonDm, dm); (err != nil) != tt.wantErr {
+				t.Errorf("EncodeJSON() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			jsonData, err := json.MarshalIndent(dm, "", "\t")
+
+			if (err != nil) != tt.wantErr {
+				t.Errorf("EncodeJSON() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			buf := bytes.NewBuffer(nil)
+
+			err = json.Indent(buf, jsonData, "", "\t")
+			if err != nil {
+				t.Errorf("EncodeJSON() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			assert.Equal(t, buf.Bytes(), jsonData)
+		})
+	}
+}

metadata/extra.go

@@ -0,0 +1,189 @@
+package metadata
+
+import (
+	"encoding/json"
+	"fmt"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+type jsonData = map[string]interface{}
+
+var names = map[types.MetadataExtension]string{
+	types.MetadataExtensionLicenses:           "licenses",
+	types.MetadataExtensionDonationKeys:       "donationKeys",
+	types.MetadataExtensionWikidataClaims:     "wikidataClaims",
+	types.MetadataExtensionLanguages:          "languages",
+	types.MetadataExtensionSourceUris:         "sourceUris",
+	types.MetadataExtensionPreviousVersions:   "previousVersions",
+	types.MetadataExtensionTimestamp:          "timestamp",
+	types.MetadataExtensionOriginalTimestamp:  "originalTimestamp",
+	types.MetadataExtensionTags:               "tags",
+	types.MetadataExtensionCategories:         "categories",
+	types.MetadataExtensionBasicMediaMetadata: "basicMediaMetadata",
+	types.MetadataExtensionViewTypes:          "viewTypes",
+	types.MetadataExtensionBridge:             "bridge",
+	types.MetadataExtensionRoutingHints:       "routingHints",
+}
+
+var namesReverse = map[string]types.MetadataExtension{
+	"licenses":           types.MetadataExtensionLicenses,
+	"donationKeys":       types.MetadataExtensionDonationKeys,
+	"wikidataClaims":     types.MetadataExtensionWikidataClaims,
+	"languages":          types.MetadataExtensionLanguages,
+	"sourceUris":         types.MetadataExtensionSourceUris,
+	"previousVersions":   types.MetadataExtensionPreviousVersions,
+	"timestamp":          types.MetadataExtensionTimestamp,
+	"originalTimestamp":  types.MetadataExtensionOriginalTimestamp,
+	"tags":               types.MetadataExtensionTags,
+	"categories":         types.MetadataExtensionCategories,
+	"basicMediaMetadata": types.MetadataExtensionBasicMediaMetadata,
+	"viewTypes":          types.MetadataExtensionViewTypes,
+	"bridge":             types.MetadataExtensionBridge,
+	"routingHints":       types.MetadataExtensionRoutingHints,
+}
+
+type ExtraMetadata struct {
+	Data map[int]interface{}
+}
+type keyValue struct {
+	Key   interface{}
+	Value interface{}
+}
+
+var _ SerializableMetadata = (*ExtraMetadata)(nil)
+
+func NewExtraMetadata(data map[int]interface{}) *ExtraMetadata {
+	return &ExtraMetadata{
+		Data: data,
+	}
+}
+
+func (em ExtraMetadata) MarshalJSON() ([]byte, error) {
+	data, err := em.encode()
+	if err != nil {
+		return nil, err
+	}
+
+	return json.Marshal(data)
+}
+
+func (em *ExtraMetadata) UnmarshalJSON(data []byte) error {
+
+	em.Data = make(map[int]interface{})
+	jsonObject := make(map[int]interface{})
+	if err := json.Unmarshal(data, &jsonObject); err != nil {
+		return err
+	}
+
+	for name, value := range jsonObject {
+		err := em.decodeItem(keyValue{Key: name, Value: value})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (em *ExtraMetadata) decodeItem(pair keyValue) error {
+	var metadataKey int
+
+	// Determine the type of the key and convert it if necessary
+	switch k := pair.Key.(type) {
+	case string:
+		if val, ok := namesReverse[k]; ok {
+			metadataKey = int(val)
+		} else {
+			return fmt.Errorf("unknown key in JSON: %s", k)
+		}
+	case int8:
+		metadataKey = int(k)
+	default:
+		return fmt.Errorf("unsupported key type")
+	}
+
+	if metadataKey == int(types.MetadataExtensionUpdateCID) {
+		cid, err := encoding.CIDFromBytes([]byte(pair.Value.(string)))
+		if err != nil {
+			return err
+		}
+		em.Data[metadataKey] = cid
+	} else {
+		em.Data[metadataKey] = pair.Value
+	}
+
+	return nil
+}
+
+func (em *ExtraMetadata) DecodeMsgpack(dec *msgpack.Decoder) error {
+	mapLen, err := dec.DecodeMapLen()
+
+	if err != nil {
+		return err
+	}
+
+	em.Data = make(map[int]interface{}, mapLen)
+
+	for i := 0; i < mapLen; i++ {
+		key, err := dec.DecodeInt8()
+		if err != nil {
+			return err
+		}
+
+		var value interface{}
+		if key == int8(types.MetadataExtensionUpdateCID) {
+			value, err = dec.DecodeString()
+		} else {
+			value, err = dec.DecodeInterface()
+		}
+		if err != nil {
+			return err
+		}
+
+		err = em.decodeItem(keyValue{Key: key, Value: value})
+		if err != nil {
+			return err
+		}
+	}
+
+	if mapLen == 0 {
+		em.Data = make(map[int]interface{})
+	}
+
+	return nil
+}
+
+func (em ExtraMetadata) EncodeMsgpack(enc *msgpack.Encoder) error {
+	data, err := em.encode()
+	if err != nil {
+		return err
+	}
+
+	return enc.Encode(data)
+}
+
+func (em ExtraMetadata) encode() (jsonData, error) {
+	jsonObject := make(map[string]interface{})
+	for key, value := range em.Data {
+		name, ok := names[types.MetadataExtension(key)]
+		if ok {
+			if types.MetadataExtension(key) == types.MetadataExtensionUpdateCID {
+				cid, err := encoding.CIDFromBytes(value.([]byte))
+				var cidString string
+				if err == nil {
+					cidString, err = cid.ToString()
+				}
+				if err == nil {
+					jsonObject["updateCID"] = cidString
+				} else {
+					jsonObject["updateCID"] = ""
+				}
+			} else {
+				jsonObject[name] = value
+			}
+		}
+	}
+
+	return jsonObject, nil
+}

metadata/file_reference.go

@@ -0,0 +1,178 @@
+package metadata
+
+import (
+	"github.com/emirpasic/gods/maps/linkedhashmap"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+var _ SerializableMetadata = (*FileReference)(nil)
+var _ SerializableMetadata = (*FileHistoryMap)(nil)
+var _ SerializableMetadata = (*ExtMap)(nil)
+
+type FileHistoryMap struct {
+	linkedhashmap.Map
+}
+type ExtMap struct {
+	linkedhashmap.Map
+}
+
+func NewExtMap() ExtMap {
+	return ExtMap{*linkedhashmap.New()}
+}
+
+func NewFileHistoryMap() FileHistoryMap {
+	return FileHistoryMap{*linkedhashmap.New()}
+}
+
+type FileReference struct {
+	Name     string         `json:"name"`
+	Created  uint64         `json:"created"`
+	Version  uint64         `json:"version"`
+	File     *FileVersion   `json:"file"`
+	Ext      ExtMap         `json:"ext"`
+	History  FileHistoryMap `json:"history"`
+	MimeType string         `json:"mimeType"`
+	URI      string         `json:"uri,omitempty"`
+	Key      string         `json:"key,omitempty"`
+}
+
+func NewFileReference(name string, created, version uint64, file *FileVersion, ext ExtMap, history FileHistoryMap, mimeType string) *FileReference {
+	return &FileReference{
+		Name:     name,
+		Created:  created,
+		Version:  version,
+		File:     file,
+		Ext:      ext,
+		History:  history,
+		MimeType: mimeType,
+		URI:      "",
+		Key:      "",
+	}
+}
+
+func (fr *FileReference) Modified() uint64 {
+	return fr.File.Ts
+}
+
+func (fr *FileReference) EncodeMsgpack(enc *msgpack.Encoder) error {
+	tempMap := &fileReferenceSerializationMap{*linkedhashmap.New()}
+
+	tempMap.Put(1, fr.Name)
+	tempMap.Put(2, fr.Created)
+	tempMap.Put(4, fr.File)
+	tempMap.Put(5, fr.Version)
+
+	if fr.MimeType != "" {
+		tempMap.Put(6, fr.MimeType)
+	}
+
+	if !fr.Ext.Empty() {
+		tempMap.Put(7, fr.Ext)
+	}
+
+	if !fr.History.Empty() {
+		tempMap.Put(8, fr.History)
+	}
+
+	return enc.Encode(tempMap)
+}
+func (fr *FileReference) DecodeMsgpack(dec *msgpack.Decoder) error {
+	mapLen, err := dec.DecodeMapLen()
+
+	if err != nil {
+		return err
+	}
+
+	hasExt := false
+	hasHistory := false
+
+	for i := 0; i < mapLen; i++ {
+		key, err := dec.DecodeInt8()
+		if err != nil {
+			return err
+		}
+
+		switch key {
+		case int8(1):
+			err := dec.Decode(&fr.Name)
+			if err != nil {
+				return err
+			}
+		case int8(2):
+			err := dec.Decode(&fr.Created)
+			if err != nil {
+				return err
+			}
+		case int8(4):
+			err := dec.Decode(&fr.File)
+			if err != nil {
+				return err
+			}
+		case int8(5):
+			val, err := dec.DecodeInt()
+			if err != nil {
+				return err
+			}
+
+			fr.Version = uint64(val)
+		case int8(6):
+			err := dec.Decode(&fr.MimeType)
+			if err != nil {
+				return err
+			}
+		case int8(7):
+			err := dec.Decode(&fr.Ext)
+			if err != nil {
+				return err
+			}
+
+			hasExt = true
+		case int8(8):
+			err := dec.Decode(&fr.History)
+			if err != nil {
+				return err
+			}
+
+			hasHistory = true
+		}
+	}
+
+	if !hasExt {
+		fr.Ext = ExtMap{*linkedhashmap.New()}
+	}
+
+	if !hasHistory {
+		fr.History = FileHistoryMap{*linkedhashmap.New()}
+	}
+
+	return nil
+}
+
+func (ext ExtMap) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return marshallMapMsgpack(enc, &ext.Map)
+}
+func (ext *ExtMap) DecodeMsgpack(dec *msgpack.Decoder) error {
+	return unmarshalMapMsgpack(dec, &ext.Map, &ExtMap{}, true)
+}
+func (fhm FileHistoryMap) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return marshallMapMsgpack(enc, &fhm.Map)
+}
+func (fhm *FileHistoryMap) DecodeMsgpack(dec *msgpack.Decoder) error {
+	return unmarshalMapMsgpack(dec, &fhm.Map, &ExtMap{}, false)
+}
+
+func (m *FileHistoryMap) UnmarshalJSON(bytes []byte) error {
+	if string(bytes) == "null" {
+		m.Map = *linkedhashmap.New()
+		return nil
+	}
+	return m.FromJSON(bytes)
+}
+
+func (m *ExtMap) UnmarshalJSON(bytes []byte) error {
+	if string(bytes) == "null" {
+		m.Map = *linkedhashmap.New()
+		return nil
+	}
+	return m.FromJSON(bytes)
+}

metadata/file_version.go

@@ -0,0 +1,116 @@
+package metadata
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"github.com/emirpasic/gods/maps/linkedhashmap"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+type FileVersion struct {
+	Ts           uint64                 `json:"ts"`
+	EncryptedCID *encoding.EncryptedCID `json:"encryptedCID,string"`
+	PlaintextCID *encoding.CID          `json:"cid,string"`
+	Thumbnail    *FileVersionThumbnail  `json:"thumbnail"`
+	Hashes       []*encoding.Multihash  `json:"hashes"`
+	Ext          map[string]interface{} `json:"ext"`
+}
+
+func NewFileVersion(ts uint64, encryptedCID *encoding.EncryptedCID, plaintextCID *encoding.CID, thumbnail *FileVersionThumbnail, hashes []*encoding.Multihash, ext map[string]interface{}) *FileVersion {
+	return &FileVersion{
+		Ts:           ts,
+		EncryptedCID: encryptedCID,
+		PlaintextCID: plaintextCID,
+		Thumbnail:    thumbnail,
+		Hashes:       hashes,
+		Ext:          ext,
+	}
+}
+
+func (fv *FileVersion) EncodeMsgpack(enc *msgpack.Encoder) error {
+
+	fmap := &fileVersionSerializationMap{*linkedhashmap.New()}
+
+	fmap.Put(8, fv.Ts)
+
+	if fv.EncryptedCID != nil {
+		fmap.Put(1, fv.EncryptedCID)
+	}
+
+	if fv.PlaintextCID != nil {
+		fmap.Put(2, fv.PlaintextCID)
+	}
+
+	if len(fv.Hashes) > 0 {
+		hashesData := make([][]byte, len(fv.Hashes))
+		for i, hash := range fv.Hashes {
+			hashesData[i] = hash.FullBytes()
+		}
+		fmap.Put(9, hashesData)
+	}
+
+	if fv.Thumbnail != nil {
+		fmap.Put(10, fv.Thumbnail)
+	}
+
+	return enc.Encode(fmap)
+}
+
+func (fv *FileVersion) DecodeMsgpack(dec *msgpack.Decoder) error {
+	mapLen, err := dec.DecodeMapLen()
+
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < mapLen; i++ {
+		key, err := dec.DecodeInt8()
+		if err != nil {
+			return err
+		}
+		switch key {
+
+		case int8(1):
+			err := dec.Decode(&fv.EncryptedCID)
+			if err != nil {
+				return err
+			}
+
+		case int8(2):
+			err := dec.Decode(&fv.PlaintextCID)
+			if err != nil {
+				return err
+			}
+		case int8(8):
+			err := dec.Decode(&fv.Ts)
+			if err != nil {
+				return err
+			}
+		case int8(9):
+			hashesData, err := dec.DecodeSlice()
+			if err != nil {
+				return err
+			}
+
+			fv.Hashes = make([]*encoding.Multihash, len(hashesData))
+			for i, hashData := range hashesData {
+				hashBytes := hashData.([]byte)
+				fv.Hashes[i] = encoding.NewMultihash(hashBytes)
+			}
+
+		case int8(10):
+			err := dec.Decode(&fv.Thumbnail)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (fv *FileVersion) CID() *encoding.CID {
+	if fv.PlaintextCID != nil {
+		return fv.PlaintextCID
+	}
+	return &fv.EncryptedCID.OriginalCID
+}

metadata/file_version_thumbnail.go

@@ -0,0 +1,98 @@
+package metadata
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"github.com/emirpasic/gods/maps/linkedhashmap"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+type FileVersionThumbnail struct {
+	ImageType   string
+	AspectRatio float64
+	CID         *encoding.EncryptedCID
+	Thumbhash   []byte
+}
+
+func NewFileVersionThumbnail(imageType string, aspectRatio float64, cid *encoding.EncryptedCID, thumbhash []byte) *FileVersionThumbnail {
+	return &FileVersionThumbnail{
+		ImageType:   imageType,
+		AspectRatio: aspectRatio,
+		CID:         cid,
+		Thumbhash:   thumbhash,
+	}
+}
+
+func (fvt *FileVersionThumbnail) EncodeMsgpack(enc *msgpack.Encoder) error {
+	fmap := &fileVersionThumbnailSerializationMap{*linkedhashmap.New()}
+
+	fmap.Put(2, fvt.AspectRatio)
+	fmap.Put(3, fvt.CID.ToBytes())
+
+	if fvt.ImageType != "" {
+		fmap.Put(1, fvt.ImageType)
+	}
+
+	if fvt.Thumbhash != nil {
+		fmap.Put(4, fvt.Thumbhash)
+	}
+
+	return enc.Encode(fmap)
+}
+func (fvt *FileVersionThumbnail) DecodeMsgpack(dec *msgpack.Decoder) error {
+	mapLen, err := dec.DecodeMapLen()
+
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < mapLen; i++ {
+		key, err := dec.DecodeInt8()
+		if err != nil {
+			return err
+		}
+		switch key {
+		case int8(1):
+			err := dec.Decode(&fvt.ImageType)
+			if err != nil {
+				return err
+			}
+		case int8(2):
+			err := dec.Decode(&fvt.AspectRatio)
+			if err != nil {
+				return err
+			}
+		case int8(3):
+			val, err := dec.DecodeBytes()
+			if err != nil {
+				return err
+			}
+			fvt.CID, err = encoding.EncryptedCIDFromBytes(val)
+			if err != nil {
+				return err
+			}
+
+		case int8(4):
+			err := dec.Decode(&fvt.Thumbhash)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+func (fvt *FileVersionThumbnail) Encode() map[int]interface{} {
+	data := map[int]interface{}{
+		2: fvt.AspectRatio,
+		3: fvt.CID.ToBytes(),
+	}
+
+	if fvt.ImageType != "" {
+		data[1] = fvt.ImageType
+	}
+
+	if fvt.Thumbhash != nil {
+		data[4] = fvt.Thumbhash
+	}
+
+	return data
+}

metadata/media_format.go

@@ -0,0 +1,159 @@
+package metadata
+
+import (
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+var (
+	_ msgpack.CustomDecoder = (*MediaFormat)(nil)
+	_ msgpack.CustomEncoder = (*MediaFormat)(nil)
+)
+
+type MediaFormat struct {
+	Subtype       string
+	Role          string
+	Ext           string
+	Cid           *encoding.CID
+	Height        int
+	Width         int
+	Languages     []string
+	Asr           int
+	Fps           int
+	Bitrate       int
+	AudioChannels int
+	Vcodec        string
+	Acodec        string
+	Container     string
+	DynamicRange  string
+	Charset       string
+	Value         []byte
+	Duration      int
+	Rows          int
+	Columns       int
+	Index         int
+	InitRange     string
+	IndexRange    string
+	Caption       string
+}
+
+func (mmd *MediaFormat) EncodeMsgpack(encoder *msgpack.Encoder) error {
+	return errors.New("Not implemented")
+}
+
+func NewMediaFormat(subtype string, role, ext, vcodec, acodec, container, dynamicRange, charset, initRange, indexRange, caption string, cid *encoding.CID, height, width, asr, fps, bitrate, audioChannels, duration, rows, columns, index int, languages []string, value []byte) *MediaFormat {
+	return &MediaFormat{
+		Subtype:       subtype,
+		Role:          role,
+		Ext:           ext,
+		Cid:           cid,
+		Height:        height,
+		Width:         width,
+		Languages:     languages,
+		Asr:           asr,
+		Fps:           fps,
+		Bitrate:       bitrate,
+		AudioChannels: audioChannels,
+		Vcodec:        vcodec,
+		Acodec:        acodec,
+		Container:     container,
+		DynamicRange:  dynamicRange,
+		Charset:       charset,
+		Value:         value,
+		Duration:      duration,
+		Rows:          rows,
+		Columns:       columns,
+		Index:         index,
+		InitRange:     initRange,
+		IndexRange:    indexRange,
+		Caption:       caption,
+	}
+}
+func (mmd *MediaFormat) DecodeMsgpack(dec *msgpack.Decoder) error {
+	intMap, err := decodeIntMap(dec)
+	if err != nil {
+		return err
+	}
+
+	for key, value := range intMap {
+		switch key {
+		case 1:
+			mmd.Cid, err = encoding.CIDFromBytes(value.([]byte))
+			if err != nil {
+				return err
+			}
+
+		case 2:
+			mmd.Subtype = value.(string)
+		case 3:
+			mmd.Role = value.(string)
+		case 4:
+			mmd.Ext = value.(string)
+		case 10:
+			mmd.Height = intParse(value)
+		case 11:
+			mmd.Width = intParse(value)
+		case 12:
+			vals := value.([]interface{})
+
+			for _, val := range vals {
+				mmd.Languages = append(mmd.Languages, val.(string))
+			}
+		case 13:
+			mmd.Asr = intParse(value)
+		case 14:
+			mmd.Fps = intParse(value)
+		case 15:
+			mmd.Bitrate = intParse(value)
+		case 18:
+			mmd.AudioChannels = intParse(value)
+		case 19:
+			mmd.Vcodec = value.(string)
+		case 20:
+			mmd.Acodec = value.(string)
+		case 21:
+			mmd.Container = value.(string)
+		case 22:
+			mmd.DynamicRange = value.(string)
+		case 23:
+			mmd.Charset = value.(string)
+		case 24:
+			mmd.Value = value.([]byte)
+		case 25:
+			mmd.Duration = intParse(value)
+		case 26:
+			mmd.Rows = intParse(value)
+		case 27:
+			mmd.Columns = intParse(value)
+		case 28:
+			mmd.Index = intParse(value)
+		case 29:
+			mmd.InitRange = value.(string)
+		case 30:
+			mmd.IndexRange = value.(string)
+		case 31:
+			mmd.Caption = value.(string)
+		}
+	}
+
+	return nil
+}
+
+func intParse(value interface{}) int {
+	switch value.(type) {
+	case int:
+		return value.(int)
+	case uint:
+		return int(value.(uint))
+	case uint16:
+		return int(value.(uint16))
+	case uint32:
+		return int(value.(uint32))
+	case int16:
+		return int(value.(int16))
+	case int8:
+		return int(value.(int8))
+	}
+	return 0
+}

metadata/media_meta_links.go

@@ -0,0 +1,73 @@
+package metadata
+
+import (
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+var (
+	_ msgpack.CustomDecoder = (*MediaMetadataLinks)(nil)
+	_ msgpack.CustomEncoder = (*MediaMetadataLinks)(nil)
+)
+
+type MediaMetadataLinks struct {
+	Count     int
+	Head      []*encoding.CID
+	Collapsed []*encoding.CID
+	Tail      []*encoding.CID
+}
+
+func (m MediaMetadataLinks) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return errors.New("Not implemented")
+}
+
+func (m MediaMetadataLinks) DecodeMsgpack(dec *msgpack.Decoder) error {
+	data, err := decodeIntMap(dec)
+	if err != nil {
+		return err
+	}
+
+	for key, value := range data {
+		switch key {
+		case 1:
+			m.Count = value.(int)
+		case 2:
+			head := value.([]interface{})
+			for _, h := range head {
+				cid, err := encoding.CIDFromBytes(h.([]byte))
+				if err != nil {
+					return err
+				}
+				m.Head = append(m.Head, cid)
+			}
+		case 3:
+			collapsed := value.([]interface{})
+			for _, c := range collapsed {
+				cid, err := encoding.CIDFromBytes(c.([]byte))
+				if err != nil {
+					return err
+				}
+				m.Collapsed = append(m.Collapsed, cid)
+			}
+		case 4:
+			tail := value.([]interface{})
+			for _, t := range tail {
+				cid, err := encoding.CIDFromBytes(t.([]byte))
+				if err != nil {
+					return err
+				}
+				m.Tail = append(m.Tail, cid)
+			}
+		}
+	}
+
+	return nil
+}
+
+func NewMediaMetadataLinks(head []*encoding.CID) *MediaMetadataLinks {
+	return &MediaMetadataLinks{
+		Count: len(head),
+		Head:  head,
+	}
+}

metadata/media_metadata.go

@@ -0,0 +1,243 @@
+package metadata
+
+import (
+	"bytes"
+	"crypto/ed25519"
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/serialize"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"git.lumeweb.com/LumeWeb/libs5-go/utils"
+	"github.com/vmihailenco/msgpack/v5"
+	"io"
+	"lukechampine.com/blake3"
+	_ "lukechampine.com/blake3"
+)
+
+var (
+	_ Metadata              = (*MediaMetadata)(nil)
+	_ msgpack.CustomDecoder = (*MediaMetadata)(nil)
+	_ msgpack.CustomEncoder = (*MediaMetadata)(nil)
+	_ msgpack.CustomDecoder = (*mediaMap)(nil)
+)
+
+type mediaMap map[string][]MediaFormat
+
+type MediaMetadata struct {
+	Name          string
+	MediaTypes    mediaMap
+	Parents       []MetadataParentLink
+	Details       MediaMetadataDetails
+	Links         *MediaMetadataLinks
+	ExtraMetadata ExtraMetadata
+	provenPubKeys []*encoding.Multihash
+	BaseMetadata
+}
+
+func (m *MediaMetadata) ProvenPubKeys() []*encoding.Multihash {
+	return m.provenPubKeys
+}
+
+func NewMediaMetadata(name string, details MediaMetadataDetails, parents []MetadataParentLink, mediaTypes map[string][]MediaFormat, links *MediaMetadataLinks, extraMetadata ExtraMetadata) *MediaMetadata {
+	return &MediaMetadata{
+		Name:          name,
+		Details:       details,
+		Parents:       parents,
+		MediaTypes:    mediaTypes,
+		Links:         links,
+		ExtraMetadata: extraMetadata,
+	}
+}
+func NewEmptyMediaMetadata() *MediaMetadata {
+	return &MediaMetadata{}
+}
+
+func (m *MediaMetadata) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return errors.New("Not implemented")
+}
+
+func (m *MediaMetadata) DecodeMsgpack(dec *msgpack.Decoder) error {
+	kind, err := serialize.InitUnmarshaller(dec, types.MetadataTypeProof, types.MetadataTypeMedia)
+	if err != nil {
+		return err
+	}
+
+	switch kind {
+	case types.MetadataTypeProof:
+		return m.decodeProof(dec)
+	case types.MetadataTypeMedia:
+		return m.decodeMedia(dec)
+	default:
+		return errors.New("Invalid metadata type")
+	}
+}
+
+func (m *MediaMetadata) decodeProof(dec *msgpack.Decoder) error {
+	all, err := io.ReadAll(dec.Buffered())
+	if err != nil {
+		return err
+	}
+
+	proofSectionLength := utils.DecodeEndian(all[0:2])
+
+	all = all[2:]
+
+	bodyBytes := all[proofSectionLength:]
+
+	if proofSectionLength == 0 {
+		return nil
+	}
+
+	childDec := msgpack.NewDecoder(bytes.NewReader(all[:proofSectionLength]))
+
+	hash := blake3.Sum256(bodyBytes)
+	b3hash := append([]byte{byte(types.HashTypeBlake3)}, hash[:]...)
+
+	arrayLen, err := childDec.DecodeArrayLen()
+	if err != nil {
+		return err
+	}
+
+	provenPubKeys := make([]*encoding.Multihash, 0)
+
+	for i := 0; i < arrayLen; i++ {
+		proofData, err := childDec.DecodeSlice()
+		if err != nil {
+			return err
+		}
+
+		var hashType int8
+		var pubkey []byte
+		var signature []byte
+
+		if len(proofData) != 4 {
+			return errors.New("Invalid proof data length")
+		}
+
+		for j := 0; j < len(proofData); j++ {
+
+			switch j {
+			case 0:
+				sigType := proofData[j].(int8)
+				if types.MetadataProofType(sigType) != types.MetadataProofTypeSignature {
+					return errors.New("Invalid proof type")
+				}
+			case 1:
+				hashType = proofData[j].(int8)
+				if types.HashType(hashType) != types.HashTypeBlake3 {
+					return errors.New("Invalid hash type")
+				}
+
+			case 2:
+				pubkey = proofData[j].([]byte)
+				if types.HashType(pubkey[0]) != types.HashTypeEd25519 {
+					return errors.New("Invalid public key type")
+				}
+
+				if len(pubkey) != 33 {
+					return errors.New("Invalid public key length")
+				}
+
+			case 3:
+				signature = proofData[j].([]byte)
+
+				if valid := ed25519.Verify(pubkey[1:], b3hash[:], signature); !valid {
+					return errors.New("Invalid signature")
+				}
+
+				provenPubKeys = append(provenPubKeys, encoding.NewMultihash(pubkey))
+			}
+		}
+	}
+
+	m.provenPubKeys = provenPubKeys
+
+	mediaDec := msgpack.NewDecoder(bytes.NewReader(bodyBytes))
+
+	mediaByte, err := mediaDec.DecodeUint8()
+	if err != nil {
+		return err
+	}
+
+	if types.MetadataType(mediaByte) != types.MetadataTypeMedia {
+		return errors.New("Invalid metadata type")
+	}
+
+	return m.decodeMedia(mediaDec)
+}
+
+func (m *MediaMetadata) decodeMedia(dec *msgpack.Decoder) error {
+	_, err := dec.DecodeArrayLen()
+	if err != nil {
+		return err
+	}
+
+	err = dec.Decode(&m.Name)
+	if err != nil {
+		return err
+	}
+
+	err = dec.Decode(&m.Details)
+	if err != nil {
+		return err
+	}
+
+	arrLen, err := dec.DecodeArrayLen()
+	if err != nil {
+		return err
+	}
+
+	parents := make([]MetadataParentLink, arrLen)
+	for i := 0; i < arrLen; i++ {
+		parents[i].SetParent(m)
+		err = dec.Decode(&parents[i])
+		if err != nil {
+			return err
+		}
+	}
+
+	err = dec.Decode(&m.MediaTypes)
+	if err != nil {
+		return err
+	}
+
+	err = dec.Decode(&m.Links)
+	if err != nil {
+		return err
+	}
+
+	err = dec.Decode(&m.ExtraMetadata)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *mediaMap) DecodeMsgpack(dec *msgpack.Decoder) error {
+	mapLen, err := dec.DecodeMapLen()
+	if err != nil {
+		return err
+	}
+
+	*m = make(map[string][]MediaFormat, mapLen)
+
+	for i := 0; i < mapLen; i++ {
+		typ, err := dec.DecodeString()
+		if err != nil {
+			return err
+		}
+
+		var formats []MediaFormat
+
+		err = dec.Decode(&formats)
+
+		if err != nil {
+			return err
+		}
+
+		(*m)[typ] = formats
+	}
+
+	return nil
+}

metadata/media_metadata_details.go

@@ -0,0 +1,34 @@
+package metadata
+
+import (
+	"errors"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+var (
+	_ msgpack.CustomDecoder = (*MediaMetadataDetails)(nil)
+	_ msgpack.CustomEncoder = (*MediaMetadataDetails)(nil)
+)
+
+type MediaMetadataDetails struct {
+	Data map[int]interface{}
+}
+
+func NewMediaMetadataDetails(data map[int]interface{}) *MediaMetadataDetails {
+	return &MediaMetadataDetails{Data: data}
+}
+
+func (mmd *MediaMetadataDetails) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return errors.New("Not implemented")
+}
+
+func (mmd *MediaMetadataDetails) DecodeMsgpack(dec *msgpack.Decoder) error {
+	intMap, err := decodeIntMap(dec)
+	if err != nil {
+		return err
+	}
+
+	mmd.Data = intMap
+
+	return nil
+}

metadata/meta.go

@@ -0,0 +1,14 @@
+package metadata
+
+import "github.com/vmihailenco/msgpack/v5"
+
+type Metadata interface {
+}
+type SerializableMetadata interface {
+	msgpack.CustomEncoder
+	msgpack.CustomDecoder
+}
+
+type BaseMetadata struct {
+	Type string `json:"type"`
+}

metadata/misc.go

@@ -0,0 +1,55 @@
+package metadata
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+type Base64UrlBinary []byte
+
+func (b *Base64UrlBinary) UnmarshalJSON(data []byte) error {
+	strData := string(data)
+	if len(strData) >= 2 && strData[0] == '"' && strData[len(strData)-1] == '"' {
+		strData = strData[1 : len(strData)-1]
+	}
+
+	if strData == "null" {
+		return nil
+	}
+
+	decodedData, err := base64.RawURLEncoding.DecodeString(strData)
+	if err != nil {
+		return err
+	}
+
+	*b = Base64UrlBinary(decodedData)
+	return nil
+}
+func (b Base64UrlBinary) MarshalJSON() ([]byte, error) {
+	return json.Marshal([]byte(base64.RawURLEncoding.EncodeToString(b)))
+}
+
+func decodeIntMap(dec *msgpack.Decoder) (map[int]interface{}, error) {
+	mapLen, err := dec.DecodeMapLen()
+
+	if err != nil {
+		return nil, err
+	}
+
+	data := make(map[int]interface{}, mapLen)
+
+	for i := 0; i < mapLen; i++ {
+		key, err := dec.DecodeInt()
+		if err != nil {
+			return nil, err
+		}
+		value, err := dec.DecodeInterface()
+		if err != nil {
+			return nil, err
+		}
+		data[key] = value
+	}
+
+	return data, nil
+}

metadata/parent_link.go

@@ -0,0 +1,90 @@
+package metadata
+
+import (
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+var (
+	_ msgpack.CustomDecoder = (*MetadataParentLink)(nil)
+	_ msgpack.CustomEncoder = (*MetadataParentLink)(nil)
+)
+
+// MetadataParentLink represents the structure for Metadata Parent Link.
+type MetadataParentLink struct {
+	CID    *encoding.CID
+	Type   types.ParentLinkType
+	Role   string
+	Signed bool
+	parent *MediaMetadata
+}
+
+func (m *MetadataParentLink) SetParent(parent *MediaMetadata) {
+	m.parent = parent
+}
+
+func (m *MetadataParentLink) EncodeMsgpack(enc *msgpack.Encoder) error {
+	return errors.New("Not implemented")
+}
+
+func (m *MetadataParentLink) DecodeMsgpack(dec *msgpack.Decoder) error {
+	mapLen, err := dec.DecodeMapLen()
+
+	if err != nil {
+		return err
+	}
+
+	cid := &encoding.CID{}
+
+	for i := 0; i < mapLen; i++ {
+		key, err := dec.DecodeInt8()
+		if err != nil {
+			return err
+		}
+		value, err := dec.DecodeInterface()
+		if err != nil {
+			return err
+		}
+
+		switch key {
+		case 0:
+			m.Type = types.ParentLinkType(value.(int))
+		case 1:
+			cid, err = encoding.CIDFromBytes(value.([]byte))
+			if err != nil {
+				return err
+			}
+
+			m.CID = cid
+		}
+	}
+
+	if m.Type == 0 {
+		m.Type = types.ParentLinkTypeUserIdentity
+	}
+
+	m.Signed = false
+
+	if m.parent != nil {
+		for _, key := range m.parent.ProvenPubKeys() {
+			if cid.Hash.Equals(key) {
+				m.Signed = true
+				break
+			}
+		}
+	}
+
+	return nil
+}
+
+// NewMetadataParentLink creates a new MetadataParentLink with the provided values.
+func NewMetadataParentLink(cid *encoding.CID, role string, signed bool) *MetadataParentLink {
+	return &MetadataParentLink{
+		CID:    cid,
+		Type:   types.ParentLinkTypeUserIdentity,
+		Role:   role,
+		Signed: signed,
+	}
+}

metadata/testdata/directory.json

@@ -0,0 +1,193 @@
+{
+    "type": "directory",
+    "details": {},
+    "directories": {
+        "arch": {
+            "name": "arch",
+            "created": 1704127624979,
+            "publicKey": "7Vd1kAf0LoYmKKQ9-C8Znl0npyjK2M4-ciiNWShOTpbD",
+            "encryptedWriteKey": "",
+            "encryptionKey": null,
+            "ext": null
+        }
+    },
+    "files": {
+        "archlinux-bootstrap-x86_64.tar.gz": {
+            "name": "archlinux-bootstrap-x86_64.tar.gz",
+            "created": 1704127689514,
+            "modified": 1704127689514,
+            "version": 0,
+            "mimeType": null,
+            "file": {
+                "ts": 1704127689514,
+                "encryptedCID": null,
+                "cid": "z6e5xDpewiueHz6n9HJz21uGmTDvvctawujzYSeu9773k8s9RkG8a",
+                "hashes": null,
+                "thumbnail": null
+            },
+            "ext": null,
+            "history": null
+        },
+        "archlinux-bootstrap-x86_64.tar.gz.sig": {
+            "name": "archlinux-bootstrap-x86_64.tar.gz.sig",
+            "created": 1704127698951,
+            "modified": 1704127698951,
+            "version": 0,
+            "mimeType": "application/pgp-signature",
+            "file": {
+                "ts": 1704127698951,
+                "encryptedCID": null,
+                "cid": "z4odYNo3uQhHhypXze7ThLAKrnSGsJ3cRf1bYsvnt65Mqi9gp",
+                "hashes": null,
+                "thumbnail": null
+            },
+            "ext": null,
+            "history": null
+        },
+        "archlinux-x86_64.iso": {
+            "name": "archlinux-x86_64.iso",
+            "created": 1704127623412,
+            "modified": 1704127623412,
+            "version": 0,
+            "mimeType": "application/x-iso9660-image",
+            "file": {
+                "ts": 1704127623412,
+                "encryptedCID": null,
+                "cid": "z6e5sRB3BeynWLyrzRPLWFetUS7PNDZWZHURnFAyikcRyGTnxjMwC",
+                "hashes": null,
+                "thumbnail": null
+            },
+            "ext": null,
+            "history": null
+        },
+        "archlinux-x86_64.iso.sig": {
+            "name": "archlinux-x86_64.iso.sig",
+            "created": 1704127698648,
+            "modified": 1704127698648,
+            "version": 0,
+            "mimeType": "application/pgp-signature",
+            "file": {
+                "ts": 1704127698648,
+                "encryptedCID": null,
+                "cid": "z4odLKvhW8QPH4dLNqVzU1Zq6nj1MBdU6HYpJmThdzag4RJma",
+                "hashes": null,
+                "thumbnail": null
+            },
+            "ext": null,
+            "history": null
+        },
+        "archlinux-2024.01.01-x86_64.iso": {
+            "name": "archlinux-2024.01.01-x86_64.iso",
+            "created": 1704127623412,
+            "modified": 1704127623412,
+            "version": 0,
+            "mimeType": "application/x-iso9660-image",
+            "file": {
+                "ts": 1704127623412,
+                "encryptedCID": null,
+                "cid": "z6e5sRB3BeynWLyrzRPLWFetUS7PNDZWZHURnFAyikcRyGTnxjMwC",
+                "hashes": null,
+                "thumbnail": null
+            },
+            "ext": null,
+            "history": null
+        },
+        "archlinux-2024.01.01-x86_64.iso.sig": {
+            "name": "archlinux-2024.01.01-x86_64.iso.sig",
+            "created": 1704127698648,
+            "modified": 1704127698648,
+            "version": 0,
+            "mimeType": "application/pgp-signature",
+            "file": {
+                "ts": 1704127698648,
+                "encryptedCID": null,
+                "cid": "z4odLKvhW8QPH4dLNqVzU1Zq6nj1MBdU6HYpJmThdzag4RJma",
+                "hashes": null,
+                "thumbnail": null
+            },
+            "ext": null,
+            "history": null
+        },
+        "archlinux-2024.01.01-x86_64.iso.torrent": {
+            "name": "archlinux-2024.01.01-x86_64.iso.torrent",
+            "created": 1704127703578,
+            "modified": 1704127703578,
+            "version": 0,
+            "mimeType": "application/x-bittorrent",
+            "file": {
+                "ts": 1704127703578,
+                "encryptedCID": null,
+                "cid": "zHnmSZc3qDZMGfhNaZ6iVjkbxeVzMonKtAnTmBtRSwXy7RLYQK",
+                "hashes": null,
+                "thumbnail": null
+            },
+            "ext": null,
+            "history": null
+        },
+        "archlinux-bootstrap-2024.01.01-x86_64.tar.gz": {
+            "name": "archlinux-bootstrap-2024.01.01-x86_64.tar.gz",
+            "created": 1704127689514,
+            "modified": 1704127689514,
+            "version": 0,
+            "mimeType": null,
+            "file": {
+                "ts": 1704127689514,
+                "encryptedCID": null,
+                "cid": "z6e5xDpewiueHz6n9HJz21uGmTDvvctawujzYSeu9773k8s9RkG8a",
+                "hashes": null,
+                "thumbnail": null
+            },
+            "ext": null,
+            "history": null
+        },
+        "archlinux-bootstrap-2024.01.01-x86_64.tar.gz.sig": {
+            "name": "archlinux-bootstrap-2024.01.01-x86_64.tar.gz.sig",
+            "created": 1704127698951,
+            "modified": 1704127698951,
+            "version": 0,
+            "mimeType": "application/pgp-signature",
+            "file": {
+                "ts": 1704127698951,
+                "encryptedCID": null,
+                "cid": "z4odYNo3uQhHhypXze7ThLAKrnSGsJ3cRf1bYsvnt65Mqi9gp",
+                "hashes": null,
+                "thumbnail": null
+            },
+            "ext": null,
+            "history": null
+        },
+        "b2sums.txt": {
+            "name": "b2sums.txt",
+            "created": 1704127703594,
+            "modified": 1704127703594,
+            "version": 0,
+            "mimeType": "text/plain",
+            "file": {
+                "ts": 1704127703594,
+                "encryptedCID": null,
+                "cid": "zHnoyywarEZHWrSMZjL55nXeMAPsYi1mhXQs7ZmWqkpSg5Auz1",
+                "hashes": null,
+                "thumbnail": null
+            },
+            "ext": null,
+            "history": null
+        },
+        "sha256sums.txt": {
+            "name": "sha256sums.txt",
+            "created": 1704127703591,
+            "modified": 1704127703591,
+            "version": 0,
+            "mimeType": "text/plain",
+            "file": {
+                "ts": 1704127703591,
+                "encryptedCID": null,
+                "cid": "zHnnZGvqwnoN2AE342zhrxGWnuJpv79esruhY259TkXYyqmPB6",
+                "hashes": null,
+                "thumbnail": null
+            },
+            "ext": null,
+            "history": null
+        }
+    },
+    "extraMetadata": {}
+}

metadata/testdata/webapp.json

@@ -0,0 +1,257 @@
+{
+    "type": "web_app",
+    "name": "web",
+    "tryFiles": [
+        "index.html"
+    ],
+    "errorPages": {
+        "404": "/404.html"
+    },
+    "paths": {
+        ".nojekyll": {
+            "cid": "uJh_DNRdxfkPcuDCWkyM_2u6EBElr0Gbu0aWu0QZUvp_XmUg",
+            "contentType": null
+        },
+        "404.html": {
+            "cid": "uJh999Ng_tTWPMEbePbRKZof5HeiiJngmVBh7gUrC3ngLjAoq",
+            "contentType": "text/html"
+        },
+        "FontAwesome/css/font-awesome.css": {
+            "cid": "uJh_M_h82B_m4J2q89FxU4w65HlI1--8jtVm3dAcWFiCwQhh5",
+            "contentType": "text/css"
+        },
+        "FontAwesome/fonts/FontAwesome.ttf": {
+            "cid": "uJh-29gvrgO3E6BtME5CtvLSHIx6flERr76pK2jXrgwKIq6yGAg",
+            "contentType": "application/x-font-ttf"
+        },
+        "FontAwesome/fonts/fontawesome-webfont.eot": {
+            "cid": "uJh8GI6QOtlnUy2BBPNF1_rL4j7SjAmES5IYP_3TNXAB-YW6HAg",
+            "contentType": "application/vnd.ms-fontobject"
+        },
+        "FontAwesome/fonts/fontawesome-webfont.svg": {
+            "cid": "uJh8-FX8_AihAVyprcTk0i1SvKn3ZfskOZj770fI2_8UHT9vHBg",
+            "contentType": "image/svg+xml"
+        },
+        "FontAwesome/fonts/fontawesome-webfont.ttf": {
+            "cid": "uJh-29gvrgO3E6BtME5CtvLSHIx6flERr76pK2jXrgwKIq6yGAg",
+            "contentType": "application/x-font-ttf"
+        },
+        "FontAwesome/fonts/fontawesome-webfont.woff": {
+            "cid": "uJh-EvJ3GDUONMdZifENOxC_-BaOjdUo-mW38NRjLxXMCNuh-AQ",
+            "contentType": "application/x-font-woff"
+        },
+        "FontAwesome/fonts/fontawesome-webfont.woff2": {
+            "cid": "uJh8HUcHcSbc2l_OpxhUOL0Ar_98EylsZ4xP6-u0Bt_4hCGgtAQ",
+            "contentType": "font/woff2"
+        },
+        "ayu-highlight.css": {
+            "cid": "uJh_BBS0byn7CHwK9kUP9EDG3PaaqNzE4Iuwh1TIjaAwufaED",
+            "contentType": "text/css"
+        },
+        "book.js": {
+            "cid": "uJh-TkrWLMe3rsSv25YWA84vpiGw7j1_9N-DjDUWnwMA7pAxj",
+            "contentType": "text/javascript"
+        },
+        "clipboard.min.js": {
+            "cid": "uJh_kBxqcztJ8FJHnSzazx-5ytgJFYzfQAWHe6k040wvRWAIq",
+            "contentType": "text/javascript"
+        },
+        "concepts/content-addressed-data.html": {
+            "cid": "uJh8wOjq44JQABX0fAKq7wDUWrFoVxBK3EZ4ZyjMOzsA-KgpH",
+            "contentType": "text/html"
+        },
+        "concepts/index.html": {
+            "cid": "uJh9pLJKkRpdBWlt7e_fVbOKTPbQtzCh_iABVxHGABrizlzEv",
+            "contentType": "text/html"
+        },
+        "concepts/peer-to-peer-network.html": {
+            "cid": "uJh9CaCY9_OKxvmSuEk0QlLgH49a92Vl4d4Y2eATCnsGPEac5",
+            "contentType": "text/html"
+        },
+        "concepts/registry.html": {
+            "cid": "uJh8iFI6fkaOYN6AG3AxbNzWhb3JrjCUgNRcyApN3BiXwJnw0",
+            "contentType": "text/html"
+        },
+        "css/chrome.css": {
+            "cid": "uJh_1Be20xtlkC2Xt5PiAq_KsTL6uHTpe57-aJgu88esilxEq",
+            "contentType": "text/css"
+        },
+        "css/general.css": {
+            "cid": "uJh8yfrMc6lxmDo-_C77H-jWkxwgToBr_XAmvtzhMtP5UsSUR",
+            "contentType": "text/css"
+        },
+        "css/print.css": {
+            "cid": "uJh976QhSYuKQm5dPbPvDNpO18UpsUomJn-8X8ae61X8RvvUC",
+            "contentType": "text/css"
+        },
+        "css/variables.css": {
+            "cid": "uJh_fAgKBRroMMKrWwNRwSpml6vh_FJObBRv6JgERqGEr3AAY",
+            "contentType": "text/css"
+        },
+        "elasticlunr.min.js": {
+            "cid": "uJh_BpIJeJ7hV2UYZMX0yjZC1tqhqhVZHdXtN83a9OFZuaYNG",
+            "contentType": "text/javascript"
+        },
+        "favicon.png": {
+            "cid": "uJh_5OT0gK218KjuSk2qAaZhWDDoLEyxyoDp2WeX7UKnJRS8W",
+            "contentType": "image/png"
+        },
+        "favicon.svg": {
+            "cid": "uJh9wwAN7mRAp5xp1gNh2MYElVvaIxo8SXGMNdXOxvSlv9isH",
+            "contentType": "image/svg+xml"
+        },
+        "fonts/OPEN-SANS-LICENSE.txt": {
+            "cid": "uJh-Dyzovz4KbYTjglbCDAWw03c36B7aNOHgnIsFPz4Ws5l4s",
+            "contentType": "text/plain"
+        },
+        "fonts/SOURCE-CODE-PRO-LICENSE.txt": {
+            "cid": "uJh9g1uChv94iJex9qSp0miZhG0dsihvYZmoDmRU2I1qOG7AR",
+            "contentType": "text/plain"
+        },
+        "fonts/fonts.css": {
+            "cid": "uJh8lzvS6ZHUOPdgUqL4pZMYwx18-1LHnUcLZD_hLapAoJyQO",
+            "contentType": "text/css"
+        },
+        "fonts/open-sans-v17-all-charsets-300.woff2": {
+            "cid": "uJh8P6ttpcyOrWOiPhOQIsxaOWKG05jnPzvuUaP6vGo4ADECt",
+            "contentType": "font/woff2"
+        },
+        "fonts/open-sans-v17-all-charsets-300italic.woff2": {
+            "cid": "uJh86sgHtF0QzpTk_rz02uICkCxwapdF7hiGvkYBpewUJNdCe",
+            "contentType": "font/woff2"
+        },
+        "fonts/open-sans-v17-all-charsets-600.woff2": {
+            "cid": "uJh-xsn_HW-SF6GLHvQc81-9uSUCSEGdalJTmmE0D9ANMdIiv",
+            "contentType": "font/woff2"
+        },
+        "fonts/open-sans-v17-all-charsets-600italic.woff2": {
+            "cid": "uJh_ONfCHHrkyEGWtfw6hojsJNBbGLW5xUKne3BueeqXN_oik",
+            "contentType": "font/woff2"
+        },
+        "fonts/open-sans-v17-all-charsets-700.woff2": {
+            "cid": "uJh9voO4iLK5W4_ucP_ZFRHeCboAg122x3Ylgd3JxizkcoLyv",
+            "contentType": "font/woff2"
+        },
+        "fonts/open-sans-v17-all-charsets-700italic.woff2": {
+            "cid": "uJh92FFWrhZ9pc06a6Y0rSid9x415zYgd9muPXODnBtFqlGCf",
+            "contentType": "font/woff2"
+        },
+        "fonts/open-sans-v17-all-charsets-800.woff2": {
+            "cid": "uJh8PMxxkzvfnSVr7CemKriubw1mjIv4D3PpCYH-TgNHb9fit",
+            "contentType": "font/woff2"
+        },
+        "fonts/open-sans-v17-all-charsets-800italic.woff2": {
+            "cid": "uJh-tjTL9pf5qIrGJ6hyVog5pBDhjKidi8I4OlJOKe9zmlmyf",
+            "contentType": "font/woff2"
+        },
+        "fonts/open-sans-v17-all-charsets-italic.woff2": {
+            "cid": "uJh_JZKjdaz_DYZKJpfzGYpUybkCiik53yZQB3f6ux7k4YHSg",
+            "contentType": "font/woff2"
+        },
+        "fonts/open-sans-v17-all-charsets-regular.woff2": {
+            "cid": "uJh86CThN35aD2OjVQlLIUUWtUIPa_xn39wXVlWcqD8HwIeSo",
+            "contentType": "font/woff2"
+        },
+        "fonts/source-code-pro-v11-all-charsets-500.woff2": {
+            "cid": "uJh_69C-IE9DktUDwkKiXwnzV7yLLoY2zZoO1hmrzz1JAnATn",
+            "contentType": "font/woff2"
+        },
+        "highlight.css": {
+            "cid": "uJh-9y3aKQZy37phJKMLvnH9A0q7v2E8Rr2fbCghoWmpP4K0E",
+            "contentType": "text/css"
+        },
+        "highlight.js": {
+            "cid": "uJh-LUv93t01qYGO2-1qZ4sa9_boKVw2wBaDl3FWUO62JIOIQAg",
+            "contentType": "text/javascript"
+        },
+        "index.html": {
+            "cid": "uJh_oll4cULUNfgd0pe_pq-cifdlVgLVgFrTZzv0Otx5lUgAu",
+            "contentType": "text/html"
+        },
+        "install/caddy.html": {
+            "cid": "uJh_7vEWUpxg4m7AJu-4rldjeJ_Si1EjkNA_I4vgSIjRX_Uwy",
+            "contentType": "text/html"
+        },
+        "install/config/index.html": {
+            "cid": "uJh-vFERg2iOoKh5ELsdRtwU5iH4TFWEgn9D40KgIQfbU9Sg2",
+            "contentType": "text/html"
+        },
+        "install/index.html": {
+            "cid": "uJh8GYnazHqYrp_b8qHA8wGfaTYpFCG68cwt1O_gdY5ZIXgg9",
+            "contentType": "text/html"
+        },
+        "mark.min.js": {
+            "cid": "uJh8AqaIgOLDy8tpyY6qIZ9ELpZNL6NKytLEO2ihNAO9-iahD",
+            "contentType": "text/javascript"
+        },
+        "metadata/directory.html": {
+            "cid": "uJh-Tnl5MTw0dy6y1TGPPU7qylSKn0OcG-QIrm7W_einNadku",
+            "contentType": "text/html"
+        },
+        "metadata/index.html": {
+            "cid": "uJh9VJYuxAudF65ApEfsbzysllxFtAQeQg399O7k-0JxA9xEx",
+            "contentType": "text/html"
+        },
+        "metadata/media.html": {
+            "cid": "uJh864pdmfnBlpHHEa8KrZSDy7l4oGALJYsrerv-L2GUPQIQv",
+            "contentType": "text/html"
+        },
+        "metadata/web-app.html": {
+            "cid": "uJh-Dyd8nDOh_7s_XGNLmrYq9UeLXel-oeKz_fkiQ97L-Fjox",
+            "contentType": "text/html"
+        },
+        "print.html": {
+            "cid": "uJh-jtbPI4rEW-kWjYu2-tZT17gBGoMyOYemdJWAJbCIYrvCc",
+            "contentType": "text/html"
+        },
+        "searcher.js": {
+            "cid": "uJh8Qa-ugWhb5VYtuHQ8PvExFNSC_KSrMJ12sRLcH0gaM3CpI",
+            "contentType": "text/javascript"
+        },
+        "searchindex.js": {
+            "cid": "uJh-SHxgLyg4tjwRhLjSyGkU6GWMaDWGtAakBlrSC_4Oa9kmaAw",
+            "contentType": "text/javascript"
+        },
+        "searchindex.json": {
+            "cid": "uJh-KzHML7Jw374DotB994AAKRCNJcpfJ5preJuy0wQA1bCqaAw",
+            "contentType": "application/json"
+        },
+        "stores/arweave.html": {
+            "cid": "uJh8ejPUBJyo1WA9u_-75kNDAJw7tviw8TIkPkVfQp08qHOIu",
+            "contentType": "text/html"
+        },
+        "stores/index.html": {
+            "cid": "uJh9KpaV_mEK9hm17lsPN0jq7M5D657FVQldsc1suTLG12R4w",
+            "contentType": "text/html"
+        },
+        "stores/local.html": {
+            "cid": "uJh__s-SqZLuen-Qk3o-r8IkBzACZmqfEDFWpW1I2hoQYKoIw",
+            "contentType": "text/html"
+        },
+        "stores/s3.html": {
+            "cid": "uJh8PwIVeCloJJYlQxI2MLOwKFO1oc50Ho0cXNHnHr2OYBPsv",
+            "contentType": "text/html"
+        },
+        "stores/sia.html": {
+            "cid": "uJh8D-zdXUFnxSc5O6Ggmk_7qdfLrK2qmPLUEr9lzMOzaXJY0",
+            "contentType": "text/html"
+        },
+        "tomorrow-night.css": {
+            "cid": "uJh9KIWHDZw2IFTRxks9TU8gMlT9ntFZMX11ZBMgSLfG8jZkG",
+            "contentType": "text/css"
+        },
+        "tools/cid-one.html": {
+            "cid": "uJh9h11sJXOGXUv24xrervCjaOQ4eR-x5w1k25MyNu64g05sx",
+            "contentType": "text/html"
+        },
+        "tools/index.html": {
+            "cid": "uJh8iU6yXW97HblRJAl9Y8NYk1gm3hqVWLjVRufQbJF6Jy64u",
+            "contentType": "text/html"
+        },
+        "tools/s5-cx.html": {
+            "cid": "uJh8Fnc6d3X15LvTNrcdQkz34a2Dd0NkzMdEoynHtoL_MnZQx",
+            "contentType": "text/html"
+        }
+    },
+    "extraMetadata": {}
+}

metadata/user_identity_metadata.go

@@ -0,0 +1,11 @@
+package metadata
+
+import "git.lumeweb.com/LumeWeb/libs5-go/encoding"
+
+type UserIdentityMetadata struct {
+	UserID         *encoding.CID
+	Details        UserIdentityMetadataDetails
+	SigningKeys    []UserIdentityPublicKey
+	EncryptionKeys []UserIdentityPublicKey
+	Links          map[int]*encoding.CID
+}

metadata/user_identity_metadata_details.go

@@ -0,0 +1,7 @@
+package metadata
+
+type UserIdentityMetadataDetails struct {
+	Created   int64
+	CreatedBy string
+	Modified  int64
+}

metadata/user_identity_publickey.go

@@ -0,0 +1,5 @@
+package metadata
+
+type UserIdentityPublicKey struct {
+	Key []byte
+}

metadata/web_app.go

@@ -0,0 +1,298 @@
+package metadata
+
+import (
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/serialize"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/emirpasic/gods/maps/linkedhashmap"
+	"github.com/vmihailenco/msgpack/v5"
+	"sort"
+)
+
+var (
+	_ Metadata             = (*WebAppMetadata)(nil)
+	_ SerializableMetadata = (*WebAppMetadata)(nil)
+	_ SerializableMetadata = (*WebAppFileMap)(nil)
+	_ SerializableMetadata = (*WebAppErrorPages)(nil)
+)
+
+type WebAppErrorPages map[int]string
+
+type WebAppMetadata struct {
+	BaseMetadata
+	Name          string           `json:"name"`
+	TryFiles      []string         `json:"tryFiles"`
+	ErrorPages    WebAppErrorPages `json:"errorPages"`
+	ExtraMetadata ExtraMetadata    `json:"extraMetadata"`
+	Paths         *WebAppFileMap   `json:"paths"`
+}
+
+func NewWebAppMetadata(name string, tryFiles []string, extraMetadata ExtraMetadata, errorPages WebAppErrorPages, paths *WebAppFileMap) *WebAppMetadata {
+	return &WebAppMetadata{
+		Name:          name,
+		TryFiles:      tryFiles,
+		ExtraMetadata: extraMetadata,
+		ErrorPages:    errorPages,
+		Paths:         paths,
+	}
+}
+func NewEmptyWebAppMetadata() *WebAppMetadata {
+	return &WebAppMetadata{}
+}
+
+func (wm *WebAppMetadata) EncodeMsgpack(enc *msgpack.Encoder) error {
+	err := serialize.InitMarshaller(enc, types.MetadataTypeWebApp)
+	if err != nil {
+		return err
+	}
+
+	items := make([]interface{}, 5)
+
+	if wm.ErrorPages == nil {
+		wm.ErrorPages = make(WebAppErrorPages)
+	}
+
+	items[0] = wm.Name
+	items[1] = wm.TryFiles
+	items[2] = &wm.ErrorPages
+	items[3] = wm.Paths
+	items[4] = wm.ExtraMetadata
+
+	return enc.Encode(items)
+}
+
+func (wm *WebAppMetadata) DecodeMsgpack(dec *msgpack.Decoder) error {
+	_, err := serialize.InitUnmarshaller(dec, types.MetadataTypeWebApp)
+	if err != nil {
+		return err
+	}
+
+	val, err := dec.DecodeArrayLen()
+
+	if err != nil {
+		return err
+	}
+
+	if val != 5 {
+		return errors.New(" Corrupted metadata")
+	}
+
+	for i := 0; i < val; i++ {
+		switch i {
+		case 0:
+			wm.Name, err = dec.DecodeString()
+			if err != nil {
+				return err
+			}
+		case 1:
+			err = dec.Decode(&wm.TryFiles)
+			if err != nil {
+				return err
+			}
+		case 2:
+			err = dec.Decode(&wm.ErrorPages)
+			if err != nil {
+				return err
+			}
+		case 3:
+			err = dec.Decode(&wm.Paths)
+			if err != nil {
+				return err
+			}
+
+		case 4:
+			err = dec.Decode(&wm.ExtraMetadata)
+			if err != nil {
+				return err
+			}
+		default:
+			return errors.New(" Corrupted metadata")
+		}
+	}
+
+	wm.Type = "web_app"
+
+	return nil
+}
+
+type WebAppFileMap struct {
+	linkedhashmap.Map
+}
+
+func NewWebAppFileMap() *WebAppFileMap {
+	return &WebAppFileMap{*linkedhashmap.New()}
+}
+
+func (wafm *WebAppFileMap) Put(key string, value WebAppMetadataFileReference) {
+	wafm.Map.Put(key, value)
+}
+
+func (wafm *WebAppFileMap) Get(key string) (WebAppMetadataFileReference, bool) {
+	value, found := wafm.Map.Get(key)
+	if !found {
+		return WebAppMetadataFileReference{}, false
+	}
+	return value.(WebAppMetadataFileReference), true
+}
+
+func (wafm *WebAppFileMap) Remove(key string) {
+	wafm.Map.Remove(key)
+}
+
+func (wafm *WebAppFileMap) Keys() []string {
+	keys := wafm.Map.Keys()
+	ret := make([]string, len(keys))
+	for i, key := range keys {
+		ret[i] = key.(string)
+	}
+	return ret
+}
+
+func (wafm *WebAppFileMap) Values() []WebAppMetadataFileReference {
+	values := wafm.Map.Values()
+	ret := make([]WebAppMetadataFileReference, len(values))
+	for i, value := range values {
+		ret[i] = value.(WebAppMetadataFileReference)
+	}
+	return ret
+}
+
+func (wafm *WebAppFileMap) Sort() {
+	keys := wafm.Keys()
+	newMap := NewWebAppFileMap()
+
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		value, _ := wafm.Get(key)
+		newMap.Put(key, value)
+	}
+	wafm.Map = newMap.Map
+}
+
+func (wafm *WebAppFileMap) EncodeMsgpack(encoder *msgpack.Encoder) error {
+	wafm.Sort()
+
+	err := encoder.EncodeArrayLen(wafm.Size())
+	if err != nil {
+		return err
+	}
+
+	for _, key := range wafm.Keys() {
+		value, _ := wafm.Get(key)
+
+		data :=
+			make([]interface{}, 3)
+
+		data[0] = key
+		data[1] = value.Cid.ToBytes()
+		data[2] = value.ContentType
+
+		err := encoder.Encode(data)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (wafm *WebAppFileMap) DecodeMsgpack(decoder *msgpack.Decoder) error {
+	arrLen, err := decoder.DecodeArrayLen()
+	if err != nil {
+		return err
+	}
+
+	wafm.Map = *linkedhashmap.New()
+
+	for i := 0; i < arrLen; i++ {
+		data := make([]interface{}, 3)
+
+		if len(data) != 3 {
+			return errors.New("Corrupted metadata")
+		}
+
+		err = decoder.Decode(&data)
+		if err != nil {
+			return err
+		}
+
+		path, ok := data[0].(string)
+		if !ok {
+			return errors.New("Corrupted metadata")
+		}
+
+		cidData, ok := data[1].([]byte)
+		if !ok {
+			return errors.New("Corrupted metadata")
+		}
+
+		contentType, ok := data[2].(string)
+		if !ok {
+			return errors.New("Corrupted metadata")
+		}
+
+		cid, err := encoding.CIDFromBytes(cidData)
+		if err != nil {
+			return err
+		}
+
+		wafm.Put(path, *NewWebAppMetadataFileReference(cid, contentType))
+	}
+
+	wafm.Sort()
+
+	return nil
+}
+
+func (w *WebAppErrorPages) EncodeMsgpack(enc *msgpack.Encoder) error {
+	if w == nil || *w == nil {
+		return enc.EncodeMapLen(0)
+	}
+
+	err := enc.EncodeMapLen(len(*w))
+	if err != nil {
+		return err
+	}
+
+	for k, v := range *w {
+		if err := enc.EncodeInt(int64(k)); err != nil {
+			return err
+		}
+		if err := enc.EncodeString(v); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (w *WebAppErrorPages) DecodeMsgpack(dec *msgpack.Decoder) error {
+	if *w == nil {
+		*w = make(map[int]string)
+	}
+
+	mapLen, err := dec.DecodeMapLen()
+	if err != nil {
+		return err
+	}
+
+	*w = make(map[int]string, mapLen)
+
+	for i := 0; i < mapLen; i++ {
+		key, err := dec.DecodeInt()
+		if err != nil {
+			return err
+		}
+
+		value, err := dec.DecodeString()
+		if err != nil {
+			return err
+		}
+
+		(*w)[key] = value
+	}
+
+	return nil
+}

metadata/web_app_reference.go

@@ -0,0 +1,15 @@
+package metadata
+
+import "git.lumeweb.com/LumeWeb/libs5-go/encoding"
+
+type WebAppMetadataFileReference struct {
+	ContentType string        `json:"contentType"`
+	Cid         *encoding.CID `json:"cid"`
+}
+
+func NewWebAppMetadataFileReference(cid *encoding.CID, contentType string) *WebAppMetadataFileReference {
+	return &WebAppMetadataFileReference{
+		Cid:         cid,
+		ContentType: contentType,
+	}
+}

metadata/web_app_test.go

@@ -0,0 +1,124 @@
+package metadata
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/assert"
+	"github.com/vmihailenco/msgpack/v5"
+	"testing"
+)
+
+func (wafr WebAppMetadataFileReference) Equal(other WebAppMetadataFileReference) bool {
+	return wafr.Cid.Equals(other.Cid) && wafr.ContentType == other.ContentType
+}
+
+func getWebappMeta() *WebAppMetadata {
+	data := getWebappContent()
+
+	var webapp WebAppMetadata
+
+	err := json.Unmarshal(data, &webapp)
+	if err != nil {
+		panic(err)
+	}
+
+	return &webapp
+}
+
+func getWebappContent() []byte {
+	data := readFile("webapp.json")
+
+	return data
+}
+
+func TestWebAppMetadata_DecodeJSON(t *testing.T) {
+	tests := []struct {
+		name    string
+		wantErr bool
+	}{
+		{
+			name:    "Decode",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			jsonDm := getWebappMeta()
+			dm := &WebAppMetadata{}
+
+			if err := msgpack.Unmarshal(readFile("webapp.bin"), dm); (err != nil) != tt.wantErr {
+				t.Errorf("DecodeMsgpack() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			if !cmp.Equal(jsonDm, dm) {
+				fmt.Println(cmp.Diff(jsonDm, dm))
+				t.Errorf("DecodeMsgpack() error = %v, wantErr %v", "msgpack does not match json", tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestWebAppMetadata_DecodeMsgpack(t *testing.T) {
+	tests := []struct {
+		name    string
+		wantErr bool
+	}{
+		{
+			name:    "Decode",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			jsonDm := getWebappMeta()
+			dm := &WebAppMetadata{}
+
+			if err := msgpack.Unmarshal(readFile("webapp.bin"), dm); (err != nil) != tt.wantErr {
+				t.Errorf("DecodeMsgpack() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			if !cmp.Equal(jsonDm, dm) {
+				t.Errorf("DecodeMsgpack() error = %v, wantErr %v", "msgpack does not match json", tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestWebAppMetadata_EncodeJSON(t *testing.T) {
+	tests := []struct {
+		name    string
+		wantErr bool
+	}{
+		{
+			name:    "Encode",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			jsonDm := getWebappContent()
+			dm := &WebAppMetadata{}
+
+			if err := json.Unmarshal(jsonDm, dm); (err != nil) != tt.wantErr {
+				t.Errorf("EncodeJSON() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			jsonData, err := json.MarshalIndent(dm, "", "\t")
+
+			if (err != nil) != tt.wantErr {
+				t.Errorf("EncodeJSON() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			buf := bytes.NewBuffer(nil)
+
+			err = json.Indent(buf, jsonData, "", "\t")
+			if err != nil {
+				t.Errorf("EncodeJSON() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			assert.Equal(t, buf.Bytes(), jsonData)
+		})
+	}
+}

net/net.go

@@ -0,0 +1,67 @@
+package net
+
+import (
+	"errors"
+	"net/url"
+	"sync"
+)
+
+var (
+	ErrTransportNotSupported = errors.New("no static method registered for type")
+)
+
+type TransportPeerConfig struct {
+	Socket interface{}
+	Uris   []*url.URL
+}
+
+type PeerStatic interface {
+	Connect(uri *url.URL) (interface{}, error) // Returns a connection/socket
+}
+
+type PeerFactory interface {
+	NewPeer(options *TransportPeerConfig) (Peer, error)
+}
+
+var (
+	transports sync.Map
+)
+
+func init() {
+	transports = sync.Map{}
+	RegisterTransport("ws", &WebSocketPeer{})
+	RegisterTransport("wss", &WebSocketPeer{})
+}
+func RegisterTransport(peerType string, factory interface{}) {
+	if _, ok := factory.(PeerFactory); !ok {
+		panic("factory must implement PeerFactory")
+	}
+
+	if _, ok := factory.(PeerStatic); !ok {
+		panic("factory must implement PeerStatic")
+	}
+
+	transports.Store(peerType, factory)
+}
+
+func CreateTransportSocket(peerType string, uri *url.URL) (interface{}, error) {
+	static, ok := transports.Load(peerType)
+	if !ok {
+		return nil, ErrTransportNotSupported
+	}
+
+	t, err := static.(PeerStatic).Connect(uri)
+
+	return t, err
+}
+
+func CreateTransportPeer(peerType string, options *TransportPeerConfig) (Peer, error) {
+	factory, ok := transports.Load(peerType)
+	if !ok {
+		return nil, errors.New("no factory registered for type: " + peerType)
+	}
+
+	t, err := factory.(PeerFactory).NewPeer(options)
+
+	return t, err
+}

net/peer.go

@@ -0,0 +1,167 @@
+package net
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"go.uber.org/zap"
+	"net"
+	"net/url"
+	"sync"
+)
+
+var (
+	_ Peer = (*BasePeer)(nil)
+)
+
+// EventCallback type for the callback function
+type EventCallback func(event []byte) error
+
+// CloseCallback type for the OnClose callback
+type CloseCallback func()
+
+// ErrorCallback type for the onError callback
+type ErrorCallback func(args ...interface{})
+
+// ListenerOptions struct for options
+type ListenerOptions struct {
+	OnClose *CloseCallback
+	OnError *ErrorCallback
+	Logger  *zap.Logger
+}
+
+type Peer interface {
+	SendMessage(message []byte) error
+	RenderLocationURI() string
+	ListenForMessages(callback EventCallback, options ListenerOptions)
+	End() error
+	EndForAbuse() error
+	SetId(id *encoding.NodeId)
+	Id() *encoding.NodeId
+	SetChallenge(challenge []byte)
+	Challenge() []byte
+	SetSocket(socket interface{})
+	Socket() interface{}
+	SetConnected(isConnected bool)
+	IsConnected() bool
+	SetConnectionURIs(uris []*url.URL)
+	ConnectionURIs() []*url.URL
+	IsHandshakeDone() bool
+	SetHandshakeDone(status bool)
+	GetIPString() string
+	GetIP() net.Addr
+	SetIP(ip net.Addr)
+	Abuser() bool
+}
+
+type BasePeer struct {
+	connectionURIs []*url.URL
+	isConnected    bool
+	challenge      []byte
+	socket         interface{}
+	id             *encoding.NodeId
+	handshaked     bool
+	lock           sync.RWMutex
+}
+
+func (b *BasePeer) IsConnected() bool {
+	b.lock.RLock()
+	defer b.lock.RUnlock()
+	return b.isConnected
+}
+
+func (b *BasePeer) SetConnected(isConnected bool) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	b.isConnected = isConnected
+}
+
+func (b *BasePeer) SendMessage(message []byte) error {
+	panic("must implement in child class")
+}
+
+func (b *BasePeer) RenderLocationURI() string {
+	panic("must implement in child class")
+}
+
+func (b *BasePeer) ListenForMessages(callback EventCallback, options ListenerOptions) {
+	panic("must implement in child class")
+}
+
+func (b *BasePeer) End() error {
+	panic("must implement in child class")
+}
+func (b *BasePeer) EndForAbuse() error {
+	panic("must implement in child class")
+}
+func (b *BasePeer) GetIPString() string {
+	panic("must implement in child class")
+}
+
+func (b *BasePeer) GetIP() net.Addr {
+	panic("must implement in child class")
+}
+
+func (b *BasePeer) SetIP(ip net.Addr) {
+	panic("must implement in child class")
+}
+
+func (b *BasePeer) Challenge() []byte {
+	b.lock.RLock()
+	defer b.lock.RUnlock()
+	return b.challenge
+}
+
+func (b *BasePeer) SetChallenge(challenge []byte) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	b.challenge = challenge
+}
+
+func (b *BasePeer) Socket() interface{} {
+	b.lock.RLock()
+	defer b.lock.RUnlock()
+	return b.socket
+}
+
+func (b *BasePeer) SetSocket(socket interface{}) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	b.socket = socket
+}
+
+func (b *BasePeer) Id() *encoding.NodeId {
+	b.lock.RLock()
+	defer b.lock.RUnlock()
+	return b.id
+}
+
+func (b *BasePeer) SetId(id *encoding.NodeId) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	b.id = id
+}
+func (b *BasePeer) SetConnectionURIs(uris []*url.URL) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	b.connectionURIs = uris
+}
+func (b *BasePeer) ConnectionURIs() []*url.URL {
+	b.lock.RLock()
+	b.lock.RUnlock()
+	return b.connectionURIs
+}
+
+func (b *BasePeer) IsHandshakeDone() bool {
+	b.lock.RLock()
+	defer b.lock.RUnlock()
+	return b.handshaked
+}
+
+func (b *BasePeer) SetHandshakeDone(status bool) {
+	b.lock.Lock()
+	defer b.lock.Unlock()
+	b.handshaked = status
+}
+
+func (b *BasePeer) Abuser() bool {
+	panic("must implement in child class")
+}

net/ws.go

@@ -0,0 +1,181 @@
+package net
+
+import (
+	"context"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"net"
+	"net/url"
+	"nhooyr.io/websocket"
+	"sync"
+)
+
+var (
+	_ PeerFactory = (*WebSocketPeer)(nil)
+	_ PeerStatic  = (*WebSocketPeer)(nil)
+	_ Peer        = (*WebSocketPeer)(nil)
+)
+
+type WebSocketPeer struct {
+	BasePeer
+	socket *websocket.Conn
+	abuser bool
+	ip     net.Addr
+}
+
+func (p *WebSocketPeer) Connect(uri *url.URL) (interface{}, error) {
+	dial, _, err := websocket.Dial(context.Background(), uri.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return dial, nil
+}
+
+func (p *WebSocketPeer) NewPeer(options *TransportPeerConfig) (Peer, error) {
+	peer := &WebSocketPeer{
+		BasePeer: BasePeer{
+			connectionURIs: options.Uris,
+			socket:         options.Socket,
+		},
+		socket: options.Socket.(*websocket.Conn),
+	}
+
+	return peer, nil
+}
+
+func (p *WebSocketPeer) SendMessage(message []byte) error {
+	err := p.socket.Write(context.Background(), websocket.MessageBinary, message)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *WebSocketPeer) RenderLocationURI() string {
+	return "WebSocket client"
+}
+
+func (p *WebSocketPeer) ListenForMessages(callback EventCallback, options ListenerOptions) {
+	errChan := make(chan error, 10)
+	doneChan := make(chan struct{})
+	var wg sync.WaitGroup
+
+	for {
+		_, message, err := p.socket.Read(context.Background())
+		if err != nil {
+			if options.OnError != nil {
+				(*options.OnError)(err)
+			}
+			break
+		}
+
+		wg.Add(1)
+		// Process each message in a separate goroutine
+		go func(msg []byte) {
+			defer wg.Done()
+			// Call the callback and send any errors to the error channel
+			if err := callback(msg); err != nil {
+				select {
+				case errChan <- err:
+				case <-doneChan:
+					// Stop sending errors if doneChan is closed
+				}
+			}
+		}(message)
+
+		// Non-blocking error check
+		select {
+		case err := <-errChan:
+			if options.OnError != nil {
+				(*options.OnError)(err)
+			}
+		default:
+		}
+	}
+
+	if options.OnClose != nil {
+		(*options.OnClose)()
+	}
+
+	// Close doneChan and wait for all goroutines to finish
+	close(doneChan)
+	wg.Wait()
+	// Handle remaining errors
+	close(errChan)
+	for err := range errChan {
+		if options.OnError != nil {
+			(*options.OnError)(err)
+		}
+	}
+}
+
+func (p *WebSocketPeer) End() error {
+	err := p.socket.Close(websocket.StatusNormalClosure, "")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+func (p *WebSocketPeer) EndForAbuse() error {
+	p.BasePeer.lock.Lock()
+	defer p.BasePeer.lock.Unlock()
+	p.abuser = true
+	err := p.socket.Close(websocket.StatusPolicyViolation, "")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+func (p *WebSocketPeer) SetId(id *encoding.NodeId) {
+	p.BasePeer.lock.Lock()
+	defer p.BasePeer.lock.Unlock()
+	p.id = id
+}
+
+func (p *WebSocketPeer) SetChallenge(challenge []byte) {
+	p.BasePeer.lock.Lock()
+	defer p.BasePeer.lock.Unlock()
+	p.challenge = challenge
+}
+
+func (p *WebSocketPeer) GetChallenge() []byte {
+	p.BasePeer.lock.RLock()
+	defer p.BasePeer.lock.RUnlock()
+	return p.challenge
+}
+
+func (p *WebSocketPeer) GetIP() net.Addr {
+	p.BasePeer.lock.RLock()
+	defer p.BasePeer.lock.RUnlock()
+	if p.ip != nil {
+		return p.ip
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	netConn := websocket.NetConn(ctx, p.socket, websocket.MessageBinary)
+
+	ipAddr := netConn.RemoteAddr()
+
+	cancel()
+
+	return ipAddr
+}
+
+func (p *WebSocketPeer) SetIP(ip net.Addr) {
+	p.BasePeer.lock.Lock()
+	defer p.BasePeer.lock.Unlock()
+	p.ip = ip
+}
+
+func (b *WebSocketPeer) GetIPString() string {
+	return b.GetIP().String()
+}
+
+func (p *WebSocketPeer) Abuser() bool {
+	p.BasePeer.lock.RLock()
+	defer p.BasePeer.lock.RUnlock()
+	return p.abuser
+}

node/node.go

@@ -0,0 +1,102 @@
+package node
+
+import (
+	"context"
+	"git.lumeweb.com/LumeWeb/libs5-go/config"
+	"git.lumeweb.com/LumeWeb/libs5-go/db"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/protocol"
+	"git.lumeweb.com/LumeWeb/libs5-go/service"
+	_default "git.lumeweb.com/LumeWeb/libs5-go/service/default"
+	"go.uber.org/zap"
+)
+
+type Node struct {
+	nodeConfig *config.NodeConfig
+	services   service.Services
+}
+
+func (n *Node) Services() service.Services {
+	return n.services
+}
+
+func NewNode(config *config.NodeConfig, services service.Services) *Node {
+	return &Node{
+		nodeConfig: config,
+		services:   services, // Services are passed in, not created here
+	}
+}
+
+func (n *Node) IsStarted() bool {
+	return n.services.IsStarted()
+}
+
+func (n *Node) Config() *config.NodeConfig {
+	return n.nodeConfig
+}
+
+func (n *Node) Logger() *zap.Logger {
+	if n.nodeConfig != nil {
+		return n.nodeConfig.Logger
+	}
+	return nil
+}
+
+func (n *Node) Db() db.KVStore {
+	if n.nodeConfig != nil {
+		return n.nodeConfig.DB
+	}
+	return nil
+}
+
+func (n *Node) Start(ctx context.Context) error {
+	protocol.RegisterProtocols()
+	protocol.RegisterSignedProtocols()
+
+	return n.services.Start(ctx)
+}
+
+func (n *Node) Init(ctx context.Context) error {
+	return n.services.Init(ctx)
+}
+
+func (n *Node) Stop(ctx context.Context) error {
+	return n.services.Stop(ctx)
+}
+
+func (n *Node) WaitOnConnectedPeers() {
+	n.services.P2P().WaitOnConnectedPeers()
+}
+
+func (n *Node) NetworkId() string {
+	return n.services.P2P().NetworkId()
+}
+
+func (n *Node) NodeId() *encoding.NodeId {
+	return n.services.P2P().NodeId()
+}
+
+func DefaultNode(config *config.NodeConfig) *Node {
+	params := service.ServiceParams{
+		Logger: config.Logger,
+		Config: config,
+		Db:     config.DB,
+	}
+
+	// Initialize services first
+	p2pService := _default.NewP2P(params)
+	registryService := _default.NewRegistry(params)
+	httpService := _default.NewHTTP(params)
+	storageService := _default.NewStorage(params)
+
+	// Aggregate services
+	services := NewServices(ServicesParams{
+		P2P:      p2pService,
+		Registry: registryService,
+		HTTP:     httpService,
+		Storage:  storageService,
+	})
+
+	// Now create the node with the services
+	return NewNode(config, services)
+}

node/services.go

@@ -0,0 +1,120 @@
+package node
+
+import (
+	"context"
+	"git.lumeweb.com/LumeWeb/libs5-go/service"
+)
+
+var (
+	_ service.Services = (*ServicesImpl)(nil)
+)
+
+type ServicesParams struct {
+	P2P      service.P2PService
+	Registry service.RegistryService
+	HTTP     service.HTTPService
+	Storage  service.StorageService
+}
+
+type ServicesImpl struct {
+	p2p      service.P2PService
+	registry service.RegistryService
+	http     service.HTTPService
+	storage  service.StorageService
+	started  bool
+	starting bool
+}
+
+func (s *ServicesImpl) HTTP() service.HTTPService {
+	return s.http
+}
+
+func (s *ServicesImpl) Storage() service.StorageService {
+	return s.storage
+}
+
+func (s *ServicesImpl) All() []service.Service {
+	services := make([]service.Service, 0)
+	services = append(services, s.p2p)
+	services = append(services, s.registry)
+	services = append(services, s.http)
+	services = append(services, s.storage)
+
+	return services
+}
+
+func (s *ServicesImpl) Registry() service.RegistryService {
+	return s.registry
+}
+
+func NewServices(params ServicesParams) service.Services {
+	sc := &ServicesImpl{
+		p2p:      params.P2P,
+		registry: params.Registry,
+		http:     params.HTTP,
+		storage:  params.Storage,
+		started:  false,
+	}
+
+	for _, svc := range sc.All() {
+		svc.SetServices(sc)
+	}
+
+	return sc
+}
+
+func (s *ServicesImpl) P2P() service.P2PService {
+	return s.p2p
+}
+
+func (s *ServicesImpl) IsStarted() bool {
+	return s.started
+}
+
+func (s *ServicesImpl) IsStarting() bool {
+	return s.starting
+}
+
+func (s *ServicesImpl) Init(ctx context.Context) error {
+	err := s.p2p.Config().DB.Open()
+	if err != nil {
+		return err
+	}
+
+	for _, svc := range s.All() {
+		err := svc.Init(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *ServicesImpl) Start(ctx context.Context) error {
+	s.starting = true
+
+	for _, svc := range s.All() {
+		err := svc.Start(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	s.started = true
+	s.starting = false
+
+	return nil
+}
+func (s *ServicesImpl) Stop(ctx context.Context) error {
+	for _, svc := range s.All() {
+		err := svc.Stop(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	s.started = false
+
+	return nil
+}

protocol/handshake_open.go

@@ -0,0 +1,119 @@
+package protocol
+
+import (
+	"fmt"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+var _ EncodeableMessage = (*HandshakeOpen)(nil)
+var _ IncomingMessage = (*HandshakeOpen)(nil)
+
+type HandshakeOpen struct {
+	challenge []byte
+	networkId string
+	handshake []byte
+	HandshakeRequirement
+}
+
+func (h *HandshakeOpen) SetHandshake(handshake []byte) {
+	h.handshake = handshake
+}
+
+func (h HandshakeOpen) Challenge() []byte {
+	return h.challenge
+}
+
+func (h HandshakeOpen) NetworkId() string {
+	return h.networkId
+}
+
+func NewHandshakeOpen(challenge []byte, networkId string) *HandshakeOpen {
+	ho := &HandshakeOpen{
+		challenge: challenge,
+		networkId: networkId,
+	}
+
+	ho.SetRequiresHandshake(false)
+
+	return ho
+}
+func (h HandshakeOpen) EncodeMsgpack(enc *msgpack.Encoder) error {
+	err := enc.EncodeUint(uint64(types.ProtocolMethodHandshakeOpen))
+	if err != nil {
+		return err
+	}
+
+	err = enc.EncodeBytes(h.challenge)
+	if err != nil {
+		return err
+	}
+
+	if h.networkId != "" {
+		err = enc.EncodeString(h.networkId)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (h *HandshakeOpen) DecodeMessage(dec *msgpack.Decoder, message IncomingMessageData) error {
+	handshake, err := dec.DecodeBytes()
+
+	if err != nil {
+		return err
+	}
+
+	h.handshake = handshake
+
+	_, err = dec.PeekCode()
+
+	networkId := ""
+
+	if err != nil {
+		if err.Error() != "EOF" {
+			return err
+		}
+		h.networkId = networkId
+		return nil
+	}
+
+	networkId, err = dec.DecodeString()
+	if err != nil {
+		return err
+	}
+
+	h.networkId = networkId
+
+	return nil
+}
+
+func (h *HandshakeOpen) HandleMessage(message IncomingMessageData) error {
+	peer := message.Peer
+	mediator := message.Mediator
+
+	if h.networkId != mediator.NetworkId() {
+		return fmt.Errorf("Peer is in different network: %s", h.networkId)
+	}
+
+	handshake := NewHandshakeDoneRequest(h.handshake, types.SupportedFeatures, mediator.SelfConnectionUris())
+	hsMessage, err := msgpack.Marshal(handshake)
+
+	if err != nil {
+		return err
+	}
+
+	secureMessage, err := mediator.SignMessageSimple(hsMessage)
+
+	if err != nil {
+		return err
+	}
+
+	err = peer.SendMessage(secureMessage)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

protocol/handshake_open_test.go

@@ -0,0 +1,216 @@
+package protocol
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+	"github.com/vmihailenco/msgpack/v5"
+	"testing"
+)
+
+func TestHandshakeOpen_EncodeMsgpack(t *testing.T) {
+	type fields struct {
+		challenge []byte
+		networkId string
+	}
+	type args struct {
+		enc *msgpack.Encoder
+		buf *bytes.Buffer
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Normal Case",
+			fields: fields{
+				challenge: []byte("test-challenge"),
+				networkId: "test-network",
+			},
+			args: args{
+				buf: new(bytes.Buffer),
+			},
+			wantErr: false,
+		},
+		{
+			name: "Empty Network ID",
+			fields: fields{
+				challenge: []byte("test-challenge"),
+				networkId: "",
+			},
+			args: args{
+				buf: new(bytes.Buffer),
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		tt.args.enc = msgpack.NewEncoder(tt.args.buf)
+		t.Run(tt.name, func(t *testing.T) {
+			h := HandshakeOpen{
+				challenge: tt.fields.challenge,
+				networkId: tt.fields.networkId,
+			}
+			if err := h.EncodeMsgpack(tt.args.enc); (err != nil) != tt.wantErr {
+				t.Errorf("EncodeMsgpack() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			// Check the contents of the buffer to verify encoding
+			encodedData := tt.args.buf.Bytes()
+			if len(encodedData) == 0 && !tt.wantErr {
+				t.Errorf("Expected non-empty encoded data, got empty")
+			}
+
+			dec := msgpack.NewDecoder(bytes.NewReader(encodedData))
+
+			protocolMethod, err := dec.DecodeUint()
+			if err != nil {
+				t.Errorf("DecodeUint() error = %v", err)
+			}
+
+			assert.EqualValues(t, types.ProtocolMethodHandshakeOpen, protocolMethod)
+
+			challenge, err := dec.DecodeBytes()
+			if err != nil {
+				t.Errorf("DecodeBytes() error = %v", err)
+			}
+
+			assert.EqualValues(t, tt.fields.challenge, challenge)
+
+			networkId, err := dec.DecodeString()
+			if err != nil {
+				if err.Error() == "EOF" && tt.fields.networkId != "" {
+					t.Logf("DecodeString() networkId missing, got EOF")
+				}
+				if err.Error() != "EOF" {
+					t.Errorf("DecodeString() error = %v", err)
+				}
+			}
+
+			assert.EqualValues(t, tt.fields.networkId, networkId)
+		})
+	}
+}
+func TestHandshakeOpen_DecodeMessage(t *testing.T) {
+	type fields struct {
+		challenge []byte
+		networkId string
+		handshake []byte
+	}
+	type args struct {
+		base64EncodedData string // Base64 encoded msgpack data
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		wantErr assert.ErrorAssertionFunc
+	}{
+		{
+			name:   "Valid Handshake and NetworkID",
+			fields: fields{}, // Fields are now empty
+			args: args{
+				base64EncodedData: "xBNzYW1wbGVIYW5kc2hha2VEYXRhr3NhbXBsZU5ldHdvcmtJRA==",
+			},
+			wantErr: assert.NoError,
+		},
+		{
+			name:   "Valid Handshake and Empty NetworkID",
+			fields: fields{}, // Fields are now empty
+			args: args{
+				base64EncodedData: "xBNzYW1wbGVIYW5kc2hha2VEYXRh",
+			},
+			wantErr: assert.NoError,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			h := &HandshakeOpen{}
+
+			decodedData, _ := base64.StdEncoding.DecodeString(tt.args.base64EncodedData)
+			reader := bytes.NewReader(decodedData)
+			dec := msgpack.NewDecoder(reader)
+
+			tt.wantErr(t, h.DecodeMessage(dec), fmt.Sprintf("DecodeMessage(%v)", tt.args.base64EncodedData))
+		})
+	}
+}
+
+func TestHandshakeOpen_HandleMessage_Success(t *testing.T) {
+	setup(t)
+	testResultEncoded := "AsQWZXhhbXBsZSBoYW5kc2hha2UgZGF0YQMA"
+	testResult, err := base64.StdEncoding.DecodeString(testResultEncoded)
+	assert.NoError(t, err)
+
+	node.EXPECT().Services().Return(services).Times(1)
+	node.EXPECT().NetworkId().Return("").Times(1)
+	services.EXPECT().P2P().Return(p2p).Times(1)
+	p2p.EXPECT().SignMessageSimple(testResult).Return(testResult, nil).Times(1)
+	peer.EXPECT().SendMessage(testResult).Return(nil).Times(1)
+
+	handshake := []byte("example handshake data")
+	handshakeOpen := NewHandshakeOpen([]byte{}, "")
+	handshakeOpen.SetHandshake(handshake)
+
+	assert.NoError(t, handshakeOpen.HandleMessage(node, peer, false))
+}
+func TestHandshakeOpen_HandleMessage_DifferentNetworkID(t *testing.T) {
+	setup(t) // Assuming setup initializes the mocks and any required objects
+
+	// Define a network ID that is different from the one in handshakeOpen
+	differentNetworkID := "differentNetworkID"
+
+	// Setup expectations for the mock objects
+	node.EXPECT().NetworkId().Return(differentNetworkID).Times(1)
+	// No other method calls are expected after the NetworkId check fails
+
+	// Create a HandshakeOpen instance with a specific network ID that differs from `differentNetworkID`
+	networkIDForHandshakeOpen := "expectedNetworkID"
+	handshakeOpen := NewHandshakeOpen([]byte{}, networkIDForHandshakeOpen)
+	handshakeOpen.SetHandshake([]byte("example handshake data"))
+
+	// Invoke HandleMessage and expect an error
+	err := handshakeOpen.HandleMessage(node, peer, false)
+	assert.Error(t, err)
+
+	// Optionally, assert that the error message is as expected
+	expectedErrorMessage := fmt.Sprintf("Peer is in different network: %s", networkIDForHandshakeOpen)
+	assert.Equal(t, expectedErrorMessage, err.Error())
+}
+
+func TestHandshakeOpen_HandleMessage_MarshalError(t *testing.T) {
+	setup(t)
+
+	node.EXPECT().Services().Return(services).Times(1)
+	node.EXPECT().NetworkId().Return("").Times(1)
+	services.EXPECT().P2P().Return(p2p).Times(1)
+	p2p.EXPECT().SignMessageSimple(gomock.Any()).Return(nil, fmt.Errorf("marshal error")).Times(1)
+
+	handshake := []byte("example handshake data")
+	handshakeOpen := NewHandshakeOpen([]byte{}, "")
+	handshakeOpen.SetHandshake(handshake)
+
+	assert.Error(t, handshakeOpen.HandleMessage(node, peer, false))
+}
+
+func TestHandshakeOpen_HandleMessage_SendMessageError(t *testing.T) {
+	setup(t)
+
+	node.EXPECT().Services().Return(services).Times(1)
+	node.EXPECT().NetworkId().Return("").Times(1)
+	services.EXPECT().P2P().Return(p2p).Times(1)
+	p2p.EXPECT().SignMessageSimple(gomock.Any()).Return([]byte{}, nil).Times(1)
+	peer.EXPECT().SendMessage(gomock.Any()).Return(fmt.Errorf("send message error")).Times(1)
+
+	handshake := []byte("example handshake data")
+	handshakeOpen := NewHandshakeOpen([]byte{}, "")
+	handshakeOpen.SetHandshake(handshake)
+
+	assert.Error(t, handshakeOpen.HandleMessage(node, peer, false))
+}

protocol/hash_query.go

@@ -0,0 +1,181 @@
+package protocol
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/net"
+	"git.lumeweb.com/LumeWeb/libs5-go/storage"
+	"git.lumeweb.com/LumeWeb/libs5-go/structs"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/vmihailenco/msgpack/v5"
+	"go.uber.org/zap"
+	"log"
+)
+
+var _ EncodeableMessage = (*HashQuery)(nil)
+var _ IncomingMessage = (*HashQuery)(nil)
+
+type HashQuery struct {
+	hash  *encoding.Multihash
+	kinds []types.StorageLocationType
+	HandshakeRequirement
+}
+
+func NewHashQuery() *HashQuery {
+	hq := &HashQuery{}
+
+	hq.SetRequiresHandshake(true)
+
+	return hq
+}
+
+func NewHashRequest(hash *encoding.Multihash, kinds []types.StorageLocationType) *HashQuery {
+	if len(kinds) == 0 {
+		kinds = []types.StorageLocationType{types.StorageLocationTypeFile}
+	}
+	return &HashQuery{
+		hash:  hash,
+		kinds: kinds,
+	}
+}
+
+func (h HashQuery) Hash() *encoding.Multihash {
+	return h.hash
+}
+
+func (h HashQuery) Kinds() []types.StorageLocationType {
+	return h.kinds
+}
+
+func (h *HashQuery) DecodeMessage(dec *msgpack.Decoder, message IncomingMessageData) error {
+	hash, err := dec.DecodeBytes()
+
+	if err != nil {
+		return err
+	}
+
+	h.hash = encoding.NewMultihash(hash)
+
+	var kinds []types.StorageLocationType
+	err = dec.Decode(&kinds)
+	if err != nil {
+		return err
+	}
+
+	h.kinds = kinds
+
+	return nil
+}
+
+func (h HashQuery) EncodeMsgpack(enc *msgpack.Encoder) error {
+	err := enc.EncodeInt(int64(types.ProtocolMethodHashQuery))
+	if err != nil {
+		return err
+	}
+
+	err = enc.EncodeBytes(h.hash.FullBytes())
+
+	if err != nil {
+		return err
+	}
+
+	err = enc.Encode(h.kinds)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (h *HashQuery) HandleMessage(message IncomingMessageData) error {
+	peer := message.Peer
+	mediator := message.Mediator
+	logger := message.Logger
+	config := message.Config
+
+	mapLocations, err := mediator.GetCachedStorageLocations(h.hash, h.kinds)
+	if err != nil {
+		log.Printf("Error getting cached storage locations: %v", err)
+		return err
+	}
+
+	if len(mapLocations) > 0 {
+		availableNodes := make([]*encoding.NodeId, 0, len(mapLocations))
+		for key := range mapLocations {
+			nodeId, err := encoding.DecodeNodeId(key)
+			if err != nil {
+				logger.Error("Error decoding node id", zap.Error(err))
+				continue
+			}
+
+			availableNodes = append(availableNodes, nodeId)
+		}
+
+		score, err := mediator.SortNodesByScore(availableNodes)
+		if err != nil {
+			return err
+		}
+
+		sortedNodeId, err := (*score[0]).ToString()
+		if err != nil {
+			return err
+		}
+
+		entry, exists := mapLocations[sortedNodeId]
+		if exists {
+			err := peer.SendMessage(entry.ProviderMessage())
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	if mediator.ProviderStore() != nil {
+		if mediator.ProviderStore().CanProvide(h.hash, h.kinds) {
+			location, err := mediator.ProviderStore().Provide(h.hash, h.kinds)
+			if err != nil {
+				return err
+			}
+
+			message := storage.PrepareProvideMessage(config.KeyPair, h.hash, location)
+
+			err = peer.SendMessage(message)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	var peers *structs.SetImpl
+	hashString, err := h.hash.ToString()
+	logger.Debug("HashQuery", zap.Any("hashString", hashString))
+	if err != nil {
+		return err
+	}
+	peersVal, ok := mediator.HashQueryRoutingTable().Get(hashString)
+	if ok {
+		peers = peersVal.(*structs.SetImpl)
+		if !peers.Contains(peer.Id()) {
+			peers.Add(peer.Id())
+		}
+
+		return nil
+	}
+
+	peerList := structs.NewSet()
+	peerList.Add(peer.Id())
+
+	mediator.HashQueryRoutingTable().Put(hashString, peerList)
+
+	for _, val := range mediator.Peers().Values() {
+		peerVal := val.(net.Peer)
+		if !peerVal.Id().Equals(peer.Id()) {
+			err := peerVal.SendMessage(message.Original)
+			if err != nil {
+				logger.Error("Failed to send message", zap.Error(err))
+			}
+		}
+	}
+
+	return nil
+}

protocol/mediator.go

@@ -0,0 +1,29 @@
+package protocol
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/net"
+	"git.lumeweb.com/LumeWeb/libs5-go/storage"
+	"git.lumeweb.com/LumeWeb/libs5-go/structs"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"net/url"
+)
+
+type Mediator interface {
+	NetworkId() string
+	NodeId() *encoding.NodeId
+	SelfConnectionUris() []*url.URL
+	SignMessageSimple(message []byte) ([]byte, error)
+	GetCachedStorageLocations(hash *encoding.Multihash, kinds []types.StorageLocationType) (map[string]storage.StorageLocation, error)
+	SortNodesByScore(nodes []*encoding.NodeId) ([]*encoding.NodeId, error)
+	ProviderStore() storage.ProviderStore
+	AddStorageLocation(hash *encoding.Multihash, nodeId *encoding.NodeId, location storage.StorageLocation, message []byte) error
+	HashQueryRoutingTable() structs.Map
+	Peers() structs.Map
+	RegistrySet(sre SignedRegistryEntry, trusted bool, receivedFrom net.Peer) error
+	RegistryGet(pk []byte) (SignedRegistryEntry, error)
+	ConnectToNode(connectionUris []*url.URL, retried bool, fromPeer net.Peer) error
+	ServicesStarted() bool
+	AddPeer(peer net.Peer) error
+	SendPublicPeersToPeer(peer net.Peer, peersToSend []net.Peer) error
+}

protocol/message.go

@@ -0,0 +1,50 @@
+package protocol
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+)
+
+var (
+	messageTypes map[int]func() IncomingMessage
+)
+
+func RegisterProtocols() {
+	messageTypes = make(map[int]func() IncomingMessage)
+
+	// Register factory functions instead of instances
+	RegisterMessageType(int(types.ProtocolMethodHandshakeOpen), func() IncomingMessage {
+		return NewHandshakeOpen([]byte{}, "")
+	})
+	RegisterMessageType(int(types.ProtocolMethodHashQuery), func() IncomingMessage {
+		return NewHashQuery()
+	})
+	RegisterMessageType(int(types.RecordTypeStorageLocation), func() IncomingMessage {
+		return NewStorageLocation()
+	})
+	RegisterMessageType(int(types.RecordTypeRegistryEntry), func() IncomingMessage {
+		return NewEmptyRegistryEntryRequest()
+	})
+	RegisterMessageType(int(types.ProtocolMethodRegistryQuery), func() IncomingMessage {
+		return NewEmptyRegistryQuery()
+	})
+	RegisterMessageType(int(types.ProtocolMethodSignedMessage), func() IncomingMessage {
+		return NewSignedMessage()
+	})
+
+}
+
+func RegisterMessageType(messageType int, factoryFunc func() IncomingMessage) {
+	if factoryFunc == nil {
+		panic("factoryFunc cannot be nil")
+	}
+	messageTypes[messageType] = factoryFunc
+}
+
+func GetMessageType(kind int) (IncomingMessage, bool) {
+	value, ok := messageTypes[kind]
+	if !ok {
+		return nil, false
+	}
+
+	return value(), true
+}

protocol/protocol.go

@@ -0,0 +1,99 @@
+package protocol
+
+import (
+	"context"
+	"crypto/rand"
+	"git.lumeweb.com/LumeWeb/libs5-go/config"
+	"git.lumeweb.com/LumeWeb/libs5-go/net"
+	"github.com/vmihailenco/msgpack/v5"
+	"go.uber.org/zap"
+	"io"
+	"math"
+)
+
+func GenerateChallenge() []byte {
+	challenge := make([]byte, 32)
+	_, err := rand.Read(challenge)
+	if err != nil {
+		panic(err)
+	}
+
+	return challenge
+}
+
+func CalculateNodeScore(goodResponses, badResponses int) float64 {
+	totalVotes := goodResponses + badResponses
+	if totalVotes == 0 {
+		return 0.5
+	}
+
+	average := float64(goodResponses) / float64(totalVotes)
+	score := average - (average-0.5)*math.Pow(2, -math.Log(float64(totalVotes+1)))
+
+	return score
+}
+
+var (
+	_ msgpack.CustomDecoder = (*IncomingMessageReader)(nil)
+)
+
+type IncomingMessage interface {
+	HandleMessage(message IncomingMessageData) error
+	DecodeMessage(dec *msgpack.Decoder, message IncomingMessageData) error
+	HandshakeRequirer
+}
+type EncodeableMessage interface {
+	msgpack.CustomEncoder
+}
+
+type IncomingMessageData struct {
+	Original []byte
+	Data     []byte
+	Ctx      context.Context
+	Logger   *zap.Logger
+	Peer     net.Peer
+	Config   *config.NodeConfig
+	VerifyId bool
+	Mediator Mediator
+}
+
+type IncomingMessageReader struct {
+	Kind int
+	Data []byte
+}
+
+func (i *IncomingMessageReader) DecodeMsgpack(dec *msgpack.Decoder) error {
+	kind, err := dec.DecodeInt()
+	if err != nil {
+		return err
+	}
+
+	i.Kind = kind
+
+	raw, err := io.ReadAll(dec.Buffered())
+
+	if err != nil {
+		return err
+	}
+
+	i.Data = raw
+
+	return nil
+}
+
+type HandshakeRequirer interface {
+	RequiresHandshake() bool
+	SetRequiresHandshake(value bool)
+}
+
+type HandshakeRequirement struct {
+	requiresHandshake bool
+}
+
+func (hr *HandshakeRequirement) RequiresHandshake() bool {
+	return hr.requiresHandshake
+}
+
+func (hr *HandshakeRequirement) SetRequiresHandshake(value bool) {
+	hr.requiresHandshake = value
+}

protocol/protocol_test.go

@@ -0,0 +1,38 @@
+package protocol
+
+import (
+	"github.com/golang/mock/gomock"
+	"os"
+	"testing"
+)
+
+// Common resources
+var (
+	mockCtrl *gomock.Controller
+	node     *interfaces.MockNode
+	peer     *net.MockPeer
+	services *interfaces.MockServices
+	p2p      *interfaces.MockP2PService
+)
+
+// Setup function
+func setup(t *testing.T) {
+	mockCtrl = gomock.NewController(t)
+	node = interfaces.NewMockNode(mockCtrl)
+	peer = net.NewMockPeer(mockCtrl)
+	services = interfaces.NewMockServices(mockCtrl)
+	p2p = interfaces.NewMockP2PService(mockCtrl)
+}
+
+// Teardown function
+func teardown() {
+	mockCtrl.Finish()
+	// Other cleanup tasks
+}
+
+// TestMain function for setup and teardown
+func TestMain(m *testing.M) {
+	code := m.Run()
+	teardown()
+	os.Exit(code)
+}

protocol/registry.go

@@ -0,0 +1,163 @@
+package protocol
+
+import (
+	ed25519p "crypto/ed25519"
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/ed25519"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"git.lumeweb.com/LumeWeb/libs5-go/utils"
+)
+
+var (
+	_ SignedRegistryEntry = (*SignedRegistryEntryImpl)(nil)
+	_ SignedRegistryEntry = (*SignedRegistryEntryImpl)(nil)
+)
+
+type SignedRegistryEntry interface {
+	PK() []byte
+	Revision() uint64
+	Data() []byte
+	Signature() []byte
+	SetPK(pk []byte)
+	SetRevision(revision uint64)
+	SetData(data []byte)
+	SetSignature(signature []byte)
+	Verify() bool
+}
+
+type RegistryEntry interface {
+	Sign() SignedRegistryEntry
+}
+
+type SignedRegistryEntryImpl struct {
+	pk        []byte
+	revision  uint64
+	data      []byte
+	signature []byte
+}
+
+func (s *SignedRegistryEntryImpl) Verify() bool {
+	return VerifyRegistryEntry(s)
+}
+
+func (s *SignedRegistryEntryImpl) PK() []byte {
+	return s.pk
+}
+
+func (s *SignedRegistryEntryImpl) SetPK(pk []byte) {
+	s.pk = pk
+}
+
+func (s *SignedRegistryEntryImpl) Revision() uint64 {
+	return s.revision
+}
+
+func (s *SignedRegistryEntryImpl) SetRevision(revision uint64) {
+	s.revision = revision
+}
+
+func (s *SignedRegistryEntryImpl) Data() []byte {
+	return s.data
+}
+
+func (s *SignedRegistryEntryImpl) SetData(data []byte) {
+	s.data = data
+}
+
+func (s *SignedRegistryEntryImpl) Signature() []byte {
+	return s.signature
+}
+
+func (s *SignedRegistryEntryImpl) SetSignature(signature []byte) {
+	s.signature = signature
+}
+
+func NewSignedRegistryEntry(pk []byte, revision uint64, data []byte, signature []byte) SignedRegistryEntry {
+	return &SignedRegistryEntryImpl{
+		pk:        pk,
+		revision:  revision,
+		data:      data,
+		signature: signature,
+	}
+}
+
+type RegistryEntryImpl struct {
+	kp       ed25519.KeyPairEd25519
+	data     []byte
+	revision uint64
+}
+
+func NewRegistryEntry(kp ed25519.KeyPairEd25519, data []byte, revision uint64) RegistryEntry {
+	return &RegistryEntryImpl{
+		kp:       kp,
+		data:     data,
+		revision: revision,
+	}
+}
+
+func (r *RegistryEntryImpl) Sign() SignedRegistryEntry {
+	return SignRegistryEntry(r.kp, r.data, r.revision)
+}
+
+func SignRegistryEntry(kp ed25519.KeyPairEd25519, data []byte, revision uint64) SignedRegistryEntry {
+	buffer := MarshalRegistryEntry(nil, data, revision)
+
+	privateKey := kp.ExtractBytes()
+	signature := ed25519p.Sign(privateKey, buffer)
+
+	return NewSignedRegistryEntry(kp.PublicKey(), uint64(revision), data, signature)
+}
+func VerifyRegistryEntry(sre SignedRegistryEntry) bool {
+	buffer := MarshalRegistryEntry(nil, sre.Data(), sre.Revision())
+	publicKey := sre.PK()[1:]
+
+	return ed25519p.Verify(publicKey, buffer, sre.Signature())
+}
+
+func MarshalSignedRegistryEntry(sre SignedRegistryEntry) []byte {
+	buffer := MarshalRegistryEntry(sre.PK(), sre.Data(), sre.Revision())
+	buffer = append(buffer, sre.Signature()...)
+
+	return buffer
+}
+func MarshalRegistryEntry(pk []byte, data []byte, revision uint64) []byte {
+	var buffer []byte
+	buffer = append(buffer, byte(types.RecordTypeRegistryEntry))
+
+	if pk != nil {
+		buffer = append(buffer, pk...)
+	}
+
+	revBytes := utils.EncodeEndian(revision, 8)
+	buffer = append(buffer, revBytes...)
+
+	buffer = append(buffer, byte(len(data)))
+	buffer = append(buffer, data...)
+
+	return buffer
+}
+
+func UnmarshalSignedRegistryEntry(event []byte) (sre SignedRegistryEntry, err error) {
+	if len(event) < 43 {
+		return nil, errors.New("Invalid registry entry")
+	}
+
+	dataLength := int(event[42])
+	if len(event) < 43+dataLength {
+		return nil, errors.New("Invalid registry entry")
+	}
+
+	pk := event[1:34]
+	revisionBytes := event[34:42]
+	revision := utils.DecodeEndian(revisionBytes)
+	signatureStart := 43 + dataLength
+	var signature []byte
+
+	if signatureStart < len(event) {
+		signature = event[signatureStart:]
+	} else {
+		return nil, errors.New("Invalid signature")
+	}
+
+	return NewSignedRegistryEntry(pk, uint64(revision), event[43:signatureStart], signature), nil
+}

protocol/registry_entry.go

@@ -0,0 +1,70 @@
+package protocol
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/internal/bases"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/vmihailenco/msgpack/v5"
+	"go.uber.org/zap"
+)
+
+var _ IncomingMessage = (*RegistryEntryRequest)(nil)
+var _ EncodeableMessage = (*RegistryEntryRequest)(nil)
+
+type RegistryEntryRequest struct {
+	sre SignedRegistryEntry
+	HandshakeRequirement
+}
+
+func NewEmptyRegistryEntryRequest() *RegistryEntryRequest {
+	rer := &RegistryEntryRequest{}
+
+	rer.SetRequiresHandshake(true)
+
+	return rer
+}
+func NewRegistryEntryRequest(sre SignedRegistryEntry) *RegistryEntryRequest {
+	return &RegistryEntryRequest{sre: sre}
+}
+
+func (s *RegistryEntryRequest) EncodeMsgpack(enc *msgpack.Encoder) error {
+	err := enc.EncodeInt(int64(types.RecordTypeRegistryEntry))
+	if err != nil {
+		return err
+	}
+
+	err = enc.EncodeBytes(MarshalSignedRegistryEntry(s.sre))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *RegistryEntryRequest) DecodeMessage(dec *msgpack.Decoder, message IncomingMessageData) error {
+	sre, err := UnmarshalSignedRegistryEntry(message.Original)
+	if err != nil {
+		return err
+	}
+
+	s.sre = sre
+
+	return nil
+}
+
+func (s *RegistryEntryRequest) HandleMessage(message IncomingMessageData) error {
+	entry, err := encoding.CIDFromRegistryPublicKey(s.sre.PK())
+	if err != nil {
+		return err
+	}
+	pid, err := message.Peer.Id().ToString()
+	if err != nil {
+		return err
+	}
+	b64, err := bases.ToBase64Url(s.sre.PK())
+	if err != nil {
+		return err
+	}
+	message.Logger.Debug("Handling registry entry set request", zap.Any("entryCID", entry), zap.Any("entryBase64", b64), zap.Any("peer", pid))
+	return message.Mediator.RegistrySet(s.sre, false, message.Peer)
+}

protocol/registry_query.go

@@ -0,0 +1,86 @@
+package protocol
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/internal/bases"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/vmihailenco/msgpack/v5"
+	"go.uber.org/zap"
+)
+
+var _ IncomingMessage = (*RegistryQuery)(nil)
+var _ EncodeableMessage = (*RegistryQuery)(nil)
+
+type RegistryQuery struct {
+	pk []byte
+	HandshakeRequirement
+}
+
+func NewEmptyRegistryQuery() *RegistryQuery {
+	rq := &RegistryQuery{}
+
+	rq.SetRequiresHandshake(true)
+
+	return rq
+}
+func NewRegistryQuery(pk []byte) *RegistryQuery {
+	return &RegistryQuery{pk: pk}
+}
+
+func (s *RegistryQuery) EncodeMsgpack(enc *msgpack.Encoder) error {
+	err := enc.EncodeInt(int64(types.ProtocolMethodRegistryQuery))
+	if err != nil {
+		return err
+	}
+
+	err = enc.EncodeBytes(s.pk)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *RegistryQuery) DecodeMessage(dec *msgpack.Decoder, message IncomingMessageData) error {
+	pk, err := dec.DecodeBytes()
+	if err != nil {
+		return err
+	}
+
+	s.pk = pk
+
+	return nil
+}
+
+func (s *RegistryQuery) HandleMessage(message IncomingMessageData) error {
+	mediator := message.Mediator
+	peer := message.Peer
+
+	entry, err := encoding.CIDFromRegistryPublicKey(s.pk)
+	if err != nil {
+		return err
+	}
+	pid, err := peer.Id().ToString()
+	if err != nil {
+		return err
+	}
+	b64, err := bases.ToBase64Url(s.pk)
+	if err != nil {
+		return err
+	}
+	message.Logger.Debug("Handling registry entry query request", zap.Any("entryCID", entry), zap.Any("entryBase64", b64), zap.Any("peer", pid))
+
+	sre, err := mediator.RegistryGet(s.pk)
+	if err != nil {
+		return err
+	}
+
+	if sre != nil {
+		err := peer.SendMessage(MarshalSignedRegistryEntry(sre))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

protocol/signed.go

@@ -0,0 +1,49 @@
+package protocol
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+type IncomingMessageDataSigned struct {
+	IncomingMessageData
+	NodeId *encoding.NodeId
+}
+
+type IncomingMessageSigned interface {
+	HandleMessage(message IncomingMessageDataSigned) error
+	DecodeMessage(dec *msgpack.Decoder, message IncomingMessageDataSigned) error
+	HandshakeRequirer
+}
+
+var (
+	signedMessageTypes map[int]func() IncomingMessageSigned
+)
+
+func RegisterSignedProtocols() {
+	signedMessageTypes = make(map[int]func() IncomingMessageSigned)
+
+	RegisterSignedMessageType(int(types.ProtocolMethodHandshakeDone), func() IncomingMessageSigned {
+		return NewHandshakeDone()
+	})
+	RegisterSignedMessageType(int(types.ProtocolMethodAnnouncePeers), func() IncomingMessageSigned {
+		return NewAnnouncePeers()
+	})
+}
+
+func RegisterSignedMessageType(messageType int, factoryFunc func() IncomingMessageSigned) {
+	if factoryFunc == nil {
+		panic("factoryFunc cannot be nil")
+	}
+	signedMessageTypes[messageType] = factoryFunc
+}
+
+func GetSignedMessageType(kind int) (IncomingMessageSigned, bool) {
+	value, ok := signedMessageTypes[kind]
+	if !ok {
+		return nil, false
+	}
+
+	return value(), true
+}

protocol/signed_announce_peers.go

@@ -0,0 +1,161 @@
+package protocol
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/net"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/vmihailenco/msgpack/v5"
+	"net/url"
+)
+
+var (
+	_ IncomingMessageSigned = (*AnnouncePeers)(nil)
+)
+
+type AnnouncePeers struct {
+	peer           net.Peer
+	connectionUris []*url.URL
+	peersToSend    []net.Peer
+	HandshakeRequirement
+}
+
+func (a *AnnouncePeers) PeersToSend() []net.Peer {
+	return a.peersToSend
+}
+
+func (a *AnnouncePeers) SetPeersToSend(peersToSend []net.Peer) {
+	a.peersToSend = peersToSend
+}
+
+func NewAnnounceRequest(peer net.Peer, peersToSend []net.Peer) *AnnouncePeers {
+	return &AnnouncePeers{peer: peer, connectionUris: nil, peersToSend: peersToSend}
+}
+
+func NewAnnouncePeers() *AnnouncePeers {
+	ap := &AnnouncePeers{peer: nil, connectionUris: nil}
+
+	ap.SetRequiresHandshake(true)
+
+	return ap
+}
+
+func (a *AnnouncePeers) DecodeMessage(dec *msgpack.Decoder, message IncomingMessageDataSigned) error {
+	// CIDFromString the number of peers.
+	numPeers, err := dec.DecodeInt()
+	if err != nil {
+		return err
+	}
+
+	// Initialize the slice for storing connection URIs.
+	var connectionURIs []*url.URL
+
+	// Loop through each peer.
+	for i := 0; i < numPeers; i++ {
+		// CIDFromString peer ID.
+		peerIdBytes, err := dec.DecodeBytes()
+		if err != nil {
+			return err
+		}
+		peerId := encoding.NewNodeId(peerIdBytes)
+
+		// Skip decoding connection status as it is not used.
+		_, err = dec.DecodeBool() // Connection status, not used.
+		if err != nil {
+			return err
+		}
+
+		// CIDFromString the number of connection URIs for this peer.
+		numUris, err := dec.DecodeInt()
+		if err != nil {
+			return err
+		}
+
+		// CIDFromString each connection URI for this peer.
+		for j := 0; j < numUris; j++ {
+			uriStr, err := dec.DecodeString()
+			if err != nil {
+				return err
+			}
+
+			uri, err := url.Parse(uriStr)
+			if err != nil {
+				return err
+			}
+
+			pid, err := peerId.ToString()
+			if err != nil {
+				return err
+			}
+
+			passwd, empty := uri.User.Password()
+			if empty {
+				passwd = ""
+			}
+
+			// Incorporate the peer ID into the URI.
+			uri.User = url.UserPassword(pid, passwd)
+
+			connectionURIs = append(connectionURIs, uri)
+		}
+	}
+
+	a.connectionUris = connectionURIs
+
+	return nil
+}
+
+func (a AnnouncePeers) HandleMessage(message IncomingMessageDataSigned) error {
+	mediator := message.Mediator
+	peer := message.Peer
+	if len(a.connectionUris) > 0 {
+		err := mediator.ConnectToNode([]*url.URL{a.connectionUris[0]}, false, peer)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (a AnnouncePeers) EncodeMsgpack(enc *msgpack.Encoder) error {
+	err := enc.EncodeUint(uint64(types.ProtocolMethodAnnouncePeers))
+	if err != nil {
+		return err
+	}
+
+	// Encode the number of peers.
+	err = enc.EncodeInt(int64(len(a.peersToSend)))
+	if err != nil {
+		return err
+	}
+
+	// Loop through each peer.
+	for _, peer := range a.peersToSend {
+		err = enc.EncodeBytes(peer.Id().Raw())
+		if err != nil {
+			return err
+		}
+
+		// Encode connection status.
+		err = enc.EncodeBool(peer.IsConnected())
+		if err != nil {
+			return err
+		}
+
+		// Encode the number of connection URIs for this peer.
+		err = enc.EncodeInt(int64(len(peer.ConnectionURIs())))
+		if err != nil {
+			return err
+		}
+
+		// Encode each connection URI for this peer.
+		for _, uri := range peer.ConnectionURIs() {
+			err = enc.EncodeString(uri.String())
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

protocol/signed_handshake_done.go

@@ -0,0 +1,165 @@
+package protocol
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"git.lumeweb.com/LumeWeb/libs5-go/utils"
+	"github.com/vmihailenco/msgpack/v5"
+	"net/url"
+)
+
+var _ IncomingMessageSigned = (*HandshakeDone)(nil)
+var _ EncodeableMessage = (*HandshakeDone)(nil)
+
+type HandshakeDone struct {
+	challenge         []byte
+	networkId         string
+	supportedFeatures int
+	connectionUris    []*url.URL
+	handshake         []byte
+	HandshakeRequirement
+}
+
+func NewHandshakeDoneRequest(handshake []byte, supportedFeatures int, connectionUris []*url.URL) *HandshakeDone {
+	ho := &HandshakeDone{
+		handshake:         handshake,
+		supportedFeatures: supportedFeatures,
+		connectionUris:    connectionUris,
+	}
+
+	ho.SetRequiresHandshake(false)
+
+	return ho
+}
+
+func (m HandshakeDone) EncodeMsgpack(enc *msgpack.Encoder) error {
+	err := enc.EncodeUint(uint64(types.ProtocolMethodHandshakeDone))
+	if err != nil {
+		return err
+	}
+
+	err = enc.EncodeBytes(m.handshake)
+	if err != nil {
+		return err
+	}
+
+	err = enc.EncodeInt(int64(m.supportedFeatures))
+	if err != nil {
+		return err
+	}
+
+	err = utils.EncodeMsgpackArray(enc, m.connectionUris)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *HandshakeDone) SetChallenge(challenge []byte) {
+	m.challenge = challenge
+}
+
+func (m *HandshakeDone) SetNetworkId(networkId string) {
+	m.networkId = networkId
+}
+
+func NewHandshakeDone() *HandshakeDone {
+	hn := &HandshakeDone{challenge: nil, networkId: "", supportedFeatures: -1}
+
+	hn.SetRequiresHandshake(false)
+
+	return hn
+}
+
+func (h HandshakeDone) HandleMessage(message IncomingMessageDataSigned) error {
+	mediator := message.Mediator
+	peer := message.Peer
+	verifyId := message.VerifyId
+	nodeId := message.NodeId
+	logger := message.Logger
+
+	if !mediator.ServicesStarted() {
+		err := peer.End()
+		if err != nil {
+			return nil
+		}
+		return nil
+	}
+
+	if !bytes.Equal(peer.Challenge(), h.challenge) {
+		return errors.New("Invalid challenge")
+	}
+
+	if !verifyId {
+		peer.SetId(nodeId)
+	} else {
+		if !peer.Id().Equals(nodeId) {
+			return fmt.Errorf("Invalid transports id on initial list")
+		}
+	}
+
+	peer.SetConnected(true)
+	peer.SetHandshakeDone(true)
+
+	if h.supportedFeatures != types.SupportedFeatures {
+		return fmt.Errorf("Remote node does not support required features")
+	}
+	err := mediator.AddPeer(peer)
+	if err != nil {
+		return err
+	}
+
+	if len(h.connectionUris) == 0 {
+		return nil
+	}
+
+	peerId, err := peer.Id().ToString()
+
+	if err != nil {
+		return err
+	}
+
+	for _, uri := range h.connectionUris {
+		uri.User = url.User(peerId)
+	}
+
+	peer.SetConnectionURIs(h.connectionUris)
+
+	logger.Info(fmt.Sprintf("[+] %s (%s)", peerId, peer.RenderLocationURI()))
+
+	err = mediator.ConnectToNode([]*url.URL{h.connectionUris[0]}, false, peer)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (h *HandshakeDone) DecodeMessage(dec *msgpack.Decoder, message IncomingMessageDataSigned) error {
+	challenge, err := dec.DecodeBytes()
+	if err != nil {
+		return err
+	}
+
+	h.challenge = challenge
+
+	supportedFeatures, err := dec.DecodeInt()
+
+	if err != nil {
+		return err
+	}
+
+	h.supportedFeatures = supportedFeatures
+
+	connectionUris, err := utils.DecodeMsgpackURLArray(dec)
+
+	if err != nil {
+		return err
+	}
+
+	h.connectionUris = connectionUris
+	return nil
+}

protocol/signed_handshake_done_test.go

@@ -0,0 +1,134 @@
+package protocol
+
+import (
+	"bytes"
+	"encoding/base64"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/vmihailenco/msgpack/v5"
+	"net/url"
+	"testing"
+)
+
+func TestHandshakeDone_EncodeMsgpack(t *testing.T) {
+	type fields struct {
+		supportedFeatures int
+		connectionUris    []*url.URL
+		handshake         []byte
+	}
+	tests := []struct {
+		name        string
+		fields      fields
+		wantErr     bool
+		wantErrFunc assert.ErrorAssertionFunc
+	}{
+		{
+			name: "Empty Fields",
+			fields: fields{
+				supportedFeatures: 0,
+				connectionUris:    []*url.URL{},
+				handshake:         []byte{},
+			},
+			wantErr: false,
+		},
+		{
+			name: "Valid Fields",
+			fields: fields{
+				supportedFeatures: 1,
+				connectionUris:    []*url.URL{{ /* initialize with valid URL data */ }},
+				handshake:         []byte{0x01, 0x02},
+			},
+			wantErr: false,
+		},
+		{
+			name: "Invalid URL",
+			fields: fields{
+				supportedFeatures: 1,
+				connectionUris:    []*url.URL{&url.URL{Host: "invalid-url"}},
+				handshake:         []byte{0x01},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := HandshakeDone{
+				supportedFeatures: tt.fields.supportedFeatures,
+				connectionUris:    tt.fields.connectionUris,
+				handshake:         tt.fields.handshake,
+			}
+
+			if tt.wantErr {
+				tt.wantErrFunc = assert.Error
+			} else {
+				tt.wantErrFunc = assert.NoError
+			}
+
+			enc := msgpack.NewEncoder(new(bytes.Buffer))
+			err := m.EncodeMsgpack(enc)
+
+			assert.NoError(t, err)
+
+			encodedData := enc.Writer().(*bytes.Buffer).Bytes()
+
+			if len(encodedData) == 0 && tt.wantErr {
+				t.Errorf("Expected non-empty encoded data, got empty")
+			}
+
+			dec := msgpack.NewDecoder(bytes.NewReader(encodedData))
+			protocol, err := dec.DecodeUint()
+
+			assert.EqualValues(t, protocol, types.ProtocolMethodHandshakeDone)
+
+			handshake, err := dec.DecodeBytes()
+
+			assert.EqualValues(t, handshake, tt.fields.handshake)
+
+			supportedFeatures, err := dec.DecodeInt()
+
+			assert.EqualValues(t, supportedFeatures, tt.fields.supportedFeatures)
+		})
+	}
+}
+
+func TestHandshakeDone_DecodeMessage_Success(t *testing.T) {
+	data := "xBFleGFtcGxlX2NoYWxsZW5nZQM="
+
+	h := HandshakeDone{}
+
+	dataDec, err := base64.StdEncoding.DecodeString(data)
+	assert.NoError(t, err)
+
+	enc := msgpack.NewDecoder(bytes.NewReader(dataDec))
+	err = h.DecodeMessage(enc)
+	assert.NoError(t, err)
+
+	assert.EqualValues(t, types.SupportedFeatures, h.supportedFeatures)
+	assert.EqualValues(t, []byte("example_challenge"), h.challenge)
+}
+func TestHandshakeDone_DecodeMessage_InvalidFeatures(t *testing.T) {
+	data := "xBFleGFtcGxlX2NoYWxsZW5nZSo="
+
+	h := HandshakeDone{}
+
+	dataDec, err := base64.StdEncoding.DecodeString(data)
+	assert.NoError(t, err)
+
+	enc := msgpack.NewDecoder(bytes.NewReader(dataDec))
+	err = h.DecodeMessage(enc)
+	assert.NotEqualValues(t, types.SupportedFeatures, h.supportedFeatures)
+	assert.EqualValues(t, []byte("example_challenge"), h.challenge)
+}
+func TestHandshakeDone_DecodeMessage_BadChallenge(t *testing.T) {
+	data := "xA1iYWRfY2hhbGxlbmdlAw=="
+
+	h := HandshakeDone{}
+
+	dataDec, err := base64.StdEncoding.DecodeString(data)
+	assert.NoError(t, err)
+
+	enc := msgpack.NewDecoder(bytes.NewReader(dataDec))
+	err = h.DecodeMessage(enc)
+	assert.EqualValues(t, types.SupportedFeatures, h.supportedFeatures)
+	assert.NotEqualValues(t, []byte("example_challenge"), h.challenge)
+}

protocol/signed_message.go

@@ -0,0 +1,194 @@
+package protocol
+
+import (
+	"bytes"
+	"crypto/ed25519"
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/config"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/vmihailenco/msgpack/v5"
+	"go.uber.org/zap"
+	"io"
+)
+
+var (
+	_ IncomingMessage       = (*SignedMessage)(nil)
+	_ msgpack.CustomDecoder = (*signedMessageReader)(nil)
+	_ msgpack.CustomEncoder = (*SignedMessage)(nil)
+)
+
+var (
+	errInvalidSignature = errors.New("Invalid signature found")
+)
+
+type SignedMessage struct {
+	nodeId    *encoding.NodeId
+	signature []byte
+	message   []byte
+	HandshakeRequirement
+}
+
+func (s *SignedMessage) NodeId() *encoding.NodeId {
+	return s.nodeId
+}
+
+func (s *SignedMessage) SetNodeId(nodeId *encoding.NodeId) {
+	s.nodeId = nodeId
+}
+
+func (s *SignedMessage) SetSignature(signature []byte) {
+	s.signature = signature
+}
+
+func (s *SignedMessage) SetMessage(message []byte) {
+	s.message = message
+}
+
+func NewSignedMessageRequest(message []byte) *SignedMessage {
+	return &SignedMessage{message: message}
+}
+
+type signedMessageReader struct {
+	kind    int
+	message msgpack.RawMessage
+}
+
+func (s *signedMessageReader) DecodeMsgpack(dec *msgpack.Decoder) error {
+	kind, err := dec.DecodeInt()
+	if err != nil {
+		return err
+	}
+
+	s.kind = kind
+
+	message, err := io.ReadAll(dec.Buffered())
+
+	if err != nil {
+		return err
+	}
+
+	s.message = message
+
+	return nil
+}
+
+func NewSignedMessage() *SignedMessage {
+	sm := &SignedMessage{}
+
+	sm.SetRequiresHandshake(false)
+
+	return sm
+}
+
+func (s *SignedMessage) HandleMessage(message IncomingMessageData) error {
+	var payload signedMessageReader
+	peer := message.Peer
+	logger := message.Logger
+
+	err := msgpack.Unmarshal(s.message, &payload)
+	if err != nil {
+		return err
+	}
+
+	if msgHandler, valid := GetSignedMessageType(payload.kind); valid {
+		logger.Debug("SignedMessage", zap.Any("type", types.ProtocolMethodMap[types.ProtocolMethod(payload.kind)]))
+		if msgHandler.RequiresHandshake() && !peer.IsHandshakeDone() {
+			nid, err := s.nodeId.ToString()
+			if err != nil {
+				return err
+			}
+			logger.Debug("Peer is not handshake done, ignoring message", zap.Any("type", types.ProtocolMethodMap[types.ProtocolMethod(payload.kind)]), zap.String("peerIP", peer.GetIPString()), zap.String("nodeId", nid))
+			return nil
+		}
+
+		signedDec := msgpack.NewDecoder(bytes.NewReader(payload.message))
+		signedMsg := IncomingMessageDataSigned{
+			IncomingMessageData: message,
+			NodeId:              s.nodeId,
+		}
+
+		err = msgHandler.DecodeMessage(signedDec, signedMsg)
+
+		if err != nil {
+			logger.Error("Error decoding signed message", zap.Error(err))
+			return err
+		}
+
+		if err = msgHandler.HandleMessage(signedMsg); err != nil {
+			logger.Error("Error handling signed message", zap.Error(err))
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *SignedMessage) DecodeMessage(dec *msgpack.Decoder, message IncomingMessageData) error {
+	nodeId, err := dec.DecodeBytes()
+	if err != nil {
+		return err
+	}
+
+	s.nodeId = encoding.NewNodeId(nodeId)
+
+	signature, err := dec.DecodeBytes()
+	if err != nil {
+		return err
+	}
+
+	s.signature = signature
+
+	signedMessage, err := dec.DecodeBytes()
+	if err != nil {
+		return err
+	}
+
+	s.message = signedMessage
+
+	if !ed25519.Verify(s.nodeId.Raw()[1:], s.message, s.signature) {
+		return errInvalidSignature
+	}
+
+	return nil
+}
+func (s *SignedMessage) EncodeMsgpack(enc *msgpack.Encoder) error {
+	err := enc.EncodeInt(int64(types.ProtocolMethodSignedMessage))
+
+	if err != nil {
+		return err
+	}
+
+	err = enc.EncodeBytes(s.nodeId.Raw())
+
+	if err != nil {
+		return err
+	}
+
+	err = enc.EncodeBytes(s.signature)
+
+	if err != nil {
+		return err
+	}
+
+	err = enc.EncodeBytes(s.message)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+func (s *SignedMessage) Sign(cfg *config.NodeConfig) error {
+	if s.nodeId == nil {
+		panic("nodeId is nil")
+	}
+
+	if s.message == nil {
+		panic("message is nil")
+	}
+
+	s.signature = ed25519.Sign(cfg.KeyPair.ExtractBytes(), s.message)
+
+	return nil
+}

protocol/storage_location.go

@@ -0,0 +1,124 @@
+package protocol
+
+import (
+	"crypto/ed25519"
+	"fmt"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/net"
+	"git.lumeweb.com/LumeWeb/libs5-go/storage"
+	"git.lumeweb.com/LumeWeb/libs5-go/structs"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"git.lumeweb.com/LumeWeb/libs5-go/utils"
+	"github.com/vmihailenco/msgpack/v5"
+	"go.uber.org/zap"
+)
+
+var _ IncomingMessage = (*StorageLocation)(nil)
+
+type StorageLocation struct {
+	hash      *encoding.Multihash
+	kind      int
+	expiry    int64
+	parts     []string
+	publicKey []byte
+	signature []byte
+	HandshakeRequirement
+}
+
+func NewStorageLocation() *StorageLocation {
+	sl := &StorageLocation{}
+
+	sl.SetRequiresHandshake(true)
+
+	return sl
+}
+
+func (s *StorageLocation) DecodeMessage(dec *msgpack.Decoder, message IncomingMessageData) error {
+	// nop, we use the incoming message -> original already stored
+	return nil
+}
+func (s *StorageLocation) HandleMessage(message IncomingMessageData) error {
+	msg := message.Original
+	mediator := message.Mediator
+	peer := message.Peer
+	logger := message.Logger
+
+	hash := encoding.NewMultihash(msg[1:34]) // Replace NewMultihash with appropriate function
+
+	typeOfData := msg[34]
+
+	expiry := utils.DecodeEndian(msg[35:39])
+
+	partCount := msg[39]
+
+	parts := []string{}
+	cursor := 40
+	for i := 0; i < int(partCount); i++ {
+		length := utils.DecodeEndian(msg[cursor : cursor+2])
+		cursor += 2
+		if len(msg) < cursor+int(length) {
+			return fmt.Errorf("Invalid message")
+		}
+		part := string(msg[cursor : cursor+int(length)])
+		parts = append(parts, part)
+		cursor += int(length)
+	}
+
+	cursor++
+
+	publicKey := msg[cursor : cursor+33]
+	signature := msg[cursor+33:]
+
+	if types.HashType(publicKey[0]) != types.HashTypeEd25519 { // Replace CID_HASH_TYPES_ED25519 with actual constant
+		return fmt.Errorf("Unsupported public key type %d", publicKey[0])
+	}
+
+	if !ed25519.Verify(publicKey[1:], msg[:cursor], signature) {
+		return fmt.Errorf("Signature verification failed")
+	}
+
+	nodeId := encoding.NewNodeId(publicKey)
+
+	err := mediator.AddStorageLocation(hash, nodeId, storage.NewStorageLocation(int(typeOfData), parts, int64(expiry)), msg)
+	if err != nil {
+		return fmt.Errorf("Failed to add storage location: %s", err)
+	}
+
+	hashStr, err := hash.ToString()
+	if err != nil {
+		return err
+	}
+
+	var list *structs.SetImpl
+	listVal, ok := mediator.HashQueryRoutingTable().Get(hashStr) // Implement HashQueryRoutingTable method
+	if !ok {
+		list = structs.NewSet()
+	} else {
+		list = listVal.(*structs.SetImpl)
+	}
+
+	for _, peerIdVal := range list.Values() {
+		peerId := peerIdVal.(*encoding.NodeId)
+
+		if peerId.Equals(nodeId) || peerId.Equals(peer) {
+			continue
+		}
+		peerIdStr, err := peerId.ToString()
+		if err != nil {
+			return err
+		}
+
+		if peerVal, ok := mediator.Peers().Get(peerIdStr); ok {
+			foundPeer := peerVal.(net.Peer)
+			err := foundPeer.SendMessage(msg)
+			if err != nil {
+				logger.Error("Failed to send message", zap.Error(err))
+				continue
+			}
+		}
+
+		mediator.HashQueryRoutingTable().Remove(hashStr)
+	}
+
+	return nil
+}

serialize/seralizer.go

@@ -0,0 +1,48 @@
+package serialize
+
+import (
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/vmihailenco/msgpack/v5"
+	"slices"
+)
+
+func InitMarshaller(enc *msgpack.Encoder, kind types.MetadataType) error {
+	err := enc.EncodeInt(types.MetadataMagicByte)
+	if err != nil {
+		return err
+	}
+	err = enc.EncodeInt(int64(kind))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func InitUnmarshaller(enc *msgpack.Decoder, kinds ...types.MetadataType) (types.MetadataType, error) {
+	val, err := enc.DecodeUint8()
+	if err != nil {
+		return 0, err
+	}
+
+	if val != types.MetadataMagicByte {
+		return 0, errors.New("Invalid magic byte")
+	}
+
+	val, err = enc.DecodeUint8()
+	if err != nil {
+		return 0, err
+	}
+
+	convertedKinds := make([]uint8, len(kinds))
+	for i, v := range kinds {
+		convertedKinds[i] = uint8(v)
+	}
+
+	if !slices.Contains(convertedKinds, val) {
+		return 0, errors.New("Invalid metadata type")
+	}
+
+	return types.MetadataType(val), nil
+}

service/default/http.go

@@ -0,0 +1,199 @@
+package _default
+
+import (
+	"context"
+	"git.lumeweb.com/LumeWeb/libs5-go/build"
+	s5net "git.lumeweb.com/LumeWeb/libs5-go/net"
+	"git.lumeweb.com/LumeWeb/libs5-go/service"
+	"github.com/julienschmidt/httprouter"
+	"go.sia.tech/jape"
+	"go.uber.org/zap"
+	"net"
+	"net/url"
+	"nhooyr.io/websocket"
+	"strings"
+)
+
+var _ service.Service = (*HTTPServiceDefault)(nil)
+
+type P2PNodesResponse struct {
+	Nodes []P2PNodeResponse `json:"nodes"`
+}
+
+type P2PNodeResponse struct {
+	Id   string   `json:"id"`
+	Uris []string `json:"uris"`
+}
+
+type HTTPServiceDefault struct {
+	service.ServiceBase
+}
+
+func NewHTTP(params service.ServiceParams) *HTTPServiceDefault {
+	return &HTTPServiceDefault{
+		ServiceBase: service.NewServiceBase(params.Logger, params.Config, params.Db),
+	}
+}
+
+func (h *HTTPServiceDefault) GetHttpRouter(inject map[string]jape.Handler) *httprouter.Router {
+	routes := map[string]jape.Handler{
+		"GET /s5/version":   h.versionHandler,
+		"GET /s5/p2p":       h.p2pHandler,
+		"GET /s5/p2p/nodes": h.p2pNodesHandler,
+		"GET /s5/p2p/peers": h.p2pPeersHandler,
+	}
+
+	for k, v := range inject {
+		routes[k] = v
+	}
+
+	return jape.Mux(routes)
+}
+
+func (h *HTTPServiceDefault) Start(ctx context.Context) error {
+	return nil
+}
+
+func (h *HTTPServiceDefault) Stop(ctx context.Context) error {
+	return nil
+}
+
+func (h *HTTPServiceDefault) Init(ctx context.Context) error {
+	return nil
+}
+
+func (h *HTTPServiceDefault) versionHandler(ctx jape.Context) {
+	_, _ = ctx.ResponseWriter.Write([]byte(build.Version))
+}
+func (h *HTTPServiceDefault) p2pHandler(ctx jape.Context) {
+	c, err := websocket.Accept(ctx.ResponseWriter, ctx.Request, nil)
+	if err != nil {
+		h.Logger().Error("error accepting websocket connection", zap.Error(err))
+		return
+	}
+
+	peer, err := s5net.CreateTransportPeer("wss", &s5net.TransportPeerConfig{
+		Socket: c,
+		Uris:   []*url.URL{},
+	})
+
+	if err != nil {
+		h.Logger().Error("error creating transport peer", zap.Error(err))
+		err := c.Close(websocket.StatusInternalError, "the sky is falling")
+		if err != nil {
+			h.Logger().Error("error closing websocket connection", zap.Error(err))
+		}
+		return
+	}
+
+	// Check for reverse proxy headers
+	realIP := ctx.Request.Header.Get("X-Real-IP")
+	forwardedFor := ctx.Request.Header.Get("X-Forwarded-For")
+
+	var clientIP net.IP
+	if realIP != "" {
+		clientIP = net.ParseIP(realIP)
+	} else if forwardedFor != "" {
+		// X-Forwarded-For can contain multiple IP addresses separated by commas
+		// We take the first IP in the list as the client's IP
+		parts := strings.Split(forwardedFor, ",")
+		clientIP = net.ParseIP(parts[0])
+	}
+
+	blockConnection := func(ip net.Addr) bool {
+		// If we have a valid client IP from headers, use that for the loopback check
+		if clientIP != nil {
+			return clientIP.IsLoopback()
+		}
+		// Otherwise, fall back to the peer's IP
+		switch v := ip.(type) {
+		case *net.IPNet:
+			return v.IP.IsLoopback()
+		case *net.TCPAddr:
+			return v.IP.IsLoopback()
+		default:
+			return false
+		}
+	}
+
+	if blockConnection(peer.GetIP()) {
+		err := peer.End()
+		if err != nil {
+			h.Logger().Error("error ending peer", zap.Error(err))
+		}
+		return
+	}
+
+	if clientIP != nil {
+		peer.SetIP(&net.IPAddr{IP: clientIP})
+	}
+
+	h.Services().P2P().ConnectionTracker().Add(1)
+
+	go func() {
+		err := h.Services().P2P().OnNewPeer(peer, false)
+		if err != nil {
+			h.Logger().Error("error handling new peer", zap.Error(err))
+		}
+		h.Services().P2P().ConnectionTracker().Done()
+	}()
+}
+
+func (h *HTTPServiceDefault) p2pNodesHandler(ctx jape.Context) {
+	localId, err := h.Services().P2P().NodeId().ToString()
+
+	if ctx.Check("error getting local node id", err) != nil {
+		return
+	}
+
+	uris := h.Services().P2P().SelfConnectionUris()
+
+	nodeList := make([]P2PNodeResponse, len(uris))
+
+	for i, uri := range uris {
+		nodeList[i] = P2PNodeResponse{
+			Id:   localId,
+			Uris: []string{uri.String()},
+		}
+	}
+
+	ctx.Encode(P2PNodesResponse{
+		Nodes: nodeList,
+	})
+}
+func (h *HTTPServiceDefault) p2pPeersHandler(ctx jape.Context) {
+	peers := h.Services().P2P().Peers().Values()
+	peerList := make([]P2PNodeResponse, 0)
+
+	for _, p := range peers {
+		peer, ok := p.(s5net.Peer)
+		if !ok {
+			continue
+		}
+
+		id, err := peer.Id().ToString()
+		if err != nil {
+			h.Logger().Error("error getting peer id", zap.Error(err))
+			continue
+		}
+
+		if len(peer.ConnectionURIs()) == 0 {
+			continue
+		}
+
+		uris := make([]string, len(peer.ConnectionURIs()))
+
+		for i, uri := range peer.ConnectionURIs() {
+			uris[i] = uri.String()
+		}
+
+		peerList = append(peerList, P2PNodeResponse{
+			Id:   id,
+			Uris: uris,
+		})
+	}
+
+	ctx.Encode(P2PNodesResponse{
+		Nodes: peerList,
+	})
+}

service/default/mediator.go

@@ -0,0 +1,88 @@
+package _default
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/net"
+	"git.lumeweb.com/LumeWeb/libs5-go/protocol"
+	"git.lumeweb.com/LumeWeb/libs5-go/service"
+	"git.lumeweb.com/LumeWeb/libs5-go/storage"
+	"git.lumeweb.com/LumeWeb/libs5-go/structs"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"net/url"
+)
+
+var _ protocol.Mediator = (*MediatorDefault)(nil)
+
+type MediatorDefault struct {
+	service.ServiceBase
+}
+
+func (m MediatorDefault) NetworkId() string {
+	return m.Services().P2P().NetworkId()
+}
+
+func (m MediatorDefault) NodeId() *encoding.NodeId {
+	return m.Services().P2P().NodeId()
+}
+
+func (m MediatorDefault) SelfConnectionUris() []*url.URL {
+	return m.Services().P2P().SelfConnectionUris()
+}
+
+func (m MediatorDefault) SignMessageSimple(message []byte) ([]byte, error) {
+	return m.Services().P2P().SignMessageSimple(message)
+}
+
+func (m MediatorDefault) GetCachedStorageLocations(hash *encoding.Multihash, kinds []types.StorageLocationType) (map[string]storage.StorageLocation, error) {
+	return m.Services().Storage().GetCachedStorageLocations(hash, kinds, false)
+}
+
+func (m MediatorDefault) SortNodesByScore(nodes []*encoding.NodeId) ([]*encoding.NodeId, error) {
+	return m.Services().P2P().SortNodesByScore(nodes)
+}
+
+func (m MediatorDefault) ProviderStore() storage.ProviderStore {
+	return m.Services().Storage().ProviderStore()
+}
+
+func (m MediatorDefault) AddStorageLocation(hash *encoding.Multihash, nodeId *encoding.NodeId, location storage.StorageLocation, message []byte) error {
+	return m.Services().Storage().AddStorageLocation(hash, nodeId, location, message)
+}
+
+func (m MediatorDefault) HashQueryRoutingTable() structs.Map {
+	return m.Services().P2P().HashQueryRoutingTable()
+}
+
+func (m MediatorDefault) Peers() structs.Map {
+	return m.Services().P2P().Peers()
+}
+
+func (m MediatorDefault) RegistrySet(sre protocol.SignedRegistryEntry, trusted bool, receivedFrom net.Peer) error {
+	return m.Services().Registry().Set(sre, trusted, receivedFrom)
+}
+
+func (m MediatorDefault) RegistryGet(pk []byte) (protocol.SignedRegistryEntry, error) {
+	return m.Services().Registry().Get(pk)
+}
+
+func (m MediatorDefault) ConnectToNode(connectionUris []*url.URL, retried bool, fromPeer net.Peer) error {
+	return m.Services().P2P().ConnectToNode(connectionUris, 0, fromPeer)
+}
+
+func (m MediatorDefault) ServicesStarted() bool {
+	return m.Services().IsStarted()
+}
+
+func (m MediatorDefault) AddPeer(peer net.Peer) error {
+	return m.Services().P2P().AddPeer(peer)
+}
+
+func (m MediatorDefault) SendPublicPeersToPeer(peer net.Peer, peersToSend []net.Peer) error {
+	return m.Services().P2P().SendPublicPeersToPeer(peer, peersToSend)
+}
+
+func NewMediator(params service.ServiceParams) *MediatorDefault {
+	return &MediatorDefault{
+		ServiceBase: service.NewServiceBase(params.Logger, params.Config, params.Db),
+	}
+}

service/default/p2p.go

@@ -0,0 +1,702 @@
+package _default
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"git.lumeweb.com/LumeWeb/libs5-go/db"
+	"git.lumeweb.com/LumeWeb/libs5-go/ed25519"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/net"
+	"git.lumeweb.com/LumeWeb/libs5-go/protocol"
+	"git.lumeweb.com/LumeWeb/libs5-go/service"
+	"git.lumeweb.com/LumeWeb/libs5-go/structs"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/vmihailenco/msgpack/v5"
+	bolt "go.etcd.io/bbolt"
+	"go.uber.org/zap"
+	"net/url"
+	"sort"
+	"sync"
+	"time"
+)
+
+var _ service.P2PService = (*P2PServiceDefault)(nil)
+
+var (
+	errUnsupportedProtocol       = errors.New("unsupported protocol")
+	errConnectionIdMissingNodeID = errors.New("connection id missing node id")
+)
+
+const nodeBucketName = "nodes"
+
+type P2PServiceDefault struct {
+	nodeKeyPair             *ed25519.KeyPairEd25519
+	localNodeID             *encoding.NodeId
+	networkID               string
+	nodesBucket             *bolt.Bucket
+	inited                  bool
+	reconnectDelay          structs.Map
+	peers                   structs.Map
+	peersPending            structs.Map
+	selfConnectionUris      []*url.URL
+	outgoingPeerBlocklist   structs.Map
+	incomingPeerBlockList   structs.Map
+	incomingIPBlocklist     structs.Map
+	outgoingPeerFailures    structs.Map
+	maxOutgoingPeerFailures uint
+	connections             sync.WaitGroup
+	hashQueryRoutingTable   structs.Map
+	bucket                  db.KVStore
+	service.ServiceBase
+}
+
+func NewP2P(params service.ServiceParams) *P2PServiceDefault {
+	uri, err := url.Parse(fmt.Sprintf("wss://%s:%d/s5/p2p", params.Config.HTTP.API.Domain, params.Config.HTTP.API.Port))
+	if err != nil {
+		params.Logger.Fatal("failed to parse HTTP API URL", zap.Error(err))
+	}
+
+	service := &P2PServiceDefault{
+		nodeKeyPair:             params.Config.KeyPair,
+		networkID:               params.Config.P2P.Network,
+		inited:                  false,
+		reconnectDelay:          structs.NewMap(),
+		peers:                   structs.NewMap(),
+		peersPending:            structs.NewMap(),
+		selfConnectionUris:      []*url.URL{uri},
+		outgoingPeerBlocklist:   structs.NewMap(),
+		incomingPeerBlockList:   structs.NewMap(),
+		incomingIPBlocklist:     structs.NewMap(),
+		outgoingPeerFailures:    structs.NewMap(),
+		maxOutgoingPeerFailures: params.Config.P2P.MaxOutgoingPeerFailures,
+		hashQueryRoutingTable:   structs.NewMap(),
+		ServiceBase:             service.NewServiceBase(params.Logger, params.Config, params.Db),
+	}
+
+	return service
+}
+
+func (p *P2PServiceDefault) SelfConnectionUris() []*url.URL {
+	return p.selfConnectionUris
+}
+
+func (p *P2PServiceDefault) Peers() structs.Map {
+	return p.peers
+}
+
+func (p *P2PServiceDefault) Start(ctx context.Context) error {
+	config := p.Config()
+	if len(config.P2P.Peers.Initial) > 0 {
+		initialPeers := config.P2P.Peers.Initial
+
+		for _, peer := range initialPeers {
+			u, err := url.Parse(peer)
+			if err != nil {
+				return err
+			}
+
+			peer := peer
+			go func() {
+				err := p.ConnectToNode([]*url.URL{u}, 0, nil)
+				if err != nil {
+					p.Logger().Error("failed to connect to initial peer", zap.Error(err), zap.String("peer", peer))
+				}
+			}()
+		}
+	}
+
+	return nil
+}
+
+func (p *P2PServiceDefault) Stop(ctx context.Context) error {
+	return nil
+}
+
+func (p *P2PServiceDefault) Init(ctx context.Context) error {
+	if p.inited {
+		return nil
+	}
+
+	p.localNodeID = encoding.NewNodeId(p.nodeKeyPair.PublicKey())
+
+	bucket, err := p.Db().Bucket(nodeBucketName)
+	if err != nil {
+		return err
+	}
+
+	err = bucket.Open()
+	if err != nil {
+		return err
+	}
+
+	p.bucket = bucket
+	p.inited = true
+
+	for _, peer := range p.Config().P2P.Peers.Blocklist {
+		_, err := encoding.DecodeNodeId(peer)
+		if err != nil {
+			return err
+		}
+
+		p.incomingPeerBlockList.Put(peer, true)
+		p.outgoingPeerBlocklist.Put(peer, true)
+	}
+
+	return nil
+}
+func (p *P2PServiceDefault) ConnectToNode(connectionUris []*url.URL, retry uint, fromPeer net.Peer) error {
+	if !p.Services().IsStarted() {
+		if !p.Services().IsStarting() {
+			return nil
+		}
+	}
+
+	unsupported, _ := url.Parse("http://0.0.0.0")
+	unsupported.Scheme = "unsupported"
+
+	var connectionUri *url.URL
+
+	for _, uri := range connectionUris {
+		if uri.Scheme == "ws" || uri.Scheme == "wss" {
+			connectionUri = uri
+			break
+		}
+	}
+
+	if connectionUri == nil {
+		for _, uri := range connectionUris {
+			if uri.Scheme == "tcp" {
+				connectionUri = uri
+				break
+			}
+		}
+	}
+
+	if connectionUri == nil {
+		connectionUri = unsupported
+	}
+
+	if connectionUri.Scheme == "unsupported" {
+		return errUnsupportedProtocol
+	}
+
+	scheme := connectionUri.Scheme
+
+	if connectionUri.User == nil {
+		return errConnectionIdMissingNodeID
+	}
+
+	username := connectionUri.User.Username()
+	id, err := encoding.DecodeNodeId(username)
+	if err != nil {
+		return err
+	}
+
+	idString, err := id.ToString()
+	if err != nil {
+		return err
+	}
+
+	if p.peersPending.Contains(idString) || p.peers.Contains(idString) {
+		p.Logger().Debug("already connected", zap.String("node", connectionUri.String()))
+		return nil
+	}
+
+	if p.outgoingPeerBlocklist.Contains(idString) {
+		p.Logger().Debug("outgoing peer is on blocklist", zap.String("node", connectionUri.String()))
+
+		var fromPeerId string
+
+		if fromPeer != nil {
+			blocked := false
+			if fromPeer.Id() != nil {
+				fromPeerId, err = fromPeer.Id().ToString()
+				if err != nil {
+					return err
+				}
+				if !p.incomingPeerBlockList.Contains(fromPeerId) {
+					p.incomingPeerBlockList.Put(fromPeerId, true)
+					blocked = true
+				}
+			}
+
+			fromPeerIP := fromPeer.GetIPString()
+
+			if !p.incomingIPBlocklist.Contains(fromPeerIP) {
+				p.incomingIPBlocklist.Put(fromPeerIP, true)
+				blocked = true
+			}
+			err = fromPeer.EndForAbuse()
+			if err != nil {
+				return err
+			}
+
+			if blocked {
+				p.Logger().Debug("blocking peer for sending peer on blocklist", zap.String("node", connectionUri.String()), zap.String("peer", fromPeerId), zap.String("ip", fromPeerIP))
+			}
+		}
+		return nil
+	}
+
+	reconnectDelay := p.reconnectDelay.GetUInt(idString)
+	if reconnectDelay == nil {
+		delay := uint(1)
+		reconnectDelay = &delay
+	}
+
+	if id.Equals(p.localNodeID) {
+		return nil
+	}
+
+	p.Logger().Debug("connect", zap.String("node", connectionUri.String()))
+
+	socket, err := net.CreateTransportSocket(scheme, connectionUri)
+	if err != nil {
+		if retry > p.Config().P2P.MaxConnectionAttempts {
+			p.Logger().Error("failed to connect, too many retries", zap.String("node", connectionUri.String()), zap.Error(err))
+			counter := uint(0)
+			if p.outgoingPeerFailures.Contains(idString) {
+				tmp := *p.outgoingPeerFailures.GetUInt(idString)
+				counter = tmp
+			}
+
+			counter++
+
+			p.outgoingPeerFailures.PutUInt(idString, counter)
+
+			if counter >= p.maxOutgoingPeerFailures {
+
+				if fromPeer != nil {
+					blocked := false
+					var fromPeerId string
+					if fromPeer.Id() != nil {
+						fromPeerId, err = fromPeer.Id().ToString()
+						if err != nil {
+							return err
+						}
+						if !p.incomingPeerBlockList.Contains(fromPeerId) {
+							p.incomingPeerBlockList.Put(fromPeerId, true)
+							blocked = true
+						}
+					}
+
+					fromPeerIP := fromPeer.GetIPString()
+					if !p.incomingIPBlocklist.Contains(fromPeerIP) {
+						p.incomingIPBlocklist.Put(fromPeerIP, true)
+						blocked = true
+					}
+					err = fromPeer.EndForAbuse()
+					if err != nil {
+						return err
+					}
+
+					if blocked {
+						p.Logger().Debug("blocking peer for sending peer on blocklist", zap.String("node", connectionUri.String()), zap.String("peer", fromPeerId), zap.String("ip", fromPeerIP))
+					}
+				}
+				p.outgoingPeerBlocklist.Put(idString, true)
+				p.Logger().Debug("blocking peer for too many failures", zap.String("node", connectionUri.String()))
+			}
+
+			return nil
+		}
+
+		if errors.Is(err, net.ErrTransportNotSupported) {
+			p.Logger().Debug("failed to connect, unsupported transport", zap.String("node", connectionUri.String()), zap.Error(err))
+			return err
+		}
+
+		retry++
+
+		p.Logger().Error("failed to connect", zap.String("node", connectionUri.String()), zap.Error(err))
+
+		delay := p.reconnectDelay.GetUInt(idString)
+		if delay == nil {
+			tmp := uint(1)
+			delay = &tmp
+		}
+		delayDeref := *delay
+		p.reconnectDelay.PutUInt(idString, delayDeref*2)
+
+		time.Sleep(time.Duration(delayDeref) * time.Second)
+
+		return p.ConnectToNode(connectionUris, retry, fromPeer)
+	}
+
+	if p.outgoingPeerFailures.Contains(idString) {
+		p.outgoingPeerFailures.Remove(idString)
+	}
+
+	peer, err := net.CreateTransportPeer(scheme, &net.TransportPeerConfig{
+		Socket: socket,
+		Uris:   []*url.URL{connectionUri},
+	})
+
+	if err != nil {
+		return err
+	}
+
+	peer.SetId(id)
+
+	p.Services().P2P().ConnectionTracker().Add(1)
+
+	peerId, err := peer.Id().ToString()
+	if err != nil {
+		return err
+	}
+	p.peersPending.Put(peerId, peer)
+
+	go func() {
+		err := p.OnNewPeer(peer, true)
+		if err != nil && !peer.Abuser() {
+			p.Logger().Error("peer error", zap.Error(err))
+		}
+		p.Services().P2P().ConnectionTracker().Done()
+	}()
+
+	return nil
+
+}
+
+func (p *P2PServiceDefault) OnNewPeer(peer net.Peer, verifyId bool) error {
+	var wg sync.WaitGroup
+
+	var pid string
+
+	if peer.Id() != nil {
+		pid, _ = peer.Id().ToString()
+	} else {
+		pid = "unknown"
+	}
+
+	pip := peer.GetIPString()
+
+	if p.incomingIPBlocklist.Contains(pid) {
+		p.Logger().Error("peer is on identity blocklist", zap.String("peer", pid))
+		err := peer.EndForAbuse()
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	if p.incomingPeerBlockList.Contains(pip) {
+		p.Logger().Debug("peer is on ip blocklist", zap.String("peer", pid), zap.String("ip", pip))
+		err := peer.EndForAbuse()
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	p.Logger().Debug("OnNewPeer started", zap.String("peer", pid))
+
+	challenge := protocol.GenerateChallenge()
+	peer.SetChallenge(challenge)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		p.OnNewPeerListen(peer, verifyId)
+	}()
+
+	handshakeOpenMsg, err := msgpack.Marshal(protocol.NewHandshakeOpen(challenge, p.networkID))
+	if err != nil {
+		return err
+	}
+
+	err = peer.SendMessage(handshakeOpenMsg)
+	if err != nil {
+		return err
+	}
+	p.Logger().Debug("OnNewPeer sent handshake", zap.String("peer", pid))
+
+	p.Logger().Debug("OnNewPeer before Wait", zap.String("peer", pid))
+	wg.Wait() // Wait for OnNewPeerListen goroutine to finish
+	p.Logger().Debug("OnNewPeer ended", zap.String("peer", pid))
+	return nil
+}
+func (p *P2PServiceDefault) OnNewPeerListen(peer net.Peer, verifyId bool) {
+	onDone := net.CloseCallback(func() {
+		if peer.Id() != nil {
+			pid, err := peer.Id().ToString()
+			if err != nil {
+				p.Logger().Error("failed to get peer id", zap.Error(err))
+				return
+			}
+			// Handle closure of the connection
+			if p.peers.Contains(pid) {
+				p.peers.Remove(pid)
+			}
+			if p.peersPending.Contains(pid) {
+				p.peersPending.Remove(pid)
+			}
+		}
+	})
+
+	onError := net.ErrorCallback(func(args ...interface{}) {
+		if !peer.Abuser() {
+			p.Logger().Error("peer error", zap.Any("args", args))
+		}
+	})
+
+	peer.ListenForMessages(func(message []byte) error {
+		var reader protocol.IncomingMessageReader
+
+		err := msgpack.Unmarshal(message, &reader)
+		if err != nil {
+			p.Logger().Error("Error decoding basic message info", zap.Error(err))
+			return err
+		}
+
+		// Now, get the specific message handler based on the message kind
+		handler, ok := protocol.GetMessageType(reader.Kind)
+		if !ok {
+			p.Logger().Error("Unknown message type", zap.Int("type", reader.Kind))
+			return fmt.Errorf("unknown message type: %d", reader.Kind)
+		}
+
+		if handler.RequiresHandshake() && !peer.IsHandshakeDone() {
+			p.Logger().Debug("Peer is not handshake done, ignoring message", zap.Any("type", types.ProtocolMethodMap[types.ProtocolMethod(reader.Kind)]))
+			return nil
+		}
+
+		data := protocol.IncomingMessageData{
+			Original: message,
+			Data:     reader.Data,
+			Ctx:      context.Background(),
+			Peer:     peer,
+			VerifyId: verifyId,
+			Config:   p.Config(),
+			Logger:   p.Logger(),
+			Mediator: NewMediator(service.ServiceParams{
+				Logger: p.Logger(),
+				Config: p.Config(),
+				Db:     p.Db(),
+			}),
+		}
+
+		if mediator, ok := data.Mediator.(service.ServicesSetter); ok {
+			mediator.SetServices(p.Services())
+		} else {
+			p.Logger().Fatal("failed to cast mediator to service.Service")
+		}
+
+		dec := msgpack.NewDecoder(bytes.NewReader(reader.Data))
+
+		err = handler.DecodeMessage(dec, data)
+		if err != nil {
+			p.Logger().Error("Error decoding message", zap.Error(err))
+			return err
+		}
+
+		// Directly decode and handle the specific message type
+		if err = handler.HandleMessage(data); err != nil {
+			p.Logger().Error("Error handling message", zap.Error(err))
+			return err
+		}
+
+		return nil
+	}, net.ListenerOptions{
+		OnClose: &onDone,
+		OnError: &onError,
+		Logger:  p.Logger(),
+	})
+}
+
+func (p *P2PServiceDefault) readNodeVotes(nodeId *encoding.NodeId) (service.NodeVotes, error) {
+	var value []byte
+
+	value, err := p.bucket.Get(nodeId.Raw())
+	if err != nil {
+		return nil, err
+	}
+
+	if value == nil {
+		return service.NewNodeVotes(), nil
+	}
+
+	score := service.NewNodeVotes()
+	err = msgpack.Unmarshal(value, &score)
+	if err != nil {
+		return nil, err
+	}
+
+	return score, nil
+}
+
+func (p *P2PServiceDefault) saveNodeVotes(nodeId *encoding.NodeId, votes service.NodeVotes) error {
+	// Marshal the votes into data
+	data, err := msgpack.Marshal(votes)
+	if err != nil {
+		return err
+	}
+
+	err = p.bucket.Put(nodeId.Raw(), data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *P2PServiceDefault) GetNodeScore(nodeId *encoding.NodeId) (float64, error) {
+	if nodeId.Equals(p.localNodeID) {
+		return 1, nil
+	}
+
+	score, err := p.readNodeVotes(nodeId)
+	if err != nil {
+		return 0.5, err
+	}
+
+	return protocol.CalculateNodeScore(score.Good(), score.Bad()), nil
+
+}
+func (p *P2PServiceDefault) SortNodesByScore(nodes []*encoding.NodeId) ([]*encoding.NodeId, error) {
+	scores := make(map[encoding.NodeIdCode]float64)
+	var errOccurred error
+
+	for _, nodeId := range nodes {
+		score, err := p.GetNodeScore(nodeId)
+		if err != nil {
+			errOccurred = err
+			scores[nodeId.HashCode()] = 0 // You may choose a different default value for error cases
+		} else {
+			scores[nodeId.HashCode()] = score
+		}
+	}
+
+	sort.Slice(nodes, func(i, j int) bool {
+		return scores[nodes[i].HashCode()] > scores[nodes[j].HashCode()]
+	})
+
+	return nodes, errOccurred
+}
+func (p *P2PServiceDefault) SignMessageSimple(message []byte) ([]byte, error) {
+	signedMessage := protocol.NewSignedMessageRequest(message)
+	signedMessage.SetNodeId(p.localNodeID)
+
+	err := signedMessage.Sign(p.Config())
+
+	if err != nil {
+		return nil, err
+	}
+
+	result, err := msgpack.Marshal(signedMessage)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func (p *P2PServiceDefault) AddPeer(peer net.Peer) error {
+	peerId, err := peer.Id().ToString()
+	if err != nil {
+		return err
+	}
+	p.peers.Put(peerId, peer)
+	p.reconnectDelay.PutUInt(peerId, 1)
+
+	if p.peersPending.Contains(peerId) {
+		p.peersPending.Remove(peerId)
+	}
+
+	return nil
+}
+func (p *P2PServiceDefault) SendPublicPeersToPeer(peer net.Peer, peersToSend []net.Peer) error {
+	announceRequest := protocol.NewAnnounceRequest(peer, peersToSend)
+
+	message, err := msgpack.Marshal(announceRequest)
+
+	if err != nil {
+		return err
+	}
+
+	signedMessage, err := p.SignMessageSimple(message)
+
+	if err != nil {
+		return err
+	}
+
+	err = peer.SendMessage(signedMessage)
+
+	return nil
+}
+func (p *P2PServiceDefault) SendHashRequest(hash *encoding.Multihash, kinds []types.StorageLocationType) error {
+	hashRequest := protocol.NewHashRequest(hash, kinds)
+	message, err := msgpack.Marshal(hashRequest)
+	if err != nil {
+		return err
+	}
+
+	for _, peer := range p.peers.Values() {
+		peerValue, ok := peer.(net.Peer)
+		if !ok {
+			p.Logger().Error("failed to cast peer to net.Peer")
+			continue
+		}
+		err = peerValue.SendMessage(message)
+	}
+
+	return nil
+}
+
+func (p *P2PServiceDefault) UpVote(nodeId *encoding.NodeId) error {
+	err := p.vote(nodeId, true)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *P2PServiceDefault) DownVote(nodeId *encoding.NodeId) error {
+	err := p.vote(nodeId, false)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *P2PServiceDefault) vote(nodeId *encoding.NodeId, upvote bool) error {
+	votes, err := p.readNodeVotes(nodeId)
+	if err != nil {
+		return err
+	}
+
+	if upvote {
+		votes.Upvote()
+	} else {
+		votes.Downvote()
+	}
+
+	err = p.saveNodeVotes(nodeId, votes)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+func (p *P2PServiceDefault) NodeId() *encoding.NodeId {
+	return p.localNodeID
+}
+
+func (p *P2PServiceDefault) WaitOnConnectedPeers() {
+	p.connections.Wait()
+}
+
+func (p *P2PServiceDefault) ConnectionTracker() *sync.WaitGroup {
+	return &p.connections
+}
+
+func (p *P2PServiceDefault) NetworkId() string {
+	return p.Config().P2P.Network
+}
+func (n *P2PServiceDefault) HashQueryRoutingTable() structs.Map {
+	return n.hashQueryRoutingTable
+}

service/default/registry.go

@@ -0,0 +1,311 @@
+package _default
+
+import (
+	"context"
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/db"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/net"
+	"git.lumeweb.com/LumeWeb/libs5-go/protocol"
+	"git.lumeweb.com/LumeWeb/libs5-go/service"
+	"git.lumeweb.com/LumeWeb/libs5-go/structs"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/olebedev/emitter"
+	"github.com/vmihailenco/msgpack/v5"
+	"go.uber.org/zap"
+	"time"
+)
+
+const registryBucketName = "registry"
+
+var (
+	_ service.Service         = (*RegistryServiceDefault)(nil)
+	_ service.RegistryService = (*RegistryServiceDefault)(nil)
+)
+
+type RegistryServiceDefault struct {
+	streams structs.Map
+	subs    structs.Map
+	bucket  db.KVStore
+	service.ServiceBase
+}
+
+func (r *RegistryServiceDefault) Start(ctx context.Context) error {
+	return nil
+}
+
+func (r *RegistryServiceDefault) Stop(ctx context.Context) error {
+	return nil
+}
+
+func (r *RegistryServiceDefault) Init(ctx context.Context) error {
+	bucket, err := r.Db().Bucket(registryBucketName)
+	if err != nil {
+		return err
+	}
+
+	err = bucket.Open()
+	if err != nil {
+		return err
+	}
+
+	r.bucket = bucket
+
+	return nil
+}
+
+func NewRegistry(params service.ServiceParams) *RegistryServiceDefault {
+	return &RegistryServiceDefault{
+		streams:     structs.NewMap(),
+		subs:        structs.NewMap(),
+		ServiceBase: service.NewServiceBase(params.Logger, params.Config, params.Db),
+	}
+}
+func (r *RegistryServiceDefault) Set(sre protocol.SignedRegistryEntry, trusted bool, receivedFrom net.Peer) error {
+	hash := encoding.NewMultihash(sre.PK())
+	hashString, err := hash.ToString()
+	if err != nil {
+		return err
+	}
+	pid, err := receivedFrom.Id().ToString()
+	if err != nil {
+		return err
+	}
+	r.Logger().Debug("[registry] set", zap.String("pk", hashString), zap.Uint64("revision", sre.Revision()), zap.String("receivedFrom", pid))
+
+	if !trusted {
+		if len(sre.PK()) != 33 {
+			return errors.New("Invalid pubkey")
+		}
+		if int(sre.PK()[0]) != int(types.HashTypeEd25519) {
+			return errors.New("Only ed25519 keys are supported")
+		}
+		if sre.Revision() < 0 || sre.Revision() > 281474976710656 {
+			return errors.New("Invalid revision")
+		}
+		if len(sre.Data()) > types.RegistryMaxDataSize {
+			return errors.New("Data too long")
+		}
+
+		if !sre.Verify() {
+			return errors.New("Invalid signature found")
+		}
+	}
+
+	existingEntry, err := r.getFromDB(sre.PK())
+	if err != nil {
+		return err
+	}
+
+	if existingEntry != nil {
+		if receivedFrom != nil {
+			if existingEntry.Revision() == sre.Revision() {
+				return nil
+			} else if existingEntry.Revision() > sre.Revision() {
+				updateMessage := protocol.MarshalSignedRegistryEntry(existingEntry)
+				err := receivedFrom.SendMessage(updateMessage)
+				if err != nil {
+					return err
+				}
+				return nil
+			}
+		}
+
+		if existingEntry.Revision() >= sre.Revision() {
+			return errors.New("Revision number too low")
+		}
+	}
+
+	key := encoding.NewMultihash(sre.PK())
+	keyString, err := key.ToString()
+	if err != nil {
+		return err
+	}
+
+	eventObj, ok := r.streams.Get(keyString)
+	if ok {
+		event := eventObj.(*emitter.Emitter)
+		go event.Emit("fire", sre)
+	}
+
+	err = r.bucket.Put(sre.PK(), protocol.MarshalSignedRegistryEntry(sre))
+	if err != nil {
+		return err
+	}
+
+	err = r.BroadcastEntry(sre, receivedFrom)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+func (r *RegistryServiceDefault) BroadcastEntry(sre protocol.SignedRegistryEntry, receivedFrom net.Peer) error {
+	hash := encoding.NewMultihash(sre.PK())
+	hashString, err := hash.ToString()
+	if err != nil {
+		return err
+	}
+	pid, err := receivedFrom.Id().ToString()
+	if err != nil {
+		return err
+	}
+	r.Logger().Debug("[registry] broadcastEntry", zap.String("pk", hashString), zap.Uint64("revision", sre.Revision()), zap.String("receivedFrom", pid))
+	updateMessage := protocol.MarshalSignedRegistryEntry(sre)
+
+	for _, p := range r.Services().P2P().Peers().Values() {
+		peer, ok := p.(net.Peer)
+		if !ok {
+			continue
+		}
+		if receivedFrom == nil || peer.Id().Equals(receivedFrom.Id()) {
+			err := peer.SendMessage(updateMessage)
+			if err != nil {
+				pid, err := peer.Id().ToString()
+				if err != nil {
+					return err
+				}
+				r.Logger().Error("Failed to send registry broadcast", zap.String("peer", pid), zap.Error(err))
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+func (r *RegistryServiceDefault) SendRegistryRequest(pk []byte) error {
+	query := protocol.NewRegistryQuery(pk)
+
+	request, err := msgpack.Marshal(query)
+	if err != nil {
+		return err
+	}
+
+	// Iterate over peers and send the request
+	for _, peerVal := range r.Services().P2P().Peers().Values() {
+		peer, ok := peerVal.(net.Peer)
+		if !ok {
+			continue
+		}
+		err := peer.SendMessage(request)
+		if err != nil {
+			pid, err := peer.Id().ToString()
+			if err != nil {
+				return err
+			}
+			r.Logger().Error("Failed to send registry request", zap.String("peer", pid), zap.Error(err))
+			return err
+		}
+	}
+
+	return nil
+}
+func (r *RegistryServiceDefault) Get(pk []byte) (protocol.SignedRegistryEntry, error) {
+	key := encoding.NewMultihash(pk)
+	keyString, err := key.ToString()
+	if err != nil {
+		return nil, err
+	}
+
+	if r.subs.Contains(keyString) {
+		r.Logger().Debug("[registry] get (cached)", zap.String("key", keyString))
+		res, err := r.getFromDB(pk)
+		if err != nil {
+			return nil, err
+		}
+		if res != nil {
+			return res, nil
+		}
+
+		err = r.SendRegistryRequest(pk)
+		if err != nil {
+			return nil, err
+		}
+		time.Sleep(200 * time.Millisecond)
+		return r.getFromDB(pk)
+	}
+
+	err = r.SendRegistryRequest(pk)
+	if err != nil {
+		return nil, err
+	}
+	r.subs.Put(keyString, key)
+	if !r.streams.Contains(keyString) {
+		event := &emitter.Emitter{}
+		r.streams.Put(keyString, event)
+	}
+
+	res, err := r.getFromDB(pk)
+	if err != nil {
+		return nil, err
+	}
+
+	if res != nil {
+		return res, nil
+	}
+
+	r.Logger().Debug("[registry] get (cached)", zap.String("key", keyString))
+	for i := 0; i < 200; i++ {
+		time.Sleep(10 * time.Millisecond)
+		res, err = r.getFromDB(pk)
+		if err != nil {
+			return nil, err
+		}
+		if res != nil {
+			break
+		}
+	}
+
+	return res, nil
+}
+
+func (r *RegistryServiceDefault) Listen(pk []byte, cb func(sre protocol.SignedRegistryEntry)) (func(), error) {
+	key, err := encoding.NewMultihash(pk).ToString()
+	if err != nil {
+		return nil, err
+	}
+
+	cbProxy := func(event *emitter.Event) {
+		sre, ok := event.Args[0].(protocol.SignedRegistryEntry)
+		if !ok {
+			r.Logger().Error("Failed to cast event to SignedRegistryEntry")
+			return
+		}
+
+		cb(sre)
+	}
+
+	if !r.streams.Contains(key) {
+		em := emitter.New(0)
+		r.streams.Put(key, em)
+		err := r.SendRegistryRequest(pk)
+		if err != nil {
+			return nil, err
+		}
+	}
+	streamVal, _ := r.streams.Get(key)
+	stream := streamVal.(*emitter.Emitter)
+	channel := stream.On("fire", cbProxy)
+
+	return func() {
+		stream.Off("fire", channel)
+	}, nil
+}
+
+func (r *RegistryServiceDefault) getFromDB(pk []byte) (sre protocol.SignedRegistryEntry, err error) {
+	value, err := r.bucket.Get(pk)
+	if err != nil {
+		return nil, err
+	}
+
+	if value != nil {
+		sre, err = protocol.UnmarshalSignedRegistryEntry(value)
+		if err != nil {
+			return nil, err
+		}
+
+		return sre, nil
+	}
+
+	return nil, nil
+}

service/default/storage.go

@@ -0,0 +1,372 @@
+package _default
+
+import (
+	"context"
+	"errors"
+	"git.lumeweb.com/LumeWeb/libs5-go/db"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/metadata"
+	"git.lumeweb.com/LumeWeb/libs5-go/service"
+	"git.lumeweb.com/LumeWeb/libs5-go/storage"
+	"git.lumeweb.com/LumeWeb/libs5-go/storage/provider"
+	"git.lumeweb.com/LumeWeb/libs5-go/structs"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/ddo/rq"
+	_ "github.com/ddo/rq"
+	"github.com/vmihailenco/msgpack/v5"
+	"go.uber.org/zap"
+	"io"
+	"net/http"
+	"time"
+)
+
+const cacheBucketName = "object-cache"
+
+var (
+	_ service.Service        = (*StorageService)(nil)
+	_ service.StorageService = (*StorageService)(nil)
+)
+
+var (
+	ErrUnsupportedMetaFormat = errors.New("unsupported metadata format")
+)
+
+type StorageService struct {
+	metadataCache structs.Map
+	providerStore storage.ProviderStore
+	bucket        db.KVStore
+	service.ServiceBase
+}
+
+func NewStorage(params service.ServiceParams) *StorageService {
+	return &StorageService{
+		metadataCache: structs.NewMap(),
+		ServiceBase:   service.NewServiceBase(params.Logger, params.Config, params.Db),
+	}
+}
+
+func (s *StorageService) Start(ctx context.Context) error {
+
+	bucket, err := s.Db().Bucket(cacheBucketName)
+	if err != nil {
+		return err
+	}
+
+	err = bucket.Open()
+	if err != nil {
+		return err
+	}
+
+	s.bucket = bucket
+
+	return nil
+}
+
+func (s *StorageService) Stop(ctx context.Context) error {
+	return nil
+}
+
+func (s *StorageService) Init(ctx context.Context) error {
+	return nil
+}
+
+func (n *StorageService) SetProviderStore(store storage.ProviderStore) {
+	n.providerStore = store
+}
+
+func (n *StorageService) ProviderStore() storage.ProviderStore {
+	return n.providerStore
+}
+
+func (s *StorageService) GetCachedStorageLocations(hash *encoding.Multihash, kinds []types.StorageLocationType, local bool) (map[string]storage.StorageLocation, error) {
+	locations := make(map[string]storage.StorageLocation)
+
+	locationMap, err := s.readStorageLocationsFromDB(hash)
+	if err != nil {
+		return nil, err
+	}
+
+	if local {
+		localLocation := s.getLocalStorageLocation(hash, kinds)
+		if localLocation != nil {
+			nodeIDStr, err := s.Services().P2P().NodeId().ToString()
+			if err != nil {
+				return nil, err
+			}
+
+			locations[nodeIDStr] = localLocation
+		}
+	}
+
+	if len(locationMap) == 0 {
+		return locations, nil
+	}
+
+	ts := time.Now().Unix()
+
+	for _, t := range kinds {
+		nodeMap, ok := (locationMap)[int(t)]
+		if !ok {
+			continue
+		}
+
+		for key, value := range nodeMap {
+			expiry, ok := value[3].(int64)
+			if !ok || expiry < ts {
+				continue
+			}
+
+			addressesInterface, ok := value[1].([]interface{})
+			if !ok {
+				continue
+			}
+
+			// Create a slice to hold the strings
+			addresses := make([]string, len(addressesInterface))
+
+			// Convert each element to string
+			for i, v := range addressesInterface {
+				str, ok := v.(string)
+				if !ok {
+					// Handle the error, maybe skip this element or set a default value
+					continue
+				}
+				addresses[i] = str
+			}
+
+			storageLocation := storage.NewStorageLocation(int(t), addresses, expiry)
+
+			if providerMessage, ok := value[4].([]byte); ok {
+				(storageLocation).SetProviderMessage(providerMessage)
+			}
+
+			locations[key] = storageLocation
+		}
+	}
+	return locations, nil
+}
+
+func (s *StorageService) getLocalStorageLocation(hash *encoding.Multihash, kinds []types.StorageLocationType) storage.StorageLocation {
+	if s.ProviderStore() != nil {
+		if s.ProviderStore().CanProvide(hash, kinds) {
+			location, _ := s.ProviderStore().Provide(hash, kinds)
+
+			message := storage.PrepareProvideMessage(s.Services().P2P().Config().KeyPair, hash, location)
+
+			location.SetProviderMessage(message)
+
+			return location
+		}
+	}
+
+	return nil
+}
+
+func (s *StorageService) readStorageLocationsFromDB(hash *encoding.Multihash) (storage.StorageLocationMap, error) {
+	var locationMap storage.StorageLocationMap
+
+	value, err := s.bucket.Get(hash.FullBytes())
+	if err != nil {
+		return nil, err
+	}
+
+	if value == nil {
+		return storage.NewStorageLocationMap(), nil
+	}
+
+	locationMap = storage.NewStorageLocationMap()
+
+	err = msgpack.Unmarshal(value, &locationMap)
+	if err != nil {
+		return nil, err
+	}
+
+	return locationMap, nil
+}
+
+func (s *StorageService) AddStorageLocation(hash *encoding.Multihash, nodeId *encoding.NodeId, location storage.StorageLocation, message []byte) error {
+	// Read existing storage locations
+	locationDb, err := s.readStorageLocationsFromDB(hash)
+	if err != nil {
+		return err
+	}
+
+	nodeIdStr, err := nodeId.ToString()
+	if err != nil {
+		return err
+	}
+
+	// Get or create the inner map for the specific type
+	innerMap, exists := locationDb[location.Type()]
+	if !exists {
+		innerMap = make(storage.NodeStorage, 1)
+		innerMap[nodeIdStr] = make(storage.NodeDetailsStorage, 1)
+	}
+
+	// Create location map with new data
+	locationMap := make(map[int]interface{}, 3)
+	locationMap[1] = location.Parts()
+	locationMap[3] = location.Expiry()
+	locationMap[4] = message
+
+	// Update the inner map with the new location
+	innerMap[nodeIdStr] = locationMap
+	locationDb[location.Type()] = innerMap
+
+	// Serialize the updated map and store it in the database
+	packedBytes, err := msgpack.Marshal(locationDb)
+	if err != nil {
+		return err
+	}
+
+	err = s.bucket.Put(hash.FullBytes(), packedBytes)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *StorageService) DownloadBytesByHash(hash *encoding.Multihash) ([]byte, error) {
+	// Initialize the download URI provider
+	dlUriProvider := provider.NewStorageLocationProvider(provider.StorageLocationProviderParams{
+		Services: s.Services(),
+		Hash:     hash,
+		LocationTypes: []types.StorageLocationType{
+			types.StorageLocationTypeFull,
+			types.StorageLocationTypeFile,
+		},
+		ServiceParams: service.ServiceParams{
+			Logger: s.Logger(),
+			Config: s.Config(),
+			Db:     s.Db(),
+		},
+	})
+	err := dlUriProvider.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	retryCount := 0
+	for {
+		dlUri, err := dlUriProvider.Next()
+		if err != nil {
+			return nil, err
+		}
+
+		s.Logger().Debug("Trying to download from", zap.String("url", dlUri.Location().BytesURL()))
+
+		req := rq.Get(dlUri.Location().BytesURL())
+		httpReq, err := req.ParseRequest()
+		if err != nil {
+			return nil, err
+		}
+
+		res, err := http.DefaultClient.Do(httpReq)
+		if err != nil {
+			err := dlUriProvider.Downvote(dlUri)
+			if err != nil {
+				return nil, err
+			}
+			retryCount++
+			if retryCount > 32 {
+				return nil, errors.New("too many retries")
+			}
+			continue
+		}
+
+		defer func(Body io.ReadCloser) {
+			err := Body.Close()
+			if err != nil {
+				s.Logger().Error("error closing body", zap.Error(err))
+			}
+		}(res.Body)
+
+		if res.StatusCode != 200 {
+			err := dlUriProvider.Downvote(dlUri)
+			retryCount++
+			if retryCount > 32 {
+				return nil, errors.New("too many retries")
+			}
+			if err != nil {
+				return nil, err
+			}
+			continue
+		}
+
+		bodyBytes, err := io.ReadAll(res.Body)
+		if err != nil {
+			return nil, err
+		}
+
+		return bodyBytes, nil
+	}
+}
+
+func (s *StorageService) DownloadBytesByCID(cid *encoding.CID) (bytes []byte, err error) {
+	bytes, err = s.DownloadBytesByHash(&cid.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return bytes, nil
+}
+
+func (s *StorageService) GetMetadataByCID(cid *encoding.CID) (md metadata.Metadata, err error) {
+	hashStr, err := cid.Hash.ToString()
+	if err != nil {
+		return nil, err
+	}
+
+	if s.metadataCache.Contains(hashStr) {
+		md, _ := s.metadataCache.Get(hashStr)
+
+		return md.(metadata.Metadata), nil
+	}
+
+	bytes, err := s.DownloadBytesByHash(&cid.Hash)
+	if err != nil {
+		return nil, err
+	}
+
+	md, err = s.ParseMetadata(bytes, cid)
+	if err != nil {
+		return nil, err
+	}
+
+	s.metadataCache.Put(hashStr, md)
+
+	return md, nil
+}
+
+func (s *StorageService) ParseMetadata(bytes []byte, cid *encoding.CID) (metadata.Metadata, error) {
+	var md metadata.Metadata
+
+	switch cid.Type {
+	case types.CIDTypeMetadataMedia, types.CIDTypeBridge: // Both cases use the same deserialization method
+		md = metadata.NewEmptyMediaMetadata()
+
+		err := msgpack.Unmarshal(bytes, md)
+		if err != nil {
+			return nil, err
+		}
+	case types.CIDTypeMetadataWebapp:
+		md = metadata.NewEmptyWebAppMetadata()
+
+		err := msgpack.Unmarshal(bytes, md)
+		if err != nil {
+			return nil, err
+		}
+	case types.CIDTypeDirectory:
+		md = metadata.NewEmptyDirectoryMetadata()
+
+		err := msgpack.Unmarshal(bytes, md)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, ErrUnsupportedMetaFormat
+	}
+
+	return md, nil
+}

service/http.go

@@ -0,0 +1,11 @@
+package service
+
+import (
+	"github.com/julienschmidt/httprouter"
+	"go.sia.tech/jape"
+)
+
+type HTTPService interface {
+	GetHttpRouter(inject map[string]jape.Handler) *httprouter.Router
+	Service
+}

service/p2p.go

@@ -0,0 +1,31 @@
+package service
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/net"
+	"git.lumeweb.com/LumeWeb/libs5-go/structs"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"net/url"
+	"sync"
+)
+
+type P2PService interface {
+	SelfConnectionUris() []*url.URL
+	Peers() structs.Map
+	ConnectToNode(connectionUris []*url.URL, retry uint, fromPeer net.Peer) error
+	OnNewPeer(peer net.Peer, verifyId bool) error
+	GetNodeScore(nodeId *encoding.NodeId) (float64, error)
+	SortNodesByScore(nodes []*encoding.NodeId) ([]*encoding.NodeId, error)
+	SignMessageSimple(message []byte) ([]byte, error)
+	AddPeer(peer net.Peer) error
+	SendPublicPeersToPeer(peer net.Peer, peersToSend []net.Peer) error
+	SendHashRequest(hash *encoding.Multihash, kinds []types.StorageLocationType) error
+	UpVote(nodeId *encoding.NodeId) error
+	DownVote(nodeId *encoding.NodeId) error
+	NodeId() *encoding.NodeId
+	WaitOnConnectedPeers()
+	ConnectionTracker() *sync.WaitGroup
+	NetworkId() string
+	HashQueryRoutingTable() structs.Map
+	Service
+}

service/registry.go

@@ -0,0 +1,15 @@
+package service
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/net"
+	"git.lumeweb.com/LumeWeb/libs5-go/protocol"
+)
+
+type RegistryService interface {
+	Set(sre protocol.SignedRegistryEntry, trusted bool, receivedFrom net.Peer) error
+	BroadcastEntry(sre protocol.SignedRegistryEntry, receivedFrom net.Peer) error
+	SendRegistryRequest(pk []byte) error
+	Get(pk []byte) (protocol.SignedRegistryEntry, error)
+	Listen(pk []byte, cb func(sre protocol.SignedRegistryEntry)) (func(), error)
+	Service
+}

service/service.go

@@ -0,0 +1,67 @@
+package service
+
+import (
+	"context"
+	"git.lumeweb.com/LumeWeb/libs5-go/config"
+	"git.lumeweb.com/LumeWeb/libs5-go/db"
+	"go.uber.org/zap"
+)
+
+type ServicesSetter interface {
+	SetServices(services Services)
+}
+
+type Service interface {
+	Start(ctx context.Context) error
+	Stop(ctx context.Context) error
+	Init(ctx context.Context) error
+	Logger() *zap.Logger
+	Config() *config.NodeConfig
+	Db() db.KVStore
+	ServicesSetter
+}
+type Services interface {
+	P2P() P2PService
+	Registry() RegistryService
+	HTTP() HTTPService
+	Storage() StorageService
+	All() []Service
+	Init(ctx context.Context) error
+	IsStarted() bool
+	IsStarting() bool
+	Start(ctx context.Context) error
+	Stop(ctx context.Context) error
+}
+
+type ServiceParams struct {
+	Logger *zap.Logger
+	Config *config.NodeConfig
+	Db     db.KVStore
+}
+
+type ServiceBase struct {
+	logger   *zap.Logger
+	config   *config.NodeConfig
+	db       db.KVStore
+	services Services
+}
+
+func NewServiceBase(logger *zap.Logger, config *config.NodeConfig, db db.KVStore) ServiceBase {
+	return ServiceBase{logger: logger, config: config, db: db}
+}
+
+func (s *ServiceBase) SetServices(services Services) {
+	s.services = services
+}
+func (s *ServiceBase) Services() Services {
+	return s.services
+}
+func (s *ServiceBase) Logger() *zap.Logger {
+	return s.logger
+}
+func (s *ServiceBase) Config() *config.NodeConfig {
+	return s.config
+}
+func (s *ServiceBase) Db() db.KVStore {
+	return s.db
+}

service/storage.go

@@ -0,0 +1,20 @@
+package service
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/metadata"
+	"git.lumeweb.com/LumeWeb/libs5-go/storage"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+)
+
+type StorageService interface {
+	SetProviderStore(store storage.ProviderStore)
+	ProviderStore() storage.ProviderStore
+	GetCachedStorageLocations(hash *encoding.Multihash, kinds []types.StorageLocationType, local bool) (map[string]storage.StorageLocation, error)
+	AddStorageLocation(hash *encoding.Multihash, nodeId *encoding.NodeId, location storage.StorageLocation, message []byte) error
+	DownloadBytesByHash(hash *encoding.Multihash) ([]byte, error)
+	DownloadBytesByCID(cid *encoding.CID) ([]byte, error)
+	GetMetadataByCID(cid *encoding.CID) (metadata.Metadata, error)
+	ParseMetadata(bytes []byte, cid *encoding.CID) (metadata.Metadata, error)
+	Service
+}

service/vote.go

@@ -0,0 +1,79 @@
+package service
+
+import (
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+var (
+	_ NodeVotes             = (*NodeVotesImpl)(nil)
+	_ msgpack.CustomDecoder = (*NodeVotesImpl)(nil)
+	_ msgpack.CustomEncoder = (*NodeVotesImpl)(nil)
+)
+
+type NodeVotesImpl struct {
+	good int
+	bad  int
+}
+
+func NewNodeVotes() NodeVotes {
+	return &NodeVotesImpl{
+		good: 0,
+		bad:  0,
+	}
+}
+
+func (n *NodeVotesImpl) Good() int {
+	return n.good
+}
+
+func (n *NodeVotesImpl) Bad() int {
+	return n.bad
+}
+
+func (n NodeVotesImpl) EncodeMsgpack(enc *msgpack.Encoder) error {
+	err := enc.EncodeInt(int64(n.good))
+	if err != nil {
+		return err
+	}
+
+	err = enc.EncodeInt(int64(n.bad))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (n *NodeVotesImpl) DecodeMsgpack(dec *msgpack.Decoder) error {
+	good, err := dec.DecodeInt()
+	if err != nil {
+		return err
+	}
+
+	bad, err := dec.DecodeInt()
+	if err != nil {
+		return err
+	}
+
+	n.good = good
+	n.bad = bad
+
+	return nil
+}
+
+func (n *NodeVotesImpl) Upvote() {
+	n.good++
+}
+
+func (n *NodeVotesImpl) Downvote() {
+	n.bad++
+}
+
+type NodeVotes interface {
+	msgpack.CustomEncoder
+	msgpack.CustomDecoder
+	Good() int
+	Bad() int
+	Upvote()
+	Downvote()
+}

storage/location.go

@@ -0,0 +1,73 @@
+package storage
+
+type StorageLocation interface {
+	BytesURL() string
+	OutboardBytesURL() string
+	String() string
+	ProviderMessage() []byte
+	Type() int
+	Parts() []string
+	BinaryParts() [][]byte
+	Expiry() int64
+	SetProviderMessage(msg []byte)
+	SetType(t int)
+	SetParts(p []string)
+	SetBinaryParts(bp [][]byte)
+	SetExpiry(e int64)
+}
+
+func (s *StorageLocationImpl) Type() int {
+	return s.kind
+}
+
+func (s *StorageLocationImpl) Parts() []string {
+	return s.parts
+}
+
+func (s *StorageLocationImpl) BinaryParts() [][]byte {
+	return s.binaryParts
+}
+
+func (s *StorageLocationImpl) Expiry() int64 {
+	return s.expiry
+}
+
+func (s *StorageLocationImpl) SetType(t int) {
+	s.kind = t
+}
+
+func (s *StorageLocationImpl) SetParts(p []string) {
+	s.parts = p
+}
+
+func (s *StorageLocationImpl) SetBinaryParts(bp [][]byte) {
+	s.binaryParts = bp
+}
+
+func (s *StorageLocationImpl) SetExpiry(e int64) {
+	s.expiry = e
+}
+
+func (s *StorageLocationImpl) SetProviderMessage(msg []byte) {
+	s.providerMessage = msg
+}
+
+func (s *StorageLocationImpl) ProviderMessage() []byte {
+	return s.providerMessage
+}
+
+func NewStorageLocation(Type int, Parts []string, Expiry int64) StorageLocation {
+	return &StorageLocationImpl{
+		kind:   Type,
+		parts:  Parts,
+		expiry: Expiry,
+	}
+}
+
+type StorageLocationImpl struct {
+	kind            int
+	parts           []string
+	binaryParts     [][]byte
+	expiry          int64
+	providerMessage []byte
+}

storage/p2p.go

@@ -0,0 +1,51 @@
+package storage
+
+import (
+	ed25519p "crypto/ed25519"
+	"git.lumeweb.com/LumeWeb/libs5-go/ed25519"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"git.lumeweb.com/LumeWeb/libs5-go/utils"
+)
+
+func PrepareProvideMessage(identity *ed25519.KeyPairEd25519, hash *encoding.Multihash, location StorageLocation) []byte {
+	// Initialize the list with the record type.
+	list := []byte{byte(types.RecordTypeStorageLocation)}
+
+	// Append the full bytes of the hash.
+	list = append(list, hash.FullBytes()...)
+
+	// Append the location type.
+	list = append(list, byte(location.Type()))
+
+	// Append the expiry time of the location, encoded as 4 bytes.
+	list = append(list, utils.EncodeEndian(uint64(location.Expiry()), 4)...)
+
+	// Append the number of parts in the location.
+	list = append(list, byte(len(location.Parts())))
+
+	// Iterate over each part in the location.
+	for _, part := range location.Parts() {
+		// Convert part to bytes.
+		bytes := []byte(part)
+
+		// Encode the length of the part as 4 bytes and append.
+		list = append(list, utils.EncodeEndian(uint64(len(bytes)), 2)...)
+
+		// Append the actual part bytes.
+		list = append(list, bytes...)
+	}
+
+	// Append a null byte at the end of the list.
+	list = append(list, 0)
+
+	// Sign the list using the node's private key.
+	signature := ed25519p.Sign(identity.ExtractBytes(), list)
+
+	// Append the public key and signature to the list.
+	finalList := append(list, identity.PublicKey()...)
+	finalList = append(finalList, signature...)
+
+	// Return the final byte slice.
+	return finalList
+}

storage/provider/provider.go

@@ -0,0 +1,250 @@
+package provider
+
+import (
+	"bytes"
+	"fmt"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/service"
+	"git.lumeweb.com/LumeWeb/libs5-go/storage"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+	"github.com/samber/lo"
+	"go.uber.org/zap"
+	"sync"
+	"time"
+)
+
+var _ storage.StorageLocationProvider = (*StorageLocationProviderImpl)(nil)
+
+type StorageLocationProviderImpl struct {
+	services        service.Services
+	hash            *encoding.Multihash
+	types           []types.StorageLocationType
+	timeoutDuration time.Duration
+	availableNodes  []*encoding.NodeId
+	uris            map[string]storage.StorageLocation
+	timeout         time.Time
+	isTimedOut      bool
+	isWaitingForUri bool
+	mutex           sync.Mutex
+	logger          *zap.Logger
+	excludeNodes    []*encoding.NodeId
+}
+
+func (s *StorageLocationProviderImpl) Start() error {
+	var err error
+
+	s.uris, err = s.services.Storage().GetCachedStorageLocations(s.hash, s.types, true)
+	if err != nil {
+		return err
+	}
+	s.mutex.Lock()
+	s.availableNodes = make([]*encoding.NodeId, 0, len(s.uris))
+	for k := range s.uris {
+		nodeId, err := encoding.DecodeNodeId(k)
+		if err != nil {
+			continue
+		}
+
+		if containsNode(s.excludeNodes, nodeId) {
+			continue
+		}
+
+		s.availableNodes = append(s.availableNodes, nodeId)
+	}
+
+	s.availableNodes, err = s.services.P2P().SortNodesByScore(s.availableNodes)
+	if err != nil {
+		s.mutex.Unlock()
+		return err
+	}
+
+	s.timeout = time.Now().Add(s.timeoutDuration)
+	s.isTimedOut = false
+	s.mutex.Unlock()
+
+	go func() {
+		requestSent := false
+
+		for {
+			s.mutex.Lock()
+			if time.Now().After(s.timeout) {
+				s.isTimedOut = true
+				s.mutex.Unlock()
+				break
+			}
+
+			newUris, err := s.services.Storage().GetCachedStorageLocations(s.hash, s.types, false)
+			if err != nil {
+				s.mutex.Unlock()
+				break
+			}
+
+			if len(s.availableNodes) == 0 && len(newUris) < 2 && !requestSent {
+				s.logger.Debug("Sending hash request")
+				err := s.services.P2P().SendHashRequest(s.hash, s.types)
+				if err != nil {
+					s.logger.Error("Error sending hash request", zap.Error(err))
+					continue
+				}
+				requestSent = true
+			}
+
+			hasNewNode := false
+			for k, v := range newUris {
+				if _, exists := s.uris[k]; !exists || s.uris[k] != v {
+					s.uris[k] = v
+					nodeId, err := encoding.DecodeNodeId(k)
+					if err != nil {
+						s.logger.Error("Error decoding node id", zap.Error(err))
+						continue
+					}
+					if containsNode(s.excludeNodes, nodeId) && requestSent {
+						continue
+					}
+					if !containsNode(s.availableNodes, nodeId) {
+						s.availableNodes = append(s.availableNodes, nodeId)
+						hasNewNode = true
+					}
+				}
+			}
+
+			if hasNewNode {
+				score, err := s.services.P2P().SortNodesByScore(s.availableNodes)
+				if err != nil {
+					s.logger.Error("Error sorting nodes by score", zap.Error(err))
+				} else {
+					s.availableNodes = score
+				}
+			}
+			s.mutex.Unlock()
+
+			time.Sleep(10 * time.Millisecond)
+		}
+	}()
+	return nil
+}
+func (s *StorageLocationProviderImpl) Next() (storage.SignedStorageLocation, error) {
+	s.timeout = time.Now().Add(s.timeoutDuration)
+
+	for {
+		if len(s.availableNodes) > 0 {
+			s.isWaitingForUri = false
+			nodeId := s.availableNodes[0]
+			s.availableNodes = s.availableNodes[1:]
+
+			nodIdStr, err := nodeId.ToString()
+			if err != nil {
+				return nil, err
+			}
+
+			uri, exists := s.uris[nodIdStr]
+			if !exists {
+				s.logger.Error("Could not find uri for node id", zap.String("nodeId", nodIdStr))
+				continue
+			}
+
+			return storage.NewSignedStorageLocation(nodeId, uri), nil
+		}
+
+		s.isWaitingForUri = true
+		if s.isTimedOut {
+			hashStr, err := s.hash.ToString()
+			if err != nil {
+				return nil, err
+			}
+			return nil, fmt.Errorf("Could not download raw file: Timed out after %s %s", s.timeoutDuration.String(), hashStr)
+		}
+
+		time.Sleep(10 * time.Millisecond) // Replace with a proper wait/notify mechanism if applicable
+	}
+}
+
+func (s *StorageLocationProviderImpl) All() ([]storage.SignedStorageLocation, error) {
+	s.timeout = time.Now().Add(s.timeoutDuration)
+
+	for {
+		if len(s.availableNodes) > 0 {
+			s.isWaitingForUri = false
+
+			return lo.FilterMap[*encoding.NodeId, storage.SignedStorageLocation](s.availableNodes, func(nodeId *encoding.NodeId, index int) (storage.SignedStorageLocation, bool) {
+				nodIdStr, err := nodeId.ToString()
+
+				if err != nil {
+					s.logger.Error("Error decoding node id", zap.Error(err))
+					return nil, false
+				}
+
+				uri, exists := s.uris[nodIdStr]
+
+				if !exists {
+					s.logger.Error("Could not find uri for node id", zap.String("nodeId", nodIdStr))
+					return nil, false
+				}
+
+				return storage.NewSignedStorageLocation(nodeId, uri), true
+			}), nil
+		}
+
+		s.isWaitingForUri = true
+		if s.isTimedOut {
+			hashStr, err := s.hash.ToString()
+			if err != nil {
+				return nil, err
+			}
+			return nil, fmt.Errorf("Could not download raw file: Timed out after %s %s", s.timeoutDuration.String(), hashStr)
+		}
+
+		time.Sleep(10 * time.Millisecond) // Replace with a proper wait/notify mechanism if applicable
+	}
+}
+
+func (s *StorageLocationProviderImpl) Upvote(uri storage.SignedStorageLocation) error {
+	err := s.services.P2P().UpVote(uri.NodeId())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *StorageLocationProviderImpl) Downvote(uri storage.SignedStorageLocation) error {
+	err := s.services.P2P().DownVote(uri.NodeId())
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func NewStorageLocationProvider(params StorageLocationProviderParams) *StorageLocationProviderImpl {
+	if params.LocationTypes == nil {
+		params.LocationTypes = []types.StorageLocationType{
+			types.StorageLocationTypeFull,
+		}
+	}
+
+	return &StorageLocationProviderImpl{
+		services:        params.Services,
+		hash:            params.Hash,
+		types:           params.LocationTypes,
+		timeoutDuration: 60 * time.Second,
+		uris:            make(map[string]storage.StorageLocation),
+		logger:          params.Logger,
+		excludeNodes:    params.ExcludeNodes,
+	}
+}
+func containsNode(slice []*encoding.NodeId, item *encoding.NodeId) bool {
+	for _, v := range slice {
+		if bytes.Equal(v.Bytes(), item.Bytes()) {
+			return true
+		}
+	}
+	return false
+}
+
+type StorageLocationProviderParams struct {
+	Services      service.Services
+	Hash          *encoding.Multihash
+	LocationTypes []types.StorageLocationType
+	ExcludeNodes  []*encoding.NodeId
+	service.ServiceParams
+}

storage/provider_store.go

@@ -0,0 +1,11 @@
+package storage
+
+import (
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"git.lumeweb.com/LumeWeb/libs5-go/types"
+)
+
+type ProviderStore interface {
+	CanProvide(hash *encoding.Multihash, kind []types.StorageLocationType) bool
+	Provide(hash *encoding.Multihash, kind []types.StorageLocationType) (StorageLocation, error)
+}

storage/signed_location.go

@@ -0,0 +1,9 @@
+package storage
+
+import "git.lumeweb.com/LumeWeb/libs5-go/encoding"
+
+type SignedStorageLocation interface {
+	String() string
+	NodeId() *encoding.NodeId
+	Location() StorageLocation
+}

storage/storage.go

@@ -0,0 +1,130 @@
+package storage
+
+import (
+	"fmt"
+	"git.lumeweb.com/LumeWeb/libs5-go/encoding"
+	"github.com/vmihailenco/msgpack/v5"
+	"strconv"
+	"time"
+)
+
+var (
+	_ msgpack.CustomDecoder = (*StorageLocationMap)(nil)
+	_ msgpack.CustomEncoder = (*StorageLocationMap)(nil)
+	_ StorageLocation       = (*StorageLocationImpl)(nil)
+	_ SignedStorageLocation = (*SignedStorageLocationImpl)(nil)
+)
+
+type StorageLocationMap map[int]NodeStorage
+type NodeStorage map[string]NodeDetailsStorage
+type NodeDetailsStorage map[int]interface{}
+
+func (s *StorageLocationImpl) BytesURL() string {
+	return s.parts[0]
+}
+
+func (s *StorageLocationImpl) OutboardBytesURL() string {
+	if len(s.parts) == 1 {
+		return s.parts[0] + ".obao"
+	}
+	return s.parts[1]
+}
+
+func (s *StorageLocationImpl) String() string {
+	expiryDate := time.Unix(s.expiry, 0)
+	return "StorageLocationImpl(" + strconv.Itoa(s.Type()) + ", " + fmt.Sprint(s.parts) + ", expiry: " + expiryDate.Format(time.RFC3339) + ")"
+}
+
+type SignedStorageLocationImpl struct {
+	nodeID   *encoding.NodeId
+	location StorageLocation
+}
+
+func NewSignedStorageLocation(NodeID *encoding.NodeId, Location StorageLocation) SignedStorageLocation {
+	return &SignedStorageLocationImpl{
+		nodeID:   NodeID,
+		location: Location,
+	}
+}
+
+func (ssl *SignedStorageLocationImpl) String() string {
+	nodeString, _ := ssl.nodeID.ToString()
+
+	if nodeString == "" {
+		nodeString = "failed to decode node id"
+	}
+
+	return "SignedStorageLocationImpl(" + ssl.location.String() + ", " + nodeString + ")"
+}
+
+func (ssl *SignedStorageLocationImpl) NodeId() *encoding.NodeId {
+	return ssl.nodeID
+}
+func (ssl *SignedStorageLocationImpl) Location() StorageLocation {
+	return ssl.location
+}
+
+func (s *StorageLocationMap) DecodeMsgpack(dec *msgpack.Decoder) error {
+	if *s == nil {
+		*s = make(StorageLocationMap)
+	}
+
+	// Decode directly into a temp map
+	temp := make(map[int]map[string]map[int]interface{})
+	err := dec.Decode(&temp)
+	if err != nil {
+		return fmt.Errorf("error decoding msgpack: %w", err)
+	}
+
+	// Convert temp map to StorageLocationMap
+	for k, v := range temp {
+		nodeStorage, exists := (*s)[k]
+		if !exists {
+			nodeStorage = make(NodeStorage, len(v)) // preallocate if size is known
+			(*s)[k] = nodeStorage
+		}
+
+		for nk, nv := range v {
+			nodeDetailsStorage, exists := nodeStorage[nk]
+			if !exists {
+				nodeDetailsStorage = make(NodeDetailsStorage, len(nv)) // preallocate if size is known
+				nodeStorage[nk] = nodeDetailsStorage
+			}
+
+			for ndk, ndv := range nv {
+				nodeDetailsStorage[ndk] = ndv
+			}
+		}
+	}
+
+	return nil
+}
+
+func (s StorageLocationMap) EncodeMsgpack(enc *msgpack.Encoder) error {
+	// Create a temporary map to hold the encoded data
+	tempMap := make(map[int]map[string]map[int]interface{})
+
+	// Populate the temporary map with data from storageLocationMap
+	for storageKey, nodeStorages := range s {
+		tempNodeStorages := make(map[string]map[int]interface{})
+		for nodeId, nodeDetails := range nodeStorages {
+			tempNodeStorages[nodeId] = nodeDetails
+		}
+		tempMap[storageKey] = tempNodeStorages
+	}
+
+	// Encode the temporary map using MessagePack
+	return enc.Encode(tempMap)
+}
+
+func NewStorageLocationMap() StorageLocationMap {
+	return StorageLocationMap{}
+}
+
+type StorageLocationProvider interface {
+	Start() error
+	Next() (SignedStorageLocation, error)
+	All() ([]SignedStorageLocation, error)
+	Upvote(uri SignedStorageLocation) error
+	Downvote(uri SignedStorageLocation) error
+}

structs/map.go

@@ -0,0 +1,156 @@
+package structs
+
+import (
+	"github.com/emirpasic/gods/maps"
+	"github.com/emirpasic/gods/maps/hashmap"
+	"log"
+	"sync"
+)
+
+var _ maps.Map = (*MapImpl)(nil)
+
+type Map interface {
+	GetInt(key interface{}) (value *int)
+	GetUInt(key interface{}) (value *uint)
+	GetString(key interface{}) (value *string)
+	PutInt(key interface{}, value int)
+	PutUInt(key interface{}, value uint)
+	Contains(value interface{}) bool
+	maps.Map
+}
+
+type MapImpl struct {
+	*hashmap.Map
+	mutex *sync.RWMutex
+}
+
+func NewMap() Map {
+	return &MapImpl{
+		Map:   hashmap.New(),
+		mutex: &sync.RWMutex{},
+	}
+}
+
+func (m *MapImpl) Get(key interface{}) (value interface{}, found bool) {
+	m.mutex.RLock()
+	defer m.mutex.RUnlock()
+	return m.Map.Get(key)
+}
+
+func (m *MapImpl) GetInt(key interface{}) (value *int) {
+	val, found := m.Get(key)
+
+	if !found {
+		return nil
+	}
+
+	if intValue, ok := val.(int); ok {
+		value = &intValue
+	} else {
+		log.Fatalf("value is not an int: %v", val)
+	}
+
+	return value
+}
+
+func (m *MapImpl) GetUInt(key interface{}) (value *uint) {
+	val, found := m.Get(key)
+
+	if !found {
+		return nil
+	}
+
+	if intValue, ok := val.(uint); ok {
+		value = &intValue
+	} else {
+		log.Fatalf("value is not an uint: %v", val)
+	}
+
+	return value
+}
+
+func (m *MapImpl) GetString(key interface{}) (value *string) {
+	val, found := m.Get(key)
+
+	if !found {
+		return nil
+	}
+
+	if _, ok := val.(string); ok {
+		value = val.(*string)
+	} else {
+		log.Fatalf("value is not a string: %v", value)
+	}
+
+	return
+}
+
+func (m *MapImpl) Put(key interface{}, value interface{}) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	m.Map.Put(key, value)
+}
+
+func (m *MapImpl) PutInt(key interface{}, value int) {
+	m.Put(key, value)
+}
+
+func (m *MapImpl) PutUInt(key interface{}, value uint) {
+	m.Put(key, value)
+}
+
+func (m *MapImpl) Remove(key interface{}) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	m.Map.Remove(key)
+}
+
+func (m *MapImpl) Keys() []interface{} {
+	m.mutex.RLock()
+	defer m.mutex.RUnlock()
+	return m.Map.Keys()
+}
+
+func (m *MapImpl) Values() []interface{} {
+	m.mutex.RLock()
+	defer m.mutex.RUnlock()
+	return m.Map.Values()
+}
+
+func (m *MapImpl) Size() int {
+	m.mutex.RLock()
+	defer m.mutex.RUnlock()
+	return m.Map.Size()
+}
+
+func (m *MapImpl) Empty() bool {
+	m.mutex.RLock()
+	defer m.mutex.RUnlock()
+	return m.Map.Empty()
+}
+
+func (m *MapImpl) Clear() {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	m.Map.Clear()
+}
+
+func (m *MapImpl) String() string {
+	m.mutex.RLock()
+	defer m.mutex.RUnlock()
+	return m.Map.String()
+}
+
+func (m *MapImpl) GetKey(value interface{}) (key interface{}, found bool) {
+	m.mutex.RLock()
+	defer m.mutex.RUnlock()
+	return m.Map.Get(value)
+}
+
+func (m *MapImpl) Contains(value interface{}) bool {
+	m.mutex.RLock()
+	defer m.mutex.RUnlock()
+	_, has := m.Map.Get(value)
+
+	return has
+}

structs/set.go

@@ -0,0 +1,39 @@
+package structs
+
+import (
+	"github.com/emirpasic/gods/sets"
+	"github.com/emirpasic/gods/sets/hashset"
+	"sync"
+)
+
+var _ sets.Set = (*SetImpl)(nil)
+
+type SetImpl struct {
+	*hashset.Set
+	mutex *sync.RWMutex
+}
+
+func NewSet() *SetImpl {
+	return &SetImpl{
+		Set:   hashset.New(),
+		mutex: &sync.RWMutex{},
+	}
+}
+
+func (s *SetImpl) Add(items ...interface{}) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	s.Set.Add(items...)
+}
+
+func (s *SetImpl) Remove(items ...interface{}) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	s.Set.Remove(items...)
+}
+
+func (s *SetImpl) Contains(items ...interface{}) bool {
+	s.mutex.RLock()
+	defer s.mutex.RUnlock()
+	return s.Set.Contains(items...)
+}

types/cid.go

@@ -0,0 +1,27 @@
+package types
+
+type CIDType int
+
+const (
+	CIDTypeRaw              CIDType = 0x26
+	CIDTypeMetadataMedia    CIDType = 0xc5
+	CIDTypeMetadataWebapp   CIDType = 0x59
+	CIDTypeResolver         CIDType = 0x25
+	CIDTypeUserIdentity     CIDType = 0x77
+	CIDTypeBridge           CIDType = 0x3a
+	CIDTypeEncryptedStatic  CIDType = 0xae
+	CIDTypeEncryptedDynamic CIDType = 0xad
+	CIDTypeDirectory        CIDType = 0x5d
+)
+
+var CIDTypeMap = map[CIDType]string{
+	CIDTypeRaw:              "Raw",
+	CIDTypeMetadataMedia:    "MetadataMedia",
+	CIDTypeMetadataWebapp:   "MetadataWebapp",
+	CIDTypeResolver:         "Resolver",
+	CIDTypeUserIdentity:     "UserIdentity",
+	CIDTypeBridge:           "Bridge",
+	CIDTypeEncryptedStatic:  "EncryptedStatic",
+	CIDTypeEncryptedDynamic: "EncryptedDynamic",
+	CIDTypeDirectory:        "Directory",
+}

types/feature.go

@@ -0,0 +1,3 @@
+package types
+
+const SupportedFeatures = 0x03

types/hash.go

@@ -0,0 +1,13 @@
+package types
+
+type HashType int
+
+const (
+	HashTypeBlake3  HashType = 0x1f
+	HashTypeEd25519 HashType = 0xed
+)
+
+var HashTypeMap = map[HashType]string{
+	HashTypeBlake3:  "Blake3",
+	HashTypeEd25519: "Ed25519",
+}

types/meta.go

@@ -0,0 +1,71 @@
+package types
+
+type MetadataExtension int
+
+const (
+	MetadataExtensionLicenses           MetadataExtension = 0x0B
+	MetadataExtensionDonationKeys       MetadataExtension = 0x0C
+	MetadataExtensionWikidataClaims     MetadataExtension = 0x0D
+	MetadataExtensionLanguages          MetadataExtension = 0x0E
+	MetadataExtensionSourceUris         MetadataExtension = 0x0F
+	MetadataExtensionUpdateCID          MetadataExtension = 0x10
+	MetadataExtensionPreviousVersions   MetadataExtension = 0x11
+	MetadataExtensionTimestamp          MetadataExtension = 0x12
+	MetadataExtensionTags               MetadataExtension = 0x13
+	MetadataExtensionCategories         MetadataExtension = 0x14
+	MetadataExtensionViewTypes          MetadataExtension = 0x15
+	MetadataExtensionBasicMediaMetadata MetadataExtension = 0x16
+	MetadataExtensionBridge             MetadataExtension = 0x17
+	MetadataExtensionOriginalTimestamp  MetadataExtension = 0x18
+	MetadataExtensionRoutingHints       MetadataExtension = 0x19
+)
+
+var MetadataMap = map[MetadataExtension]string{
+	MetadataExtensionLicenses:           "MetadataExtensionLicenses",
+	MetadataExtensionDonationKeys:       "MetadataExtensionDonationKeys",
+	MetadataExtensionWikidataClaims:     "MetadataExtensionWikidataClaims",
+	MetadataExtensionLanguages:          "MetadataExtensionLanguages",
+	MetadataExtensionSourceUris:         "MetadataExtensionSourceUris",
+	MetadataExtensionUpdateCID:          "MetadataExtensionUpdateCID",
+	MetadataExtensionPreviousVersions:   "MetadataExtensionPreviousVersions",
+	MetadataExtensionTimestamp:          "MetadataExtensionTimestamp",
+	MetadataExtensionTags:               "MetadataExtensionTags",
+	MetadataExtensionCategories:         "MetadataExtensionCategories",
+	MetadataExtensionViewTypes:          "MetadataExtensionViewTypes",
+	MetadataExtensionBasicMediaMetadata: "MetadataExtensionBasicMediaMetadata",
+	MetadataExtensionBridge:             "MetadataExtensionBridge",
+	MetadataExtensionOriginalTimestamp:  "MetadataExtensionOriginalTimestamp",
+	MetadataExtensionRoutingHints:       "MetadataExtensionRoutingHints",
+}
+
+const MetadataMagicByte = 0x5f
+
+type MetadataType uint8
+
+const (
+	MetadataTypeMedia        MetadataType = 0x02
+	MetadataTypeWebApp       MetadataType = 0x03
+	MetadataTypeDirectory    MetadataType = 0x04
+	MetadataTypeProof        MetadataType = 0x05
+	MetadataTypeUserIdentity MetadataType = 0x07
+)
+
+var MetadataTypeMap = map[string]MetadataType{
+	"Media":        MetadataTypeMedia,
+	"WebApp":       MetadataTypeWebApp,
+	"Directory":    MetadataTypeDirectory,
+	"Proof":        MetadataTypeProof,
+	"UserIdentity": MetadataTypeUserIdentity,
+}
+
+type MetadataProofType uint8
+
+const (
+	MetadataProofTypeSignature MetadataProofType = 0x01
+	MetadataProofTypeTimestamp MetadataProofType = 0x02
+)
+
+var MetadataProofTypeMap = map[string]MetadataProofType{
+	"Signature": MetadataProofTypeSignature,
+	"Timestamp": MetadataProofTypeTimestamp,
+}

types/parent_link.go

@@ -0,0 +1,15 @@
+package types
+
+type ParentLinkType int
+
+const (
+	ParentLinkTypeUserIdentity ParentLinkType = 0x01
+	ParentLinkTypeBoard        ParentLinkType = 0x05
+	ParentLinkTypeBridgeUser   ParentLinkType = 0x0A
+)
+
+var ParentLinkTypeMap = map[ParentLinkType]string{
+	ParentLinkTypeUserIdentity: "UserIdentity",
+	ParentLinkTypeBoard:        "Board",
+	ParentLinkTypeBridgeUser:   "BridgeUser",
+}

types/protocol.go

@@ -0,0 +1,27 @@
+package types
+
+type ProtocolMethod int
+
+const (
+	ProtocolMethodHandshakeOpen ProtocolMethod = 0x1
+	ProtocolMethodHandshakeDone ProtocolMethod = 0x2
+	ProtocolMethodSignedMessage ProtocolMethod = 0xA
+	ProtocolMethodHashQuery     ProtocolMethod = 0x4
+	ProtocolMethodAnnouncePeers ProtocolMethod = 0x8
+	ProtocolMethodRegistryQuery ProtocolMethod = 0xD
+	RecordTypeStorageLocation   ProtocolMethod = 0x05
+	RecordTypeStreamEvent       ProtocolMethod = 0x09
+	RecordTypeRegistryEntry     ProtocolMethod = 0x07
+)
+
+var ProtocolMethodMap = map[ProtocolMethod]string{
+	ProtocolMethodHandshakeOpen: "HandshakeOpen",
+	ProtocolMethodHandshakeDone: "IsHandshakeDone",
+	ProtocolMethodSignedMessage: "SignedMessage",
+	ProtocolMethodHashQuery:     "HashQuery",
+	ProtocolMethodAnnouncePeers: "AnnouncePeers",
+	ProtocolMethodRegistryQuery: "RegistryQuery",
+	RecordTypeStorageLocation:   "StorageLocation",
+	RecordTypeStreamEvent:       "StreamEvent",
+	RecordTypeRegistryEntry:     "RegistryEntry",
+}

types/registry.go

@@ -0,0 +1,15 @@
+package types
+
+type RegistryType int
+
+const (
+	RegistryTypeCID          RegistryType = 0x5a
+	RegistryTypeEncryptedCID RegistryType = 0x5e
+)
+
+var RegistryTypeMap = map[RegistryType]string{
+	RegistryTypeCID:          "CID",
+	RegistryTypeEncryptedCID: "EncryptedCID",
+}
+
+const RegistryMaxDataSize = 64

types/storage.go

@@ -0,0 +1,17 @@
+package types
+
+type StorageLocationType int
+
+const (
+	StorageLocationTypeArchive StorageLocationType = 0
+	StorageLocationTypeFile    StorageLocationType = 3
+	StorageLocationTypeFull    StorageLocationType = 5
+	StorageLocationTypeBridge  StorageLocationType = 7
+)
+
+var StorageLocationTypeMap = map[StorageLocationType]string{
+	StorageLocationTypeArchive: "Archive",
+	StorageLocationTypeFile:    "File",
+	StorageLocationTypeFull:    "Full",
+	StorageLocationTypeBridge:  "Bridge",
+}

utils/array.go

@@ -0,0 +1,92 @@
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"github.com/vmihailenco/msgpack/v5"
+	"net/url"
+)
+
+func EncodeMsgpackArray(enc *msgpack.Encoder, array interface{}) error {
+	switch v := array.(type) {
+	case []*url.URL:
+		// Handle []*url.URL slice
+		err := enc.EncodeInt(int64(len(v)))
+		if err != nil {
+			return err
+		}
+		for _, item := range v {
+			err = enc.EncodeString(item.String())
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	default:
+		// Handle generic case
+		arr, ok := array.([]interface{})
+		if !ok {
+			return errors.New("unsupported type for EncodeMsgpackArray")
+		}
+		err := enc.EncodeInt(int64(len(arr)))
+		if err != nil {
+			return err
+		}
+		for _, item := range arr {
+			err = enc.Encode(item)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+func DecodeMsgpackArray(dec *msgpack.Decoder) ([]interface{}, error) {
+	arrayLen, err := dec.DecodeInt()
+	if err != nil {
+		return nil, err
+	}
+
+	array := make([]interface{}, arrayLen)
+
+	for i := 0; i < int(arrayLen); i++ {
+		item, err := dec.DecodeInterface()
+		if err != nil {
+			return nil, err
+		}
+
+		array[i] = item
+	}
+
+	return array, nil
+}
+func DecodeMsgpackURLArray(dec *msgpack.Decoder) ([]*url.URL, error) {
+	arrayLen, err := dec.DecodeInt()
+	if err != nil {
+		return nil, err
+	}
+
+	urlArray := make([]*url.URL, arrayLen)
+
+	for i := 0; i < int(arrayLen); i++ {
+		item, err := dec.DecodeInterface()
+		if err != nil {
+			return nil, err
+		}
+
+		// Type assert each item to *url.URL
+		urlItem, ok := item.(string)
+		if !ok {
+			// Handle the case where the item is not a *url.URL
+			return nil, fmt.Errorf("expected string, got %T", item)
+		}
+
+		urlArray[i], err = url.Parse(urlItem)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return urlArray, nil
+}

utils/bytes.go

@@ -0,0 +1,18 @@
+package utils
+
+import "bytes"
+
+func ConcatBytes(slices ...[]byte) []byte {
+	return bytes.Join(slices, nil)
+}
+
+func HashCode(bytes []byte) int {
+	if len(bytes) < 4 {
+		return 0
+	}
+
+	return int(bytes[0]) |
+		int(bytes[1])<<8 |
+		int(bytes[2])<<16 |
+		int(bytes[3])<<24
+}

utils/endian.go

@@ -0,0 +1,22 @@
+package utils
+
+func EncodeEndian(value uint64, length int) []byte {
+	res := make([]byte, length)
+
+	for i := 0; i < length; i++ {
+		res[i] = byte(value & 0xff)
+		value = value >> 8
+	}
+	return res
+}
+func DecodeEndian(bytes []byte) uint64 {
+	var total uint64 = 0
+	var multiplier uint64 = 1
+
+	for _, b := range bytes {
+		total += uint64(b) * multiplier
+		multiplier *= 256
+	}
+
+	return total
+}