package service import ( "bytes" "context" "errors" "fmt" "git.lumeweb.com/LumeWeb/libs5-go/ed25519" "git.lumeweb.com/LumeWeb/libs5-go/encoding" "git.lumeweb.com/LumeWeb/libs5-go/net" "git.lumeweb.com/LumeWeb/libs5-go/protocol" "git.lumeweb.com/LumeWeb/libs5-go/protocol/base" "git.lumeweb.com/LumeWeb/libs5-go/protocol/signed" "git.lumeweb.com/LumeWeb/libs5-go/storage" "git.lumeweb.com/LumeWeb/libs5-go/structs" "git.lumeweb.com/LumeWeb/libs5-go/types" "git.lumeweb.com/LumeWeb/libs5-go/utils" "github.com/vmihailenco/msgpack/v5" bolt "go.etcd.io/bbolt" "go.uber.org/zap" "net/url" "sort" "sync" "time" ) var _ Service = (*P2PService)(nil) var _ storage.StorageLocationProviderP2PService = (*P2PService)(nil) var _ P2PServiceInterface = (*P2PService)(nil) var ( errUnsupportedProtocol = errors.New("unsupported protocol") errConnectionIdMissingNodeID = errors.New("connection id missing node id") ) const nodeBucketName = "nodes" type P2PServiceInterface interface { SelfConnectionUris() []*url.URL Peers() structs.Map ConnectToNode(connectionUris []*url.URL, retried bool, fromPeer net.Peer) error OnNewPeer(peer net.Peer, verifyId bool) error GetNodeScore(nodeId *encoding.NodeId) (float64, error) SortNodesByScore(nodes []*encoding.NodeId) ([]*encoding.NodeId, error) SignMessageSimple(message []byte) ([]byte, error) AddPeer(peer net.Peer) error SendPublicPeersToPeer(peer net.Peer, peersToSend []net.Peer) error SendHashRequest(hash *encoding.Multihash, kinds []types.StorageLocationType) error UpVote(nodeId *encoding.NodeId) error DownVote(nodeId *encoding.NodeId) error NodeId() *encoding.NodeId WaitOnConnectedPeers() ConnectionTracker() *sync.WaitGroup NetworkId() string HashQueryRoutingTable() structs.Map } type P2PService struct { nodeKeyPair *ed25519.KeyPairEd25519 localNodeID *encoding.NodeId networkID string nodesBucket *bolt.Bucket inited bool reconnectDelay structs.Map peers structs.Map peersPending structs.Map selfConnectionUris []*url.URL outgoingPeerBlocklist structs.Map incomingPeerBlockList structs.Map incomingIPBlocklist structs.Map outgoingPeerFailures structs.Map maxOutgoingPeerFailures uint connections sync.WaitGroup hashQueryRoutingTable structs.Map ServiceBase } func NewP2P(params ServiceParams) *P2PService { uri, err := url.Parse(fmt.Sprintf("wss://%s:%d/s5/p2p", params.Config.HTTP.API.Domain, params.Config.HTTP.API.Port)) if err != nil { params.Logger.Fatal("failed to parse HTTP API URL", zap.Error(err)) } service := &P2PService{ nodeKeyPair: params.Config.KeyPair, networkID: params.Config.P2P.Network, inited: false, reconnectDelay: structs.NewMap(), peers: structs.NewMap(), peersPending: structs.NewMap(), selfConnectionUris: []*url.URL{uri}, outgoingPeerBlocklist: structs.NewMap(), incomingPeerBlockList: structs.NewMap(), incomingIPBlocklist: structs.NewMap(), outgoingPeerFailures: structs.NewMap(), maxOutgoingPeerFailures: params.Config.P2P.MaxOutgoingPeerFailures, hashQueryRoutingTable: structs.NewMap(), ServiceBase: ServiceBase{ logger: params.Logger, config: params.Config, db: params.Db, }, } return service } func (p *P2PService) SelfConnectionUris() []*url.URL { return p.selfConnectionUris } func (p *P2PService) Peers() structs.Map { return p.peers } func (p *P2PService) Start() error { config := p.config if len(config.P2P.Peers.Initial) > 0 { initialPeers := config.P2P.Peers.Initial for _, peer := range initialPeers { u, err := url.Parse(peer) if err != nil { return err } peer := peer go func() { err := p.ConnectToNode([]*url.URL{u}, false, nil) if err != nil { p.logger.Error("failed to connect to initial peer", zap.Error(err), zap.String("peer", peer)) } }() } } return nil } func (p *P2PService) Stop() error { return nil } func (p *P2PService) Init() error { if p.inited { return nil } p.localNodeID = encoding.NewNodeId(p.nodeKeyPair.PublicKey()) err := utils.CreateBucket(nodeBucketName, p.db) if err != nil { return err } p.inited = true return nil } func (p *P2PService) ConnectToNode(connectionUris []*url.URL, retried bool, fromPeer net.Peer) error { if !p.services.IsStarted() { return nil } unsupported, _ := url.Parse("http://0.0.0.0") unsupported.Scheme = "unsupported" var connectionUri *url.URL for _, uri := range connectionUris { if uri.Scheme == "ws" || uri.Scheme == "wss" { connectionUri = uri break } } if connectionUri == nil { for _, uri := range connectionUris { if uri.Scheme == "tcp" { connectionUri = uri break } } } if connectionUri == nil { connectionUri = unsupported } if connectionUri.Scheme == "unsupported" { return errUnsupportedProtocol } scheme := connectionUri.Scheme if connectionUri.User == nil { return errConnectionIdMissingNodeID } username := connectionUri.User.Username() id, err := encoding.DecodeNodeId(username) if err != nil { return err } idString, err := id.ToString() if err != nil { return err } if p.peersPending.Contains(idString) || p.peers.Contains(idString) { p.logger.Debug("already connected", zap.String("node", connectionUri.String())) return nil } if p.outgoingPeerBlocklist.Contains(idString) { p.logger.Debug("outgoing peer is on blocklist", zap.String("node", connectionUri.String())) var fromPeerId string if fromPeer != nil { blocked := false if fromPeer.Id() != nil { fromPeerId, err = fromPeer.Id().ToString() if err != nil { return err } if !p.incomingPeerBlockList.Contains(fromPeerId) { p.incomingPeerBlockList.Put(fromPeerId, true) blocked = true } } fromPeerIP := fromPeer.GetIP() if !p.incomingIPBlocklist.Contains(fromPeerIP) { p.incomingIPBlocklist.Put(fromPeerIP, true) blocked = true } err = fromPeer.EndForAbuse() if err != nil { return err } if blocked { p.logger.Debug("blocking peer for sending peer on blocklist", zap.String("node", connectionUri.String()), zap.String("peer", fromPeerId), zap.String("ip", fromPeerIP)) } } return nil } reconnectDelay := p.reconnectDelay.GetUInt(idString) if reconnectDelay == nil { delay := uint(1) reconnectDelay = &delay } if id.Equals(p.localNodeID) { return nil } p.logger.Debug("connect", zap.String("node", connectionUri.String())) socket, err := net.CreateTransportSocket(scheme, connectionUri) if err != nil { if retried { p.logger.Error("failed to connect, too many retries", zap.String("node", connectionUri.String()), zap.Error(err)) counter := uint(0) if p.outgoingPeerFailures.Contains(idString) { tmp := *p.outgoingPeerFailures.GetUInt(idString) counter = tmp } counter++ p.outgoingPeerFailures.PutUInt(idString, counter) if counter >= p.maxOutgoingPeerFailures { if fromPeer != nil { blocked := false var fromPeerId string if fromPeer.Id() != nil { fromPeerId, err = fromPeer.Id().ToString() if err != nil { return err } if !p.incomingPeerBlockList.Contains(fromPeerId) { p.incomingPeerBlockList.Put(fromPeerId, true) blocked = true } } fromPeerIP := fromPeer.GetIP() if !p.incomingIPBlocklist.Contains(fromPeerIP) { p.incomingIPBlocklist.Put(fromPeerIP, true) blocked = true } err = fromPeer.EndForAbuse() if err != nil { return err } if blocked { p.logger.Debug("blocking peer for sending peer on blocklist", zap.String("node", connectionUri.String()), zap.String("peer", fromPeerId), zap.String("ip", fromPeerIP)) } } p.outgoingPeerBlocklist.Put(idString, true) p.logger.Debug("blocking peer for too many failures", zap.String("node", connectionUri.String())) } return nil } retried = true p.logger.Error("failed to connect", zap.String("node", connectionUri.String()), zap.Error(err)) delay := p.reconnectDelay.GetUInt(idString) if delay == nil { tmp := uint(1) delay = &tmp } delayDeref := *delay p.reconnectDelay.PutUInt(idString, delayDeref*2) time.Sleep(time.Duration(delayDeref) * time.Second) return p.ConnectToNode(connectionUris, retried, fromPeer) } if p.outgoingPeerFailures.Contains(idString) { p.outgoingPeerFailures.Remove(idString) } peer, err := net.CreateTransportPeer(scheme, &net.TransportPeerConfig{ Socket: socket, Uris: []*url.URL{connectionUri}, }) if err != nil { return err } peer.SetId(id) p.services.P2P().ConnectionTracker().Add(1) peerId, err := peer.Id().ToString() if err != nil { return err } p.peersPending.Put(peerId, peer) go func() { err := p.OnNewPeer(peer, true) if err != nil && !peer.Abuser() { p.logger.Error("peer error", zap.Error(err)) } p.services.P2P().ConnectionTracker().Done() }() return nil } func (p *P2PService) OnNewPeer(peer net.Peer, verifyId bool) error { var wg sync.WaitGroup var pid string if peer.Id() != nil { pid, _ = peer.Id().ToString() } else { pid = "unknown" } pip := peer.GetIP() if p.incomingIPBlocklist.Contains(pid) { p.logger.Error("peer is on identity blocklist", zap.String("peer", pid)) err := peer.EndForAbuse() if err != nil { return err } return nil } if p.incomingPeerBlockList.Contains(pip) { p.logger.Debug("peer is on ip blocklist", zap.String("peer", pid), zap.String("ip", pip)) err := peer.EndForAbuse() if err != nil { return err } return nil } p.logger.Debug("OnNewPeer started", zap.String("peer", pid)) challenge := protocol.GenerateChallenge() peer.SetChallenge(challenge) wg.Add(1) go func() { defer wg.Done() p.OnNewPeerListen(peer, verifyId) }() handshakeOpenMsg, err := msgpack.Marshal(protocol.NewHandshakeOpen(challenge, p.networkID)) if err != nil { return err } err = peer.SendMessage(handshakeOpenMsg) if err != nil { return err } p.logger.Debug("OnNewPeer sent handshake", zap.String("peer", pid)) p.logger.Debug("OnNewPeer before Wait", zap.String("peer", pid)) wg.Wait() // Wait for OnNewPeerListen goroutine to finish p.logger.Debug("OnNewPeer ended", zap.String("peer", pid)) return nil } func (p *P2PService) OnNewPeerListen(peer net.Peer, verifyId bool) { onDone := net.CloseCallback(func() { if peer.Id() != nil { pid, err := peer.Id().ToString() if err != nil { p.logger.Error("failed to get peer id", zap.Error(err)) return } // Handle closure of the connection if p.peers.Contains(pid) { p.peers.Remove(pid) } if p.peersPending.Contains(pid) { p.peersPending.Remove(pid) } } }) onError := net.ErrorCallback(func(args ...interface{}) { if !peer.Abuser() { p.logger.Error("peer error", zap.Any("args", args)) } }) peer.ListenForMessages(func(message []byte) error { var reader base.IncomingMessageReader err := msgpack.Unmarshal(message, &reader) if err != nil { p.logger.Error("Error decoding basic message info", zap.Error(err)) return err } // Now, get the specific message handler based on the message kind handler, ok := protocol.GetMessageType(reader.Kind) if !ok { p.logger.Error("Unknown message type", zap.Int("type", reader.Kind)) return fmt.Errorf("unknown message type: %d", reader.Kind) } data := base.IncomingMessageData{ Original: message, Data: reader.Data, Ctx: context.Background(), Peer: peer, VerifyId: verifyId, Config: p.config, } dec := msgpack.NewDecoder(bytes.NewReader(reader.Data)) err = handler.DecodeMessage(dec, data) if err != nil { p.logger.Error("Error decoding message", zap.Error(err)) return err } // Directly decode and handle the specific message type if err := handler.HandleMessage(data); err != nil { p.logger.Error("Error handling message", zap.Error(err)) return err } return nil }, net.ListenerOptions{ OnClose: &onDone, OnError: &onError, Logger: p.logger, }) } func (p *P2PService) readNodeVotes(nodeId *encoding.NodeId) (NodeVotes, error) { var value []byte var found bool err := p.db.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(nodeBucketName)) if b == nil { return fmt.Errorf("Bucket %s not found", nodeBucketName) } value = b.Get(nodeId.Raw()) if value == nil { return nil } found = true return nil }) if err != nil { return nil, err } if !found { return NewNodeVotes(), nil } var score NodeVotes err = msgpack.Unmarshal(value, &score) if err != nil { return nil, err } return score, nil } func (p *P2PService) saveNodeVotes(nodeId *encoding.NodeId, votes NodeVotes) error { // Marshal the votes into data data, err := msgpack.Marshal(votes) if err != nil { return err } // Use a transaction to save the data err = p.db.Update(func(tx *bolt.Tx) error { // Get or create the bucket b := tx.Bucket([]byte(nodeBucketName)) // Put the data into the bucket return b.Put(nodeId.Raw(), data) }) return err } func (p *P2PService) GetNodeScore(nodeId *encoding.NodeId) (float64, error) { if nodeId.Equals(p.localNodeID) { return 1, nil } score, err := p.readNodeVotes(nodeId) if err != nil { return 0.5, err } return protocol.CalculateNodeScore(score.Good(), score.Bad()), nil } func (p *P2PService) SortNodesByScore(nodes []*encoding.NodeId) ([]*encoding.NodeId, error) { scores := make(map[encoding.NodeIdCode]float64) var errOccurred error for _, nodeId := range nodes { score, err := p.GetNodeScore(nodeId) if err != nil { errOccurred = err scores[nodeId.HashCode()] = 0 // You may choose a different default value for error cases } else { scores[nodeId.HashCode()] = score } } sort.Slice(nodes, func(i, j int) bool { return scores[nodes[i].HashCode()] > scores[nodes[j].HashCode()] }) return nodes, errOccurred } func (p *P2PService) SignMessageSimple(message []byte) ([]byte, error) { signedMessage := signed.NewSignedMessageRequest(message) signedMessage.SetNodeId(p.localNodeID) err := signedMessage.Sign(p.config) if err != nil { return nil, err } result, err := msgpack.Marshal(signedMessage) if err != nil { return nil, err } return result, nil } func (p *P2PService) AddPeer(peer net.Peer) error { peerId, err := peer.Id().ToString() if err != nil { return err } p.peers.Put(peerId, peer) p.reconnectDelay.PutUInt(peerId, 1) if p.peersPending.Contains(peerId) { p.peersPending.Remove(peerId) } return nil } func (p *P2PService) SendPublicPeersToPeer(peer net.Peer, peersToSend []net.Peer) error { announceRequest := signed.NewAnnounceRequest(peer, peersToSend) message, err := msgpack.Marshal(announceRequest) if err != nil { return err } signedMessage, err := p.SignMessageSimple(message) if err != nil { return err } err = peer.SendMessage(signedMessage) return nil } func (p *P2PService) SendHashRequest(hash *encoding.Multihash, kinds []types.StorageLocationType) error { hashRequest := protocol.NewHashRequest(hash, kinds) message, err := msgpack.Marshal(hashRequest) if err != nil { return err } for _, peer := range p.peers.Values() { peerValue, ok := peer.(net.Peer) if !ok { p.logger.Error("failed to cast peer to net.Peer") continue } err = peerValue.SendMessage(message) } return nil } func (p *P2PService) UpVote(nodeId *encoding.NodeId) error { err := p.vote(nodeId, true) if err != nil { return err } return nil } func (p *P2PService) DownVote(nodeId *encoding.NodeId) error { err := p.vote(nodeId, false) if err != nil { return err } return nil } func (p *P2PService) vote(nodeId *encoding.NodeId, upvote bool) error { votes, err := p.readNodeVotes(nodeId) if err != nil { return err } if upvote { votes.Upvote() } else { votes.Downvote() } err = p.saveNodeVotes(nodeId, votes) if err != nil { return err } return nil } func (p *P2PService) NodeId() *encoding.NodeId { return p.localNodeID } func (p *P2PService) WaitOnConnectedPeers() { p.connections.Wait() } func (p *P2PService) ConnectionTracker() *sync.WaitGroup { return &p.connections } func (p *P2PService) NetworkId() string { return p.config.P2P.Network } func (n *P2PService) HashQueryRoutingTable() structs.Map { return n.hashQueryRoutingTable }