Compare commits

...

6 Commits

8 changed files with 148 additions and 58 deletions

View File

@ -1,3 +1,10 @@
# [0.1.0-develop.2](https://git.lumeweb.com/LumeWeb/libs5/compare/v0.1.0-develop.1...v0.1.0-develop.2) (2023-08-31)
### Bug Fixes
* add main to package.json ([4672112](https://git.lumeweb.com/LumeWeb/libs5/commit/46721129f3a9b6c03511e4c4e5404ff9da7d2b77))
# [0.1.0-develop.1](https://git.lumeweb.com/LumeWeb/libs5/compare/v0.0.1...v0.1.0-develop.1) (2023-08-31) # [0.1.0-develop.1](https://git.lumeweb.com/LumeWeb/libs5/compare/v0.0.1...v0.1.0-develop.1) (2023-08-31)

4
npm-shrinkwrap.json generated
View File

@ -1,12 +1,12 @@
{ {
"name": "@lumeweb/libs5", "name": "@lumeweb/libs5",
"version": "0.1.0-develop.1", "version": "0.1.0-develop.2",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@lumeweb/libs5", "name": "@lumeweb/libs5",
"version": "0.1.0-develop.1", "version": "0.1.0-develop.2",
"dependencies": { "dependencies": {
"@noble/curves": "^1.1.0", "@noble/curves": "^1.1.0",
"@noble/hashes": "^1.3.1", "@noble/hashes": "^1.3.1",

View File

@ -1,7 +1,8 @@
{ {
"name": "@lumeweb/libs5", "name": "@lumeweb/libs5",
"version": "0.1.0-develop.1", "version": "0.1.0-develop.2",
"type": "module", "type": "module",
"main": "lib/index.js",
"repository": { "repository": {
"type": "git", "type": "git",
"url": "gitea@git.lumeweb.com:LumeWeb/libs5.git" "url": "gitea@git.lumeweb.com:LumeWeb/libs5.git"
@ -26,5 +27,8 @@
}, },
"publishConfig": { "publishConfig": {
"access": "public" "access": "public"
} },
"files": [
"lib"
]
} }

View File

@ -1,3 +1,9 @@
import { S5Node } from "#node.js";
import type { S5NodeConfig } from "#node.js";
export * from "./types.js"; export * from "./types.js";
export * from "./service/p2p.js"; export type { S5NodeConfig };
export * from "./service/registry.js";
export function createNode(config: S5NodeConfig) {
return new S5Node(config);
}

View File

@ -1,9 +1,80 @@
import { Multihash } from "./multihash.js"; import { Multihash } from "./multihash.js";
import NodeId from "./nodeId.js"; import NodeId from "./nodeId.js";
import { S5Config } from "./types.js"; import { Logger, S5Config, S5Services } from "./types.js";
import Unpacker from "./serialization/unpack.js"; import Unpacker from "./serialization/unpack.js";
import Packer from "./serialization/pack.js"; import Packer from "./serialization/pack.js";
import StorageLocation from "./storage.js"; import StorageLocation from "./storage.js";
import KeyPairEd25519 from "#ed25519.js";
import { AbstractLevel } from "abstract-level";
import { P2PService } from "#service/p2p.js";
import { RegistryService } from "#service/registry.js";
const DEFAULT_LOGGER = {
info(s: any) {
console.info(s);
},
verbose(s: any) {
console.log(s);
},
warn(s: any) {
console.warn(s);
},
error(s: any) {
console.error(s);
},
catched(e: any, context?: string | null) {
console.error(e, context);
},
};
export interface S5NodeConfig {
p2p?: {
network: string;
peers?: {
initial?: string[];
};
};
keyPair: KeyPairEd25519;
db: AbstractLevel<Uint8Array, string, Uint8Array>;
logger?: Logger;
}
export class S5Node {
private _nodeConfig: S5NodeConfig;
private _config?: S5Config;
constructor(config: S5NodeConfig) {
this._nodeConfig = config;
}
public async start() {
this._config = {
keyPair: this._nodeConfig.keyPair,
db: this._nodeConfig.db,
logger: this._nodeConfig.logger ?? DEFAULT_LOGGER,
cacheDb: this._nodeConfig.db.sublevel("s5-object-cache", {}),
services: {} as any,
};
const p2p = new P2PService(this);
const registry = new RegistryService(this);
await p2p.init();
await registry.init();
await p2p.start();
}
get services() {
return this._config?.services as S5Services;
}
get config() {
return this._config as S5Config;
}
get db() {
return this._config?.db as AbstractLevel<Uint8Array, string, Uint8Array>;
}
get logger() {
return this._config?.logger as Logger;
}
}
export async function readStorageLocationsFromDB({ export async function readStorageLocationsFromDB({
hash, hash,

View File

@ -22,23 +22,18 @@ import Unpacker from "#serialization/unpack.js";
import { ed25519 } from "@noble/curves/ed25519"; import { ed25519 } from "@noble/curves/ed25519";
import { AbstractLevel, AbstractSublevel } from "abstract-level"; import { AbstractLevel, AbstractSublevel } from "abstract-level";
import StorageLocation from "#storage.js"; import StorageLocation from "#storage.js";
import { addStorageLocation, stringifyNode } from "#node.js"; import { addStorageLocation, S5Node, stringifyNode } from "#node.js";
import { URL } from "url"; import { URL } from "url";
import { Buffer } from "buffer"; import { Buffer } from "buffer";
import { connect as tcpConnect, TcpPeer } from "../peer/tcp.js"; import { connect as tcpConnect, TcpPeer } from "../peer/tcp.js";
import { connect as wsConnect, WebSocketPeer } from "../peer/webSocket.js"; import { connect as wsConnect, WebSocketPeer } from "../peer/webSocket.js";
export class P2PService { export class P2PService {
get peers(): Map<string, Peer> { private node: S5Node;
return this._peers;
}
private config: S5Config;
private logger: Logger; private logger: Logger;
private nodeKeyPair: KeyPairEd25519; private nodeKeyPair: KeyPairEd25519;
private localNodeId?: NodeId; private localNodeId?: NodeId;
private networkId?: string; private networkId?: string;
private _peers: Map<string, Peer> = new Map();
private reconnectDelay: Map<string, number> = new Map(); private reconnectDelay: Map<string, number> = new Map();
private selfConnectionUris: Array<URL> = []; private selfConnectionUris: Array<URL> = [];
private nodesDb?: AbstractSublevel< private nodesDb?: AbstractSublevel<
@ -47,25 +42,30 @@ export class P2PService {
string, string,
Uint8Array Uint8Array
>; >;
private hashQueryRoutingTable: Map<Multihash, Set<NodeId>> = new Map(); private hashQueryRoutingTable: Map<Multihash, Set<NodeId>> = new Map();
constructor(config: S5Config) { constructor(node: S5Node) {
this.config = config; this.node = node;
this.networkId = config?.p2p?.network; this.networkId = node.config.p2p?.network;
this.nodeKeyPair = config.keyPair; this.nodeKeyPair = node.config.keyPair;
this.logger = config.logger; this.logger = node.logger;
config.services.p2p = this; node.config.services.p2p = this;
}
private _peers: Map<string, Peer> = new Map();
get peers(): Map<string, Peer> {
return this._peers;
} }
async init(): Promise<void> { async init(): Promise<void> {
this.localNodeId = new NodeId(this.nodeKeyPair.publicKey); // Define the NodeId constructor this.localNodeId = new NodeId(this.nodeKeyPair.publicKey); // Define the NodeId constructor
this.nodesDb = this.config.db.sublevel<string, Uint8Array>("s5-nodes", {}); this.nodesDb = this.node.db.sublevel<string, Uint8Array>("s5-nodes", {});
} }
async start(): Promise<void> { async start(): Promise<void> {
const initialPeers = this.config?.p2p?.peers?.initial || []; const initialPeers = this.node.config?.p2p?.peers?.initial || [];
for (const p of initialPeers) { for (const p of initialPeers) {
this.connectToNode([new URL(p)]); this.connectToNode([new URL(p)]);
@ -114,8 +114,8 @@ export class P2PService {
return; return;
} else if (method === recordTypeRegistryEntry) { } else if (method === recordTypeRegistryEntry) {
const sre = const sre =
this.config.services.registry.deserializeRegistryEntry(event); this.node.services.registry.deserializeRegistryEntry(event);
await this.config.services.registry.set(sre, false, peer); await this.node.services.registry.set(sre, false, peer);
return; return;
} else if (method === recordTypeStorageLocation) { } else if (method === recordTypeStorageLocation) {
const hash = new Multihash(event.subarray(1, 34)); const hash = new Multihash(event.subarray(1, 34));
@ -157,7 +157,7 @@ export class P2PService {
nodeId, nodeId,
location: new StorageLocation(type, parts, expiry), location: new StorageLocation(type, parts, expiry),
message: event, message: event,
config: this.config, config: this.node.config,
}); });
const list = const list =
@ -361,26 +361,6 @@ export class P2PService {
return calculateScore(map.get(1), map.get(2)); return calculateScore(map.get(1), map.get(2));
} }
private async _vote(nodeId: NodeId, upvote: boolean): Promise<void> {
const node = await this.nodesDb?.get(stringifyNode(nodeId));
const map = node
? Unpacker.fromPacked(node).unpackMap()
: new Map<number, number>(
Object.entries({ 1: 0, 2: 0 }).map(([k, v]) => [+k, v]),
);
if (upvote) {
map.set(1, (map.get(1) ?? 0) + 1);
} else {
map.set(2, (map.get(2) ?? 0) + 1);
}
await this.nodesDb?.put(
stringifyNode(nodeId),
new Packer().pack(map).takeBytes(),
);
}
async upvote(nodeId: NodeId): Promise<void> { async upvote(nodeId: NodeId): Promise<void> {
await this._vote(nodeId, true); await this._vote(nodeId, true);
} }
@ -525,4 +505,24 @@ export class P2PService {
await this.connectToNode(connectionUris, retried); await this.connectToNode(connectionUris, retried);
} }
} }
private async _vote(nodeId: NodeId, upvote: boolean): Promise<void> {
const node = await this.nodesDb?.get(stringifyNode(nodeId));
const map = node
? Unpacker.fromPacked(node).unpackMap()
: new Map<number, number>(
Object.entries({ 1: 0, 2: 0 }).map(([k, v]) => [+k, v]),
);
if (upvote) {
map.set(1, (map.get(1) ?? 0) + 1);
} else {
map.set(2, (map.get(2) ?? 0) + 1);
}
await this.nodesDb?.put(
stringifyNode(nodeId),
new Packer().pack(map).takeBytes(),
);
}
} }

View File

@ -14,7 +14,7 @@ import Packer from "#serialization/pack.js";
import { Buffer } from "buffer"; import { Buffer } from "buffer";
import { EventEmitter } from "events"; import { EventEmitter } from "events";
import KeyPairEd25519 from "#ed25519.js"; import KeyPairEd25519 from "#ed25519.js";
import { stringifyBytes } from "#node.js"; import { S5Node, stringifyBytes } from "#node.js";
interface SignedRegistryEntry { interface SignedRegistryEntry {
pk: Uint8Array; // public key with multicodec prefix pk: Uint8Array; // public key with multicodec prefix
@ -30,17 +30,17 @@ export class RegistryService {
string, string,
Uint8Array Uint8Array
>; >;
private config: S5Config; private node: S5Node;
private logger: Logger; private logger: Logger;
private streams: Map<string, EventEmitter> = new Map<string, EventEmitter>(); private streams: Map<string, EventEmitter> = new Map<string, EventEmitter>();
constructor(config: S5Config) { constructor(node: S5Node) {
this.config = config; this.node = node;
this.logger = this.config.logger; this.logger = this.node.logger;
} }
async init(): Promise<void> { async init(): Promise<void> {
this.db = this.config.db.sublevel<string, Uint8Array>("s5-registry-db", {}); this.db = this.node.db.sublevel<string, Uint8Array>("s5-registry-db", {});
} }
async set( async set(
@ -106,7 +106,7 @@ export class RegistryService {
this.logger.verbose("[registry] broadcastEntry"); this.logger.verbose("[registry] broadcastEntry");
const updateMessage = serializeRegistryEntry(sre); const updateMessage = serializeRegistryEntry(sre);
for (const p of Object.values(this.config.services.p2p.peers)) { for (const p of Object.values(this.node.services.p2p.peers)) {
if (receivedFrom == null || p.id !== receivedFrom.id) { if (receivedFrom == null || p.id !== receivedFrom.id) {
p.sendMessage(updateMessage); p.sendMessage(updateMessage);
} }
@ -122,7 +122,7 @@ export class RegistryService {
const req = p.takeBytes(); const req = p.takeBytes();
// TODO: Use shard system if there are more than X peers // TODO: Use shard system if there are more than X peers
for (const peer of Object.values(this.config.services.p2p.peers)) { for (const peer of Object.values(this.node.services.p2p.peers)) {
peer.sendMessage(req); peer.sendMessage(req);
} }
} }

View File

@ -36,6 +36,11 @@ export interface Logger {
catched(e: any, context?: string | null): void; catched(e: any, context?: string | null): void;
} }
export interface S5Services {
p2p: P2PService;
registry: RegistryService;
}
export interface S5Config { export interface S5Config {
p2p?: { p2p?: {
network: string; network: string;
@ -47,10 +52,7 @@ export interface S5Config {
logger: Logger; logger: Logger;
db: AbstractLevel<Uint8Array, string, Uint8Array>; db: AbstractLevel<Uint8Array, string, Uint8Array>;
cacheDb: AbstractLevel<Uint8Array, string, Uint8Array>; cacheDb: AbstractLevel<Uint8Array, string, Uint8Array>;
services: { services: S5Services;
p2p: P2PService;
registry: RegistryService;
};
} }
export interface SignedMessage { export interface SignedMessage {
nodeId: NodeId; nodeId: NodeId;