*Initial version

This commit is contained in:
Derrick Hammer 2022-11-16 02:45:10 -05:00
parent c2a31154e7
commit 9b6a916f00
Signed by: pcfreak30
GPG Key ID: C997C339BE476FF2
8 changed files with 720 additions and 1 deletions

View File

@ -1,6 +1,8 @@
MIT License
Copyright (c) <year> <copyright holders>
Copyright (c) 2022 Hammer Technologies LLC
Credits to https://github.com/RangerMauve/hyper-presence for original version
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

21
messages.proto Normal file
View File

@ -0,0 +1,21 @@
syntax = "proto2";
enum Type {
BOOTSTRAP_REQUEST = 1;
BOOTSTRAP_RESPONSE = 2;
CONNECTED = 3;
DISCONNECTED = 4;
STATE = 5;
}
message Message {
required Type type = 1;
map<string, State> bootstrap = 2; // For bootstrap events
optional bytes data = 3; // For state event
optional bytes id = 4; // For connected and disconnected events
}
message State {
repeated bytes connectedTo = 1;
optional bytes data = 2;
}

29
package.json Normal file
View File

@ -0,0 +1,29 @@
{
"name": "@lumeweb/dht-online",
"type": "commonjs",
"version": "0.1.0",
"main": "dist/index.js",
"dependencies": {
"@lumeweb/dht-flood": "https://git.lumeweb.com/LumeWeb/dht-flood.git",
"@protobuf-ts/plugin": "^2.8.1",
"@types/codecs": "^2.2.3",
"codecs": "^3.0.0",
"compact-encoding": "^2.11.0",
"jsnetworkx": "^0.3.4",
"lru": "^3.1.0",
"protocol-buffers-encodings": "^1.2.0",
"protomux-rpc": "^1.3.0"
},
"devDependencies": {
"@types/b4a": "^1.6.0",
"@types/debug": "^4.1.7",
"crypto": "^1.0.1",
"debug": "^4.3.4",
"hyperswarm": "^4.3.5",
"prettier": "^2.7.1",
"protoc": "^1.1.3",
"sodium-universal": "^3.1.0",
"tape": "^5.6.1",
"ts-proto": "^1.131.2"
}
}

292
src/dhtOnlineBase.ts Normal file
View File

@ -0,0 +1,292 @@
import EventEmitter from "events";
// @ts-ignore
import { DiGraph, hasPath } from "jsnetworkx";
import codecs from "codecs";
import { Message, State, Type } from "./messages.js";
const DEFAULT_ENCODING = "json";
type Bootstrap = {
[key: string]: State;
};
export default class DHTOnlineBase extends EventEmitter {
private id: Buffer;
private bootstrapped: boolean;
private graph: any;
private connectedTo: Set<any>;
private data: {};
private encoding: codecs.Codec<any>;
constructor(id: Buffer, { encoding = DEFAULT_ENCODING } = {}) {
super();
if (!id) throw new TypeError("Must provide id for self");
this.id = id;
this.bootstrapped = false;
this.graph = new DiGraph();
this.connectedTo = new Set();
this.data = {};
this.encoding = codecs(encoding || DEFAULT_ENCODING);
this._online = [this._maybeHexify(this.id)];
}
private _online: string[];
get online(): string[] {
return this._online;
}
broadcast(data: any, ttl?: number) {
throw new TypeError("Broadcast has not been implemented");
}
getPeerData(id: Buffer | string) {
return this.graph.node.get(this._maybeHexify(id));
}
protected setData(data: any) {
this.data = data;
this._setPeer(this.id, data);
this._broadcastData();
}
private _broadcastData() {
const rawData = this.data;
if (!Object.keys(rawData).length) {
return;
}
const data = this.encoding.encode(rawData);
this.broadcast(
Message.toBinary(
Message.create({
type: Type.STATE,
data,
})
)
);
}
protected onAddPeer(id: Buffer) {
const stringId = id.toString("hex");
if (this.connectedTo.has(stringId)) {
return;
} // Already know we're connected here
this.connectedTo.add(stringId);
this._addPeerConnection(this.id, id);
this.emit("peer-add", id);
this._recalculate();
this.broadcast(
Message.toBinary(
Message.create({
type: Type.CONNECTED,
id,
})
)
);
this._broadcastData();
if (this.bootstrapped) {
return;
}
// If this is the first person we've met, get their graph
this.broadcast(
Message.toBinary(
Message.create({
type: Type.BOOTSTRAP_REQUEST,
})
),
0
);
}
protected onRemovePeer(id: Buffer) {
this.connectedTo.delete(id.toString("hex"));
this._removePeerConnection(this.id, id);
this.emit("peer-remove");
this._recalculate();
this.broadcast(
Message.toBinary(
Message.create({
type: Type.DISCONNECTED,
id,
})
)
);
}
protected onGetBroadcast(message: Buffer, id: Buffer) {
let decoded;
try {
decoded = Message.fromBinary(message);
} catch {
return;
}
const { type } = decoded;
if (!type) {
throw new Error("Missing Type In Message");
}
if (type === Type.STATE) {
const { data: rawData } = decoded;
const data = this.encoding.decode(rawData);
this._setPeer(id, data);
this.emit("peer-data", data, id);
this._recalculate();
} else if (type === Type.CONNECTED) {
const { id: toId } = decoded;
this._addPeerConnection(id, Buffer.from(toId as Uint8Array));
this.emit("peer-add-seen", id, toId);
this._recalculate();
} else if (type === Type.DISCONNECTED) {
const { id: toId } = decoded;
this._removePeerConnection(id, Buffer.from(toId as Uint8Array));
this.emit("peer-remove-seen", id, toId);
this._recalculate();
} else if (type === Type.BOOTSTRAP_REQUEST) {
const bootstrap = this._getBootstrapInfo();
this.broadcast(
Message.toBinary(
Message.create({
type: Type.BOOTSTRAP_RESPONSE,
bootstrap,
})
),
0
);
} else if (type === Type.BOOTSTRAP_RESPONSE) {
const { bootstrap } = decoded;
this._bootstrapFrom(bootstrap);
}
}
private _hasSeenPeer(id: Buffer | string) {
return this.graph.hasNode(this._maybeHexify(id));
}
private _setPeer(id: Buffer | string, data: any) {
this.graph.addNode(this._maybeHexify(id), data);
}
private _removePeer(id: Buffer | string) {
this.graph.removeNode(this._maybeHexify(id));
}
private _ensurePeer(id: Buffer | string) {
id = this._maybeHexify(id);
if (!this._hasSeenPeer(id)) {
this._setPeer(id, {});
}
}
private _addPeerConnection(
origin: Buffer | string,
destination: Buffer | string
) {
this._ensurePeer(origin);
this._ensurePeer(destination);
this.graph.addEdge(
this._maybeHexify(origin),
this._maybeHexify(destination)
);
}
private _removePeerConnection(origin: Buffer, destination: Buffer) {
try {
this._ensurePeer(origin);
this._ensurePeer(destination);
this.graph.removeEdge(
origin.toString("hex"),
destination.toString("hex")
);
} catch (e: any) {
if (e.name !== "JSNetworkXError") throw e;
}
}
private _bootstrapFrom(bootstrap: Bootstrap) {
if (this.bootstrapped) {
return;
}
for (const id in bootstrap) {
const { data, connectedTo } = bootstrap[id];
const parsedData = data ? this.encoding.decode(data) : null;
let peerData = parsedData || {};
if (id === this.id.toString("hex")) continue;
// If we're already tracking them
if (this._hasSeenPeer(id)) {
// See what data we already have for them
// Add their existing data to what we got from the bootstrap
const existingPeerData = this.getPeerData(id);
peerData = { ...existingPeerData, ...peerData };
}
this._setPeer(id, peerData);
for (const connection of connectedTo) {
this._addPeerConnection(id, Buffer.from(connection));
}
}
this.emit("bootstrapped");
this._recalculate();
}
private _getPeerConnectedTo(id: string) {
return this.graph.successors(id);
}
private _getBootstrapInfo() {
const state: Bootstrap = {};
for (const [id, rawData] of this.graph.nodes(true)) {
const connectedTo = this.graph
.neighbors(id)
.map((id: string) => Buffer.from(id, "hex"));
const data = rawData ? this.encoding.encode(rawData) : null;
state[id] = { data, connectedTo };
}
return state;
}
// Calculate who's online and emit an event
private _recalculate() {
const online = this.graph.nodes().filter((id: string) => {
return hasPath(this.graph, {
source: this._maybeHexify(this.id),
target: id,
});
});
const offline = this.graph.nodes().filter((id: string) => {
return !hasPath(this.graph, {
source: this._maybeHexify(this.id),
target: id,
});
});
for (const id of offline) {
this.graph.removeNode(id);
}
this._online = online;
this.emit("online", online);
}
private _maybeHexify(data: Buffer | string): string {
if (Buffer.isBuffer(data)) {
return data.toString("hex");
}
return data;
}
}

50
src/index.ts Normal file
View File

@ -0,0 +1,50 @@
import DHTOnlineBase from "./dhtOnlineBase.js";
import DHTFlood from "@lumeweb/dht-flood";
const DISCONNECT_SMOOTH = 500;
export default class DHTOnline extends DHTOnlineBase {
private flood: DHTFlood;
private swarm: any;
constructor(
swarm: any,
{ id = swarm.keyPair.publicKey, data = {}, ...opts } = {}
) {
super(id, opts);
this.flood = new DHTFlood({ id, swarm, ...opts });
this.swarm = swarm;
this.flood.on("peer-open", (peer) => this.handlePeerAdd(peer));
this.flood.on("peer-remove", (peer) => this.handlePeerRemove(peer));
this.flood.on("message", (message, id) => this.onGetBroadcast(message, id));
this.swarm.on("connection", (peer: any) =>
this.flood.send(peer, Buffer.from("hello"), 0)
);
this.setData(data);
[...this.swarm.peers.values()].forEach(this.handlePeerAdd.bind(this));
}
handlePeerAdd(peer: any) {
const id = peer.remotePublicKey;
this.onAddPeer(id);
}
handlePeerRemove(peer: any) {
const id = peer.remotePublicKey;
// Wait for a bit and check if we're still disconnected before removing the peer
setTimeout(() => {
if (this.swarm._allConnections.has(id)) {
return;
}
this.onRemovePeer(id);
}, DISCONNECT_SMOOTH);
}
broadcast(message: any, ttl?: number) {
this.flood.broadcast(message, ttl);
}
}

220
src/messages.ts Normal file
View File

@ -0,0 +1,220 @@
// @generated by protobuf-ts 2.8.1
// @generated from protobuf file "messages.proto" (syntax proto2)
// tslint:disable
import type { BinaryWriteOptions } from "@protobuf-ts/runtime";
import type { IBinaryWriter } from "@protobuf-ts/runtime";
import { WireType } from "@protobuf-ts/runtime";
import type { BinaryReadOptions } from "@protobuf-ts/runtime";
import type { IBinaryReader } from "@protobuf-ts/runtime";
import { UnknownFieldHandler } from "@protobuf-ts/runtime";
import type { PartialMessage } from "@protobuf-ts/runtime";
import { reflectionMergePartial } from "@protobuf-ts/runtime";
import { MESSAGE_TYPE } from "@protobuf-ts/runtime";
import { MessageType } from "@protobuf-ts/runtime";
/**
* @generated from protobuf message Message
*/
export interface Message {
/**
* @generated from protobuf field: Type type = 1;
*/
type: Type;
/**
* @generated from protobuf field: map<string, State> bootstrap = 2;
*/
bootstrap: {
[key: string]: State;
}; // For bootstrap events
/**
* @generated from protobuf field: optional bytes data = 3;
*/
data?: Uint8Array; // For state event
/**
* @generated from protobuf field: optional bytes id = 4;
*/
id?: Uint8Array; // For connected and disconnected events
}
/**
* @generated from protobuf message State
*/
export interface State {
/**
* @generated from protobuf field: repeated bytes connectedTo = 1;
*/
connectedTo: Uint8Array[];
/**
* @generated from protobuf field: optional bytes data = 2;
*/
data?: Uint8Array;
}
/**
* @generated from protobuf enum Type
*/
export enum Type {
/**
* @generated synthetic value - protobuf-ts requires all enums to have a 0 value
*/
UNSPECIFIED$ = 0,
/**
* @generated from protobuf enum value: BOOTSTRAP_REQUEST = 1;
*/
BOOTSTRAP_REQUEST = 1,
/**
* @generated from protobuf enum value: BOOTSTRAP_RESPONSE = 2;
*/
BOOTSTRAP_RESPONSE = 2,
/**
* @generated from protobuf enum value: CONNECTED = 3;
*/
CONNECTED = 3,
/**
* @generated from protobuf enum value: DISCONNECTED = 4;
*/
DISCONNECTED = 4,
/**
* @generated from protobuf enum value: STATE = 5;
*/
STATE = 5
}
// @generated message type with reflection information, may provide speed optimized methods
class Message$Type extends MessageType<Message> {
constructor() {
super("Message", [
{ no: 1, name: "type", kind: "enum", T: () => ["Type", Type] },
{ no: 2, name: "bootstrap", kind: "map", K: 9 /*ScalarType.STRING*/, V: { kind: "message", T: () => State } },
{ no: 3, name: "data", kind: "scalar", opt: true, T: 12 /*ScalarType.BYTES*/ },
{ no: 4, name: "id", kind: "scalar", opt: true, T: 12 /*ScalarType.BYTES*/ }
]);
}
create(value?: PartialMessage<Message>): Message {
const message = { type: 0, bootstrap: {} };
globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this });
if (value !== undefined)
reflectionMergePartial<Message>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: Message): Message {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* Type type */ 1:
message.type = reader.int32();
break;
case /* map<string, State> bootstrap */ 2:
this.binaryReadMap2(message.bootstrap, reader, options);
break;
case /* optional bytes data */ 3:
message.data = reader.bytes();
break;
case /* optional bytes id */ 4:
message.id = reader.bytes();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
private binaryReadMap2(map: Message["bootstrap"], reader: IBinaryReader, options: BinaryReadOptions): void {
let len = reader.uint32(), end = reader.pos + len, key: keyof Message["bootstrap"] | undefined, val: Message["bootstrap"][any] | undefined;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case 1:
key = reader.string();
break;
case 2:
val = State.internalBinaryRead(reader, reader.uint32(), options);
break;
default: throw new globalThis.Error("unknown map entry field for field Message.bootstrap");
}
}
map[key ?? ""] = val ?? State.create();
}
internalBinaryWrite(message: Message, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* Type type = 1; */
if (message.type !== 0)
writer.tag(1, WireType.Varint).int32(message.type);
/* map<string, State> bootstrap = 2; */
for (let k of Object.keys(message.bootstrap)) {
writer.tag(2, WireType.LengthDelimited).fork().tag(1, WireType.LengthDelimited).string(k);
writer.tag(2, WireType.LengthDelimited).fork();
State.internalBinaryWrite(message.bootstrap[k], writer, options);
writer.join().join();
}
/* optional bytes data = 3; */
if (message.data !== undefined)
writer.tag(3, WireType.LengthDelimited).bytes(message.data);
/* optional bytes id = 4; */
if (message.id !== undefined)
writer.tag(4, WireType.LengthDelimited).bytes(message.id);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message Message
*/
export const Message = new Message$Type();
// @generated message type with reflection information, may provide speed optimized methods
class State$Type extends MessageType<State> {
constructor() {
super("State", [
{ no: 1, name: "connectedTo", kind: "scalar", repeat: 2 /*RepeatType.UNPACKED*/, T: 12 /*ScalarType.BYTES*/ },
{ no: 2, name: "data", kind: "scalar", opt: true, T: 12 /*ScalarType.BYTES*/ }
]);
}
create(value?: PartialMessage<State>): State {
const message = { connectedTo: [] };
globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this });
if (value !== undefined)
reflectionMergePartial<State>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: State): State {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* repeated bytes connectedTo */ 1:
message.connectedTo.push(reader.bytes());
break;
case /* optional bytes data */ 2:
message.data = reader.bytes();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: State, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* repeated bytes connectedTo = 1; */
for (let i = 0; i < message.connectedTo.length; i++)
writer.tag(1, WireType.LengthDelimited).bytes(message.connectedTo[i]);
/* optional bytes data = 2; */
if (message.data !== undefined)
writer.tag(2, WireType.LengthDelimited).bytes(message.data);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message State
*/
export const State = new State$Type();

82
test.js Normal file
View File

@ -0,0 +1,82 @@
const test = require("tape");
const Hyperswarm = require("hyperswarm");
const sodium = require("sodium-universal");
const b4a = require("b4a");
const { default: DHTOnline } = require("./");
const crypto = require("crypto");
const topicName = crypto.randomBytes(10);
const topic = b4a.allocUnsafe(32);
sodium.crypto_generichash(topic, b4a.from(topicName));
test("Basic presence test / data propagation", (t) => {
t.plan(6);
const peer1 = createPeer();
const peer2 = createPeer();
Promise.all([peer1, peer2]).then((peers) => {
const peer1 = peers.shift();
const peer2 = peers.shift();
const p1 = new DHTOnline(peer1);
const p2 = new DHTOnline(peer2);
p1.setData({ message: "Hello" });
p2.setData({ message: "World!" });
t.ok(p1.id, "Generated id 1");
t.ok(p2.id, "Generated id 2");
p1.on("online", handleOnline);
p2.on("online", handleOnline);
p1.on("peer-remove", handleDisconnect);
let hasFinished = false;
function handleOnline(list) {
if (list.length === 2) {
const peerData1 = p1.getPeerData(p2.id);
const peerData2 = p2.getPeerData(p1.id);
const hasP1 = peerData1 && Object.keys(peerData1).length;
const hasP2 = peerData2 && Object.keys(peerData2).length;
if (!hasP1 || !hasP2) {
return;
}
p1.removeListener("online", handleOnline);
p2.removeListener("online", handleOnline);
setTimeout(() => {
t.pass("Seeing everyone online");
t.deepEqual(peerData1, p2.data, "Got peer data from peer 2");
t.deepEqual(peerData2, p1.data, "Got peer data from peer 2");
hasFinished = true;
peer2._allConnections.get(peer1.keyPair.publicKey).end();
}, 1000);
}
}
function handleDisconnect() {
if (!hasFinished) return t.error("Disconnected before finished");
t.pass("Peer removed on disconnect");
t.end();
}
t.teardown(() => {
[peer1, peer2].forEach((item) => item.destroy());
});
});
});
async function createPeer() {
const swarm = new Hyperswarm();
await swarm.dht.ready();
await swarm.listen();
swarm.join(topic);
return swarm;
}

23
tsconfig.json Normal file
View File

@ -0,0 +1,23 @@
{
"compilerOptions": {
"declaration": true,
"strict": true,
"module": "commonjs",
"target": "esnext",
"esModuleInterop": true,
"sourceMap": false,
"rootDir": "src",
"outDir": "dist",
"typeRoots": [
"node_modules/@types",
],
"moduleResolution": "node",
"declarationMap": true,
"declarationDir": "dist",
"emitDeclarationOnly": false,
"allowJs": true
},
"include": [
"src"
]
}