*Major refactor to support pluggable backends

This commit is contained in:
Derrick Hammer 2023-01-05 20:31:14 -05:00
parent 9a3a933dc5
commit 2ebfcd1363
Signed by: pcfreak30
GPG Key ID: C997C339BE476FF2
8 changed files with 164 additions and 84 deletions

View File

@ -11,14 +11,15 @@
"@lumeweb/relay-types": "https://git.lumeweb.com/LumeWeb/relay-types.git", "@lumeweb/relay-types": "https://git.lumeweb.com/LumeWeb/relay-types.git",
"@noble/ed25519": "^1.7.1", "@noble/ed25519": "^1.7.1",
"b4a": "^1.6.1", "b4a": "^1.6.1",
"node-cache": "^5.1.2", "lmdb": "^2.7.3",
"object-merger": "^1.0.3" "node-cache": "^5.1.2"
}, },
"devDependencies": { "devDependencies": {
"@lumeweb/cfg": "https://git.lumeweb.com/LumeWeb/cfg.git", "@lumeweb/cfg": "https://git.lumeweb.com/LumeWeb/cfg.git",
"@protobuf-ts/plugin": "^2.8.2", "@protobuf-ts/plugin": "^2.8.2",
"@protobuf-ts/runtime": "^2.8.2", "@protobuf-ts/runtime": "^2.8.2",
"@types/node": "^18.11.18", "@types/node": "^18.11.18",
"hyperswarm": "^4.3.5",
"prettier": "^2.8.1" "prettier": "^2.8.1"
} }
} }

View File

@ -1,13 +1,13 @@
import { defineConfig } from "rollup";
import preset from "@lumeweb/relay-plugin-rollup-preset"; import preset from "@lumeweb/relay-plugin-rollup-preset";
import merger from "object-merger"; export default preset(
{
export default defineConfig(
merger(preset(), {
input: "src/index.ts", input: "src/index.ts",
output: { output: {
file: "dist/registry.js", file: "dist/registry.js",
format: "cjs", format: "cjs",
inlineDynamicImports: true,
}, },
}) },
{},
{ ignore: ["bun:ffi"], transformMixedEsModules: true }
); );

View File

@ -1,18 +1,18 @@
import type { Plugin, PluginAPI } from "@lumeweb/relay-types"; import type { Plugin, PluginAPI } from "@lumeweb/relay-types";
import DHTFlood from "@lumeweb/dht-flood"; import DHTFlood from "@lumeweb/dht-flood";
import { Message, Query, MessageType } from "./messages.js"; import { Message, MessageType, Query } from "./messages.js";
import EventEmitter from "events"; import EventEmitter from "events";
import NodeCache from "node-cache";
import b4a from "b4a"; import b4a from "b4a";
import { SignedRegistryEntry } from "./types.js"; import { RegistryStorage, SignedRegistryEntry } from "./types.js";
import { verifyEntry } from "./utils.js"; import { verifyEntry } from "./utils.js";
import { getStorage, hasStorage } from "./storage/index.js";
const PROTOCOL = "lumeweb.registry"; const PROTOCOL = "lumeweb.registry";
const events = new EventEmitter(); const events = new EventEmitter();
let messenger: DHTFlood; let messenger: DHTFlood;
let api: PluginAPI; let api: PluginAPI;
let memStore: NodeCache; let store: RegistryStorage;
function setup() { function setup() {
messenger = new DHTFlood({ messenger = new DHTFlood({
@ -45,6 +45,11 @@ function setup() {
}); });
} }
function setupEventHandlers() {
events.on("create", handleCreate);
events.on("query", handleQuery);
}
function entryFromMessage(message: Message): SignedRegistryEntry { function entryFromMessage(message: Message): SignedRegistryEntry {
return { return {
pk: message.pubkey, pk: message.pubkey,
@ -68,47 +73,17 @@ function sendDirectOrBroadcast(message: Message, pubkey: Buffer) {
async function getEntry( async function getEntry(
pubkey: Uint8Array pubkey: Uint8Array
): Promise<SignedRegistryEntry | boolean> { ): Promise<SignedRegistryEntry | boolean> {
let pubkeyHex = b4a.from(pubkey).toString("hex"); return store.get(b4a.from(pubkey).toString("hex"));
if (memStore) {
return await memStore.get<SignedRegistryEntry>(pubkeyHex);
} }
return false; async function handleCreate(message: Message, origin: Buffer) {
} {
const setEntry = async (entry: SignedRegistryEntry): Promise<boolean> => {
async function setEntry(entry: SignedRegistryEntry): Promise<boolean> {
let pubkeyHex = b4a.from(entry.pk).toString("hex"); let pubkeyHex = b4a.from(entry.pk).toString("hex");
if (memStore) {
return await memStore.set<SignedRegistryEntry>(pubkeyHex, entry);
}
return false; return store.set(pubkeyHex, entry);
} };
const setAndRespond = async (entry: SignedRegistryEntry, set = true) => {
const plugin: Plugin = {
name: "registry",
async plugin(_api: PluginAPI): Promise<void> {
api = _api;
setup();
if (api.pluginConfig.bool("store.memory")) {
memStore = new NodeCache();
}
events.on("create", async (message: Message, origin: Buffer) => {
let newEntry = entryFromMessage(message);
if (!newEntry.signature?.length) {
return;
}
if (!verifyEntry(newEntry)) {
return;
}
if (newEntry.data.length > 48) {
return;
}
let entry = (await getEntry(newEntry.pk)) as SignedRegistryEntry;
async function setAndRespond(entry: SignedRegistryEntry, set = true) {
let ret = true; let ret = true;
if (set) { if (set) {
ret = await setEntry(newEntry); ret = await setEntry(newEntry);
@ -124,8 +99,21 @@ const plugin: Plugin = {
}), }),
origin origin
); );
api.logger.info("added entry %s", b4a.from(entry.pk).toString("hex"));
} }
};
let newEntry = entryFromMessage(message);
if (!newEntry.signature?.length) {
return;
} }
if (!verifyEntry(newEntry)) {
return;
}
if (newEntry.data.length > 48) {
return;
}
let entry = (await getEntry(newEntry.pk)) as SignedRegistryEntry;
if (entry) { if (entry) {
if (newEntry.revision <= entry.revision) { if (newEntry.revision <= entry.revision) {
@ -136,9 +124,10 @@ const plugin: Plugin = {
return; return;
} }
setAndRespond(newEntry); setAndRespond(newEntry);
}); }
}
events.on("query", async (query: Query, origin: Buffer) => { async function handleQuery(query: Query, origin: Buffer) {
let entry = (await getEntry(query.pubkey)) as SignedRegistryEntry; let entry = (await getEntry(query.pubkey)) as SignedRegistryEntry;
if (entry) { if (entry) {
@ -153,7 +142,24 @@ const plugin: Plugin = {
origin origin
); );
} }
}); }
const plugin: Plugin = {
name: "registry",
async plugin(_api: PluginAPI): Promise<void> {
api = _api;
const storageType = api.pluginConfig.str("store.type");
const storageOptions = api.pluginConfig.str("store.type", {});
if (!hasStorage(storageType)) {
api.logger.error("Storage type %s does not exist", storageType);
return;
}
store = getStorage(storageType, { ...storageOptions, api });
setup();
setupEventHandlers();
}, },
}; };

View File

@ -0,0 +1,25 @@
import { SignedRegistryEntry, RegistryStorage } from "../../types.js";
import { open, RootDatabase } from "lmdb";
import { PluginAPI } from "@lumeweb/relay-types";
import path from "node:path";
export default class Lmdb implements RegistryStorage {
private _database: RootDatabase<SignedRegistryEntry>;
constructor(opts: { api: PluginAPI }) {
const config = opts.api.config.str("configdir");
this._database = open<SignedRegistryEntry>({
path: path.join(path.dirname(config), "data", "registry"),
sharedStructuresKey: Symbol.for("structures"),
});
}
async get(key: string): Promise<boolean | SignedRegistryEntry> {
const ret = await this._database.get(key);
return ret ?? false;
}
async set(key: string, value: SignedRegistryEntry): Promise<boolean> {
return await this._database.put(key, value);
}
}

View File

@ -0,0 +1,15 @@
import { SignedRegistryEntry, RegistryStorage } from "../../types.js";
import NodeCache from "node-cache";
export default class Memory implements RegistryStorage {
private _storage = new NodeCache();
async get(key: string): Promise<boolean | SignedRegistryEntry> {
const ret = await this._storage.get<SignedRegistryEntry>(key);
return ret ?? false;
}
async set(key: string, value: SignedRegistryEntry): Promise<boolean> {
return await this._storage.set<SignedRegistryEntry>(key, value);
}
}

18
src/storage/factory.ts Normal file
View File

@ -0,0 +1,18 @@
import { RegistryStorage, RegistryStorageConstructor } from "../types.js";
const backends = new Map<string, RegistryStorageConstructor>();
export function register(
type: string,
instance: new (opts?: any) => RegistryStorage
): void {
backends.set(type, instance);
}
export function get(type: string, opts = {}): RegistryStorage {
const ctor = backends.get(type);
return new ctor(opts);
}
export function has(type: string): boolean {
return backends.has(type);
}

8
src/storage/index.ts Normal file
View File

@ -0,0 +1,8 @@
import { register } from "./factory.js";
import Memory from "./backends/memory.js";
import Lmdb from "./backends/lmdb.js";
register("memory", Memory);
register("lmdb", Lmdb);
export { get as getStorage, has as hasStorage } from "./factory.js";

View File

@ -10,3 +10,10 @@ export interface SignedRegistryEntry {
// signature of this registry entry // signature of this registry entry
signature?: Uint8Array; signature?: Uint8Array;
} }
export type RegistryStorageConstructor = new (opts?: any) => RegistryStorage;
export interface RegistryStorage {
get(key: string): Promise<boolean | SignedRegistryEntry>;
set(key: string, value: SignedRegistryEntry): Promise<boolean>;
}