.npmrc

@@ -1,4 +0,0 @@
-public-hoist-pattern[]=udx-native
-public-hoist-pattern[]=sodium-native
-public-hoist-pattern[]=@hyperswarm/dht
-public-hoist-pattern[]=hypercore-crypto

.releaserc

@@ -1,38 +0,0 @@
-{
-    "plugins": [
-        "@semantic-release/commit-analyzer",
-        "@semantic-release/release-notes-generator",
-        "@semantic-release/changelog",
-        [
-            "@semantic-release/exec",
-            {
-                "publishCmd": "git clone https://git.lumeweb.com/LumeWeb/circle-ci-publish.git ci; ./ci/publish.sh \"${nextRelease.version}\""
-            }
-        ],
-        [
-            "@semantic-release/npm",
-            {
-                "npmPublish": false
-            }
-        ],
-        [
-            "@semantic-release/git",
-            {
-                "assets": [
-                    "package.json"
-                ]
-            }
-        ]
-    ],
-    "branches": [
-        "master",
-        {
-            name: "develop",
-            prerelease: true
-        },
-        {
-            name: "develop-*",
-            prerelease: true
-        },
-    ]
-}

.woodpecker.yml

@@ -0,0 +1,49 @@
+pipeline:
+    build:
+        image: git.lumeweb.com/lumeweb/ci-node
+        commands:
+            - corepack enable
+            - corepack prepare yarn@stable --activate
+            - yarn
+            - yarn build
+    package:
+        image: ghcr.io/goreleaser/nfpm
+        commands:
+            - nfpm pkg --packager deb
+    publish_focal:
+        image: git.lumeweb.com/lumeweb/aptly-publisher
+        settings:
+            apt_username:
+                from_secret: APT_USERNAME
+            apt_password:
+                from_secret: APT_PASSWORD
+            repo: apt.web3relay.io
+            folder: ubuntu
+            distro: focal
+            gpg_password:
+                from_secret: GPG_PASSWORD
+    publish_jammy:
+        image: git.lumeweb.com/lumeweb/aptly-publisher
+        settings:
+            apt_username:
+                from_secret: APT_USERNAME
+            apt_password:
+                from_secret: APT_PASSWORD
+            repo: apt.web3relay.io
+            folder: ubuntu
+            distro: jammy
+            gpg_password:
+                from_secret: GPG_PASSWORD
+    purge_do_cdn:
+        image: git.lumeweb.com/lumeweb/do-cdn-purge
+        settings:
+            DIGITALOCEAN_ACCESS_TOKEN:
+                from_secret: digitalocean_access_token
+            DIGITALOCEAN_CDN_ID:
+                from_secret: digitalocean_cdn_id
+    purge_cf_cdn:
+        image: git.lumeweb.com/lumeweb/cf-cdn-purge
+        settings:
+            CLOUDFLARE_ACCESS_TOKEN:
+                from_secret: cloudflare_access_token
+            DOMAIN: web3relay.io

.yarn/patches/micro-ed25519-hdkey-npm-0.1.2-5d1d767d1d.patch

@@ -0,0 +1,18 @@
+diff --git a/index.js b/index.js
+index dc628dab1a201e6679a1716b3d51b56e4723b49d..9daeb44298586d51d7bb6e859a1dd696846e0078 100644
+--- a/index.js
++++ b/index.js
+@@ -115,11 +115,11 @@ class HDKey {
+         });
+     }
+     sign(message) {
+-        message = ensureBytes(message, 32);
++        message = ensureBytes(message);
+         return ed25519.sync.sign(message, this.privateKey);
+     }
+     verify(message, signature) {
+-        message = ensureBytes(message, 32);
++        message = ensureBytes(message);
+         signature = ensureBytes(signature, 64);
+         return ed25519.sync.verify(signature, message, this.publicKeyRaw);
+     }

.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

package.json

@@ -1,75 +1,79 @@
 {
     "name": "@lumeweb/relay",
     "type": "commonjs",
-    "version": "0.1.0-develop.2",
+    "version": "0.1.0",
     "description": "",
     "main": "build/index.js",
-    "repository": {
+    "types": "src/types.ts",
-        "url": "gitea@git.lumeweb.com:LumeWeb/relay.git"
-    },
     "author": {
         "name": "Derrick Hammer",
         "email": "contact@lumeweb.com"
     },
     "scripts": {
-        "semantic-release": "semantic-release",
+        "compile": "yarn tsc",
-        "compile": "tsc",
         "prebuild": "bash prebuild.sh",
-        "package": "pkg -c pkg.json build/index.js -t linux  --public --no-native-build -C gzip",
+        "package": "yarn pkg -c pkg.json build/index.js -t linux  --public --no-native-build -C gzip",
-        "build": "npm run compile && npm run prebuild && npm run package",
+        "package-debug": "yarn pkg -c pkg.json build/index.js -b -t linux --no-bytecode  --public",
-        "postinstall": "patch-package"
+        "build": "yarn run compile && npm run prebuild && npm run package",
+        "barebuild": "yarn run compile && npm run package"
     },
     "dependencies": {
-        "@fastify/websocket": "^8.1.0",
+        "@fastify/websocket": "^7.1.1",
-        "@hyperswarm/dht-relay": "^0.4.0",
+        "@hyperswarm/dht-relay": "^0.3.0",
-        "@lumeweb/interface-relay": "^0.0.2-develop.3",
+        "@lumeweb/cfg": "https://git.lumeweb.com/LumeWeb/cfg.git",
-        "@lumeweb/relay-cfg": "^0.1.15",
+        "@lumeweb/dht-cache": "https://git.lumeweb.com/LumeWeb/dht-cache.git",
-        "@scure/bip39": "^1.2.0",
+        "@scure/bip39": "^1.1.0",
-        "@types/node": "^18.15.11",
+        "@types/acme-client": "^3.3.0",
-        "@types/ws": "^8.5.4",
+        "@types/node": "^18.0.0",
+        "@types/node-cron": "^3.0.2",
+        "@types/ws": "^8.5.3",
+        "ajv": "^8.11.0",
         "async-mutex": "^0.3.2",
-        "b4a": "^1.6.3",
+        "b4a": "^1.6.1",
         "compact-encoding": "^2.11.0",
-        "dotenv": "^16.0.3",
+        "date-fns": "^2.28.0",
-        "ed25519-keygen": "github:LumeWeb/ed25519-keygen",
+        "dotenv": "^16.0.1",
-        "ethers": "^5.7.2",
+        "ethers": "^5.6.9",
         "eventemitter2": "^6.4.9",
-        "fastify": "^4.19.2",
+        "fastify": "^4.10.2",
-        "fetch-blob": "github:LumeWeb/fetch-blob",
+        "fetch-blob": "https://github.com/LumeWeb/fetch-blob.git",
-        "hyperswarm": "^4.4.0",
+        "hyperswarm": "^4.3.5",
-        "json-stable-stringify": "^1.0.2",
+        "json-stable-stringify": "^1.0.1",
-        "json-stringify-deterministic": "^1.0.8",
+        "json-stringify-deterministic": "^1.0.7",
-        "loady": "github:LumeWeb/loady",
+        "loady": "https://github.com/LumeWeb/loady.git",
-        "msgpackr": "^1.8.5",
+        "micro-ed25519-hdkey": "patch:micro-ed25519-hdkey@npm%3A0.1.2#~/.yarn/patches/micro-ed25519-hdkey-npm-0.1.2-5d1d767d1d.patch",
+        "msgpackr": "^1.6.1",
         "node-cache": "^5.1.2",
-        "node-fetch": "^2.6.9",
+        "node-cron": "^3.0.1",
-        "p-defer": "git+https://git.lumeweb.com/LumeWeb/p-defer.git",
+        "node-fetch": "2",
-        "p-timeout": "git+https://git.lumeweb.com/LumeWeb/p-timeout.git",
+        "p-timeout": "https://git.lumeweb.com/LumeWeb/p-timeout.git",
-        "pino": "^8.11.0",
+        "pino": "^8.8.0",
-        "pino-pretty": "^9.4.0",
+        "pino-pretty": "^9.1.1",
         "promise-retry": "^2.0.1",
-        "protomux": "^3.4.1",
+        "protomux": "^3.4.0",
-        "protomux-rpc": "^1.4.1",
+        "protomux-rpc": "^1.3.0",
+        "random-access-memory": "^4.1.0",
         "random-key": "^0.3.2",
-        "slugify": "^1.6.6",
+        "slugify": "^1.6.5",
-        "sodium-universal": "^4.0.0"
+        "sodium-universal": "^3.1.0"
     },
     "devDependencies": {
-        "@semantic-release/changelog": "^6.0.3",
+        "@lumeweb/relay-types": "https://git.lumeweb.com/LumeWeb/relay-types.git",
-        "@semantic-release/exec": "^6.0.3",
-        "@semantic-release/git": "^10.0.1",
         "@types/b4a": "^1.6.0",
         "@types/minimatch": "^3.0.5",
-        "@types/node-fetch": "^2.6.3",
+        "@types/node-fetch": "^2.6.2",
-        "cli-progress": "^3.12.0",
+        "cli-progress": "^3.11.2",
         "hyper-typings": "^1.0.0",
-        "node-gyp": "^9.3.1",
+        "node-gyp": "^9.1.0",
-        "patch-package": "^6.5.1",
+        "pkg": "^5.8.0",
-        "pkg": "^5.8.1",
         "prebuildify": "^5.0.1",
-        "prettier": "^2.8.7",
+        "prettier": "^2.7.1",
-        "semantic-release": "21",
+        "rollup": "^2.77.0",
-        "supports-color": "github:LumeWeb/supports-color",
+        "supports-color": "https://github.com/LumeWeb/supports-color.git",
-        "typescript": "^4.9.5"
+        "typescript": "^4.9.4"
+    },
+    "packageManager": "yarn@4.0.0-rc.33",
+    "resolutions": {
+        "micro-ed25519-hdkey@npm:^0.1.2": "patch:micro-ed25519-hdkey@npm%3A0.1.2#~/.yarn/patches/micro-ed25519-hdkey-npm-0.1.2-5d1d767d1d.patch"
     }
 }

patches/@lumeweb+interface-relay+0.0.2-develop.3.patch

@@ -1,12 +0,0 @@
-diff --git a/node_modules/@lumeweb/interface-relay/package.json b/node_modules/@lumeweb/interface-relay/package.json
-index 0bfaba0..c503d91 100644
---- a/node_modules/@lumeweb/interface-relay/package.json
-+++ b/node_modules/@lumeweb/interface-relay/package.json
-@@ -2,7 +2,6 @@
-     "name": "@lumeweb/interface-relay",
-     "version": "0.0.2-develop.3",
-     "main": "lib/index.js",
--    "type": "module",
-     "types": "lib/index.d.ts",
-     "module": "lib/index.mjs",
-     "exports": {

patches/sodium-native+4.0.1.patch

@@ -1,20 +0,0 @@
-diff --git a/node_modules/sodium-native/package.json b/node_modules/sodium-native/package.json
-index bda9dd4..3a5541a 100644
---- a/node_modules/sodium-native/package.json
-+++ b/node_modules/sodium-native/package.json
-@@ -3,15 +3,6 @@
-   "version": "4.0.1",
-   "description": "Low level bindings for libsodium",
-   "main": "index.js",
--  "files": [
--    "index.js",
--    "deps/**",
--    "modules/**",
--    "binding.c",
--    "binding.gyp",
--    "macros.h",
--    "prebuilds/**"
--  ],
-   "dependencies": {
-     "node-gyp-build": "^4.3.0"
-   },

pkg.json

@@ -1,5 +1,9 @@
 {
     "assets": [
+        "node_modules/*/build/Release/*.node",
+        "node_modules/libskynet",
+        "node_modules/libskynetnode",
+        "node_modules/@lumeweb"
     ],
     "outputPath": "dist"
 }

src/config.ts

@@ -1,11 +1,11 @@
 // @ts-ignore
-import { Config } from "@lumeweb/relay-cfg";
+import Config from "@lumeweb/cfg";
 import * as os from "os";
 import * as fs from "fs";
 import path from "path";
 import log from "./log.js";
 
-const config = new Config("lumeweb-relay", "core.confDir");
+const config = new Config("lumeweb-relay", "core.confdir");
 
 let configDir;
 
@@ -26,7 +26,6 @@ switch (os.platform()) {
 config.inject({
   "core.confDir": configDir,
   "core.port": 8080,
-  "core.relayPort": 0,
   "core.appPort": 80,
   "core.logLevel": "info",
   "core.pluginDir": path.resolve(configDir, "..", "plugins"),
@@ -34,7 +33,7 @@ config.inject({
 
 config.load();
 
-configDir = config.str("core.confDir") as string;
+configDir = config.str("core.confdir");
 
 if (fs.existsSync(configDir)) {
   try {

src/lib/seed.ts

@@ -1,4 +1,4 @@
-import { HDKey } from "ed25519-keygen/hdkey";
+import { HDKey } from "micro-ed25519-hdkey";
 import config from "../config";
 import * as bip39 from "@scure/bip39";
 import { wordlist } from "@scure/bip39/wordlists/english";
@@ -8,7 +8,7 @@ import b4a from "b4a";
 const BIP44_PATH = "m/44'/1627'/0'/0'/0'";
 
 export function getSeed() {
-  const seed = config.str("core.seed") as string;
+  const seed = config.str("core.seed");
 
   let valid = bip39.validateMnemonic(seed, wordlist);
   if (!valid) {

src/modules/app.ts

@@ -20,10 +20,7 @@ export async function start() {
 
   await getPluginAPI().emitAsync("core.appServer.buildRoutes");
 
-  await app.listen({
+  await app.listen({ port: config.uint("core.appport"), host: "0.0.0.0" });
-    port: config.uint("core.appPort") as number,
-    host: "0.0.0.0",
-  });
 
   getPluginAPI().emit("core.appServer.started");
 }

src/modules/plugin.ts

@@ -1,14 +1,14 @@
 import config from "../config.js";
 import type { RPCServer } from "./rpc/server.js";
 import { getRpcServer } from "./rpc/server.js";
-import type { Plugin, RPCMethod } from "@lumeweb/interface-relay";
+import type { Plugin, RPCMethod } from "@lumeweb/relay-types";
 import slugify from "slugify";
 import * as fs from "fs";
 import path from "path";
 import type { Logger } from "pino";
 
 import { getHDKey, getSeed } from "../lib/seed.js";
-import type { Config } from "@lumeweb/relay-cfg";
+import type Config from "@lumeweb/cfg";
 import EventEmitter2 from "eventemitter2";
 import log from "../log.js";
 import {
@@ -17,7 +17,7 @@ import {
   ProtocolManager,
 } from "./swarm.js";
 import { get as getApp } from "./app.js";
-import type { HDKey } from "ed25519-keygen/hdkey";
+import type { HDKey } from "micro-ed25519-hdkey";
 import corePlugins from "../plugins";
 import Util from "./plugin/util";
 
@@ -250,9 +250,7 @@ export async function loadPlugins() {
     await apiManager.loadPluginInstance(plugin);
   }
 
-  for (const plugin of [
+  for (const plugin of [...new Set(config.array("core.plugins", []))] as []) {
-    ...new Set(config.array("core.plugins", [] as any)),
-  ] as []) {
     await apiManager.loadPlugin(plugin);
   }
 

src/modules/relay.ts

@@ -16,24 +16,18 @@ import * as http2 from "http2";
 import websocket from "@fastify/websocket";
 
 export async function start() {
-  const swarm = getSwarm();
+  const dht = getSwarm();
 
   let relayServer = fastify({
+    http2: true,
     logger: log.child({ module: "relay-server" }),
   });
 
-  await relayServer.register(websocket);
+  relayServer.register(websocket);
 
   relayServer.get("/", { websocket: true }, (connection) => {
-    relay(swarm.dht, new Stream(false, connection.socket));
+    relay(dht, new Stream(false, connection.socket));
-    connection.socket.binaryType = "nodebuffer";
   });
 
-  let port = config.uint("core.relayPort");
+  await relayServer.listen({ port: config.uint("core.port"), host: "0.0.0.0" });
-
-  if (!port) {
-    port = config.uint("core.port") as number;
-  }
-
-  await relayServer.listen({ port, host: "0.0.0.0" });
 }

src/modules/rpc.ts

@@ -7,24 +7,16 @@ import { errorExit } from "../lib/error.js";
 import stringify from "json-stable-stringify";
 import {
   getRpcServer,
-  RPC_PROTOCOL_ID,
   RPC_PROTOCOL_SYMBOL,
   setupStream,
 } from "./rpc/server.js";
 import { get as getSwarm, SecretStream } from "./swarm.js";
 import b4a from "b4a";
-// @ts-ignore
-import Protomux from "protomux";
 
 export async function start() {
-  getSwarm().on("connection", (stream: SecretStream) => {
+  getSwarm().on("connection", (stream: SecretStream) =>
-    Protomux.from(stream).pair(
+    getRpcServer().setup(stream)
-      { protocol: "protomux-rpc", id: RPC_PROTOCOL_ID },
-      async () => {
-        getRpcServer().setup(stream);
-      }
   );
-  });
 }
 
 export async function getRpcByPeer(peer: Buffer | string) {
@@ -38,16 +30,15 @@ export async function getRpcByPeer(peer: Buffer | string) {
   }
 
   return new Promise((resolve) => {
-    const listener = (peer: any, info: any) => {
+    const listener = () => {};
+    swarm.on("connection", (peer: any, info: any) => {
       if (info.publicKey.toString("hex") !== peer.toString("hex")) {
         return;
       }
       swarm.removeListener("connection", listener);
 
       resolve(setupStream(peer));
-    };
+    });
-
-    swarm.on("connection", listener);
 
     swarm.joinPeer(peer);
   });

src/modules/rpc/server.ts

@@ -3,7 +3,7 @@ import {
   RPCMethod,
   RPCRequest,
   RPCResponse,
-} from "@lumeweb/interface-relay";
+} from "@lumeweb/relay-types";
 import EventEmitter from "events";
 // @ts-ignore
 import ProtomuxRPC from "protomux-rpc";
@@ -21,7 +21,7 @@ import jsonStringify from "json-stringify-deterministic";
 const sodium = require("sodium-universal");
 let server: RPCServer;
 
-export const RPC_PROTOCOL_ID = b4a.from("lumeweb");
+const RPC_PROTOCOL_ID = b4a.from("lumeweb");
 export const RPC_PROTOCOL_SYMBOL = Symbol.for(RPC_PROTOCOL_ID.toString());
 
 export function getRpcServer(): RPCServer {
@@ -43,8 +43,6 @@ export function setupStream(stream: SecretStream) {
     valueEncoding: c.json,
   });
 
-  stream.setKeepAlive(5000);
-
   return stream[RPC_PROTOCOL_SYMBOL];
 }
 

src/modules/swarm.ts

@@ -69,9 +69,6 @@ export class ProtocolManager {
     this._swarm = swarm;
 
     this._swarm.on("connection", (peer: any) => {
-      if (!peer.userData) {
-        peer.userData = null;
-      }
       for (const protocol of this._protocols) {
         Protomux.from(peer).pair(
           { protocol: protocol[0] },

src/plugins/core.ts

@@ -1,15 +1,8 @@
-import { Plugin, PluginAPI } from "@lumeweb/interface-relay";
+import { Plugin, PluginAPI } from "@lumeweb/relay-types";
-
-import defer from "p-defer";
 
 const plugin: Plugin = {
   name: "core",
   async plugin(api: PluginAPI): Promise<void> {
-    const pluginsLoaded = defer();
-    api.once("core.pluginsLoaded", () => {
-      pluginsLoaded.resolve();
-    });
-
     api.registerMethod("ping", {
       cacheable: false,
       async handler(): Promise<any> {
@@ -20,8 +13,6 @@ const plugin: Plugin = {
     api.registerMethod("get_methods", {
       cacheable: false,
       async handler(): Promise<any> {
-        await pluginsLoaded.promise;
-
         return api.rpcServer.getMethods();
       },
     });

src/plugins/dht.ts

@@ -1,4 +1,4 @@
-import { Plugin, PluginAPI } from "@lumeweb/interface-relay";
+import { Plugin, PluginAPI } from "@lumeweb/relay-types";
 import b4a from "b4a";
 
 const plugin: Plugin = {

src/plugins/rpc.ts

@@ -3,11 +3,12 @@ import {
   PluginAPI,
   RPCBroadcastRequest,
   RPCBroadcastResponse,
+  RPCCacheItem,
   RPCRequest,
   RPCResponse,
-} from "@lumeweb/interface-relay";
+} from "@lumeweb/relay-types";
 import { getRpcByPeer } from "../modules/rpc";
-import { get as getSwarm } from "../modules/swarm";
+import { get as getSwarm, LUMEWEB_TOPIC_HASH } from "../modules/swarm";
 import b4a from "b4a";
 import pTimeout, { ClearablePromise } from "p-timeout";
 
@@ -49,6 +50,91 @@ const plugin: Plugin = {
   name: "rpc",
   async plugin(_api: PluginAPI): Promise<void> {
     api = _api;
+    if (api.config.bool("cache")) {
+      api.registerMethod("get_cached_item", {
+        cacheable: false,
+        async handler(req: string): Promise<RPCResponse> {
+          if (typeof req !== "string") {
+            throw new Error("item must be a string");
+          }
+
+          const cache = api.rpcServer.cache?.data;
+
+          if (!cache?.has(req)) {
+            throw new Error("item does not exist");
+          }
+
+          return {
+            data: true,
+            ...cache.get<RPCCacheItem>(req)?.value,
+            signature: cache.get<RPCCacheItem>(req)?.signature,
+          };
+        },
+      });
+      api.registerMethod("clear_cached_item", {
+        cacheable: false,
+        async handler(req: string): Promise<void> {
+          if (typeof req !== "string") {
+            throw new Error("item must be a string");
+          }
+          try {
+            api.rpcServer.cache.deleteItem(req);
+          } catch (e: any) {
+            throw e;
+          }
+        },
+      });
+      api.registerMethod("get_peers", {
+        cacheable: false,
+        async handler(): Promise<string[]> {
+          const pubkey = b4a.from(api.identity.publicKeyRaw).toString("hex");
+
+          const online = api.rpcServer.cache?.dhtCache.online || new Set();
+          if (online.has(pubkey)) {
+            online.delete(pubkey);
+          }
+
+          return [...online];
+        },
+      });
+      if (api.logger.level === "debug") {
+        api.registerMethod("get_direct_peers", {
+          cacheable: false,
+          async handler(): Promise<string[]> {
+            const online = api.rpcServer.cache.dhtCache.online;
+            const pubkey = b4a
+              .from(api.swarm.keyPair.publicKeyRaw())
+              .toString("hex");
+
+            if (online.has(pubkey)) {
+              online.delete(pubkey);
+            }
+
+            const topic = LUMEWEB_TOPIC_HASH.toString("hex");
+            return [...api.swarm.peers.values()]
+              .filter((item: any) =>
+                [...item._seenTopics.keys()].includes(topic)
+              )
+              .map((item: any) => item.publicKey.toString("hex"))
+              .filter((item: any) => online.has(item));
+          },
+        });
+        api.registerMethod("get_bootstrap_info", {
+          cacheable: false,
+          async handler(): Promise<string[]> {
+            // @ts-ignore
+            return api.rpcServer.cache.dhtCache._getBootstrapInfo();
+          },
+        });
+        api.registerMethod("get_connected_peers", {
+          cacheable: false,
+          async handler(): Promise<string[]> {
+            // @ts-ignore
+            return [...api.rpcServer.cache.dhtCache.connectedTo];
+          },
+        });
+      }
+    }
     api.registerMethod("broadcast_request", {
       cacheable: false,
       async handler(req: RPCBroadcastRequest): Promise<RPCBroadcastResponse> {

tsconfig.json

@@ -2,7 +2,7 @@
     "compilerOptions": {
         "target": "es2020",
         "module": "commonjs",
-        "moduleResolution": "nodenext",
+        "moduleResolution": "node",
         "rootDir": "src",
         "outDir": "build",
         "esModuleInterop": true,