fix: add stream setKeepAlive for rpc connections

# [0.1.0-develop.2](https://git.lumeweb.com/LumeWeb/relay/compare/v0.1.0-develop.1...v0.1.0-develop.2) (2023-07-06)

### Bug Fixes

* fastify needs register to be awaited, connection binaryType set to nodebuffer after hyper setup, and no http2 ([d1c79ac](d1c79acd72))
* need to pass the dht property to dht-relay ([7cb443f](7cb443f62b))

.npmrc

@@ -0,0 +1,4 @@
+public-hoist-pattern[]=udx-native
+public-hoist-pattern[]=sodium-native
+public-hoist-pattern[]=@hyperswarm/dht
+public-hoist-pattern[]=hypercore-crypto

.releaserc

@@ -0,0 +1,38 @@
+{
+    "plugins": [
+        "@semantic-release/commit-analyzer",
+        "@semantic-release/release-notes-generator",
+        "@semantic-release/changelog",
+        [
+            "@semantic-release/exec",
+            {
+                "publishCmd": "git clone https://git.lumeweb.com/LumeWeb/circle-ci-publish.git ci; ./ci/publish.sh \"${nextRelease.version}\""
+            }
+        ],
+        [
+            "@semantic-release/npm",
+            {
+                "npmPublish": false
+            }
+        ],
+        [
+            "@semantic-release/git",
+            {
+                "assets": [
+                    "package.json"
+                ]
+            }
+        ]
+    ],
+    "branches": [
+        "master",
+        {
+            name: "develop",
+            prerelease: true
+        },
+        {
+            name: "develop-*",
+            prerelease: true
+        },
+    ]
+}

.woodpecker.yml

@@ -1,49 +0,0 @@
-pipeline:
-    build:
-        image: git.lumeweb.com/lumeweb/ci-node
-        commands:
-            - corepack enable
-            - corepack prepare yarn@stable --activate
-            - yarn
-            - yarn build
-    package:
-        image: ghcr.io/goreleaser/nfpm
-        commands:
-            - nfpm pkg --packager deb
-    publish_focal:
-        image: git.lumeweb.com/lumeweb/aptly-publisher
-        settings:
-            apt_username:
-                from_secret: APT_USERNAME
-            apt_password:
-                from_secret: APT_PASSWORD
-            repo: apt.web3relay.io
-            folder: ubuntu
-            distro: focal
-            gpg_password:
-                from_secret: GPG_PASSWORD
-    publish_jammy:
-        image: git.lumeweb.com/lumeweb/aptly-publisher
-        settings:
-            apt_username:
-                from_secret: APT_USERNAME
-            apt_password:
-                from_secret: APT_PASSWORD
-            repo: apt.web3relay.io
-            folder: ubuntu
-            distro: jammy
-            gpg_password:
-                from_secret: GPG_PASSWORD
-    purge_do_cdn:
-        image: git.lumeweb.com/lumeweb/do-cdn-purge
-        settings:
-            DIGITALOCEAN_ACCESS_TOKEN:
-                from_secret: digitalocean_access_token
-            DIGITALOCEAN_CDN_ID:
-                from_secret: digitalocean_cdn_id
-    purge_cf_cdn:
-        image: git.lumeweb.com/lumeweb/cf-cdn-purge
-        settings:
-            CLOUDFLARE_ACCESS_TOKEN:
-                from_secret: cloudflare_access_token
-            DOMAIN: web3relay.io

.yarn/patches/micro-ed25519-hdkey-npm-0.1.2-5d1d767d1d.patch

@@ -1,18 +0,0 @@
-diff --git a/index.js b/index.js
-index dc628dab1a201e6679a1716b3d51b56e4723b49d..9daeb44298586d51d7bb6e859a1dd696846e0078 100644
---- a/index.js
-+++ b/index.js
-@@ -115,11 +115,11 @@ class HDKey {
-         });
-     }
-     sign(message) {
--        message = ensureBytes(message, 32);
-+        message = ensureBytes(message);
-         return ed25519.sync.sign(message, this.privateKey);
-     }
-     verify(message, signature) {
--        message = ensureBytes(message, 32);
-+        message = ensureBytes(message);
-         signature = ensureBytes(signature, 64);
-         return ed25519.sync.verify(signature, message, this.publicKeyRaw);
-     }

.yarnrc.yml

@@ -1 +0,0 @@
-nodeLinker: node-modules

package.json

@@ -1,79 +1,75 @@
 {
     "name": "@lumeweb/relay",
     "type": "commonjs",
-    "version": "0.1.0",
+    "version": "0.1.0-develop.2",
     "description": "",
     "main": "build/index.js",
-    "types": "src/types.ts",
+    "repository": {
+        "url": "gitea@git.lumeweb.com:LumeWeb/relay.git"
+    },
     "author": {
         "name": "Derrick Hammer",
         "email": "contact@lumeweb.com"
     },
     "scripts": {
-        "compile": "yarn tsc",
+        "semantic-release": "semantic-release",
+        "compile": "tsc",
         "prebuild": "bash prebuild.sh",
-        "package": "yarn pkg -c pkg.json build/index.js -t linux  --public --no-native-build -C gzip",
+        "package": "pkg -c pkg.json build/index.js -t linux  --public --no-native-build -C gzip",
-        "package-debug": "yarn pkg -c pkg.json build/index.js -b -t linux --no-bytecode  --public",
+        "build": "npm run compile && npm run prebuild && npm run package",
-        "build": "yarn run compile && npm run prebuild && npm run package",
+        "postinstall": "patch-package"
-        "barebuild": "yarn run compile && npm run package"
     },
     "dependencies": {
-        "@fastify/websocket": "^7.1.1",
+        "@fastify/websocket": "^8.1.0",
-        "@hyperswarm/dht-relay": "^0.3.0",
+        "@hyperswarm/dht-relay": "^0.4.0",
-        "@lumeweb/cfg": "https://git.lumeweb.com/LumeWeb/cfg.git",
+        "@lumeweb/interface-relay": "^0.0.2-develop.3",
-        "@lumeweb/dht-cache": "https://git.lumeweb.com/LumeWeb/dht-cache.git",
+        "@lumeweb/relay-cfg": "^0.1.15",
-        "@scure/bip39": "^1.1.0",
+        "@scure/bip39": "^1.2.0",
-        "@types/acme-client": "^3.3.0",
+        "@types/node": "^18.15.11",
-        "@types/node": "^18.0.0",
+        "@types/ws": "^8.5.4",
-        "@types/node-cron": "^3.0.2",
-        "@types/ws": "^8.5.3",
-        "ajv": "^8.11.0",
         "async-mutex": "^0.3.2",
-        "b4a": "^1.6.1",
+        "b4a": "^1.6.3",
         "compact-encoding": "^2.11.0",
-        "date-fns": "^2.28.0",
+        "dotenv": "^16.0.3",
-        "dotenv": "^16.0.1",
+        "ed25519-keygen": "github:LumeWeb/ed25519-keygen",
-        "ethers": "^5.6.9",
+        "ethers": "^5.7.2",
         "eventemitter2": "^6.4.9",
-        "fastify": "^4.10.2",
+        "fastify": "^4.19.2",
-        "fetch-blob": "https://github.com/LumeWeb/fetch-blob.git",
+        "fetch-blob": "github:LumeWeb/fetch-blob",
-        "hyperswarm": "^4.3.5",
+        "hyperswarm": "^4.4.0",
-        "json-stable-stringify": "^1.0.1",
+        "json-stable-stringify": "^1.0.2",
-        "json-stringify-deterministic": "^1.0.7",
+        "json-stringify-deterministic": "^1.0.8",
-        "loady": "https://github.com/LumeWeb/loady.git",
+        "loady": "github:LumeWeb/loady",
-        "micro-ed25519-hdkey": "patch:micro-ed25519-hdkey@npm%3A0.1.2#~/.yarn/patches/micro-ed25519-hdkey-npm-0.1.2-5d1d767d1d.patch",
+        "msgpackr": "^1.8.5",
-        "msgpackr": "^1.6.1",
         "node-cache": "^5.1.2",
-        "node-cron": "^3.0.1",
+        "node-fetch": "^2.6.9",
-        "node-fetch": "2",
+        "p-defer": "git+https://git.lumeweb.com/LumeWeb/p-defer.git",
-        "p-timeout": "https://git.lumeweb.com/LumeWeb/p-timeout.git",
+        "p-timeout": "git+https://git.lumeweb.com/LumeWeb/p-timeout.git",
-        "pino": "^8.8.0",
+        "pino": "^8.11.0",
-        "pino-pretty": "^9.1.1",
+        "pino-pretty": "^9.4.0",
         "promise-retry": "^2.0.1",
-        "protomux": "^3.4.0",
+        "protomux": "^3.4.1",
-        "protomux-rpc": "^1.3.0",
+        "protomux-rpc": "^1.4.1",
-        "random-access-memory": "^4.1.0",
         "random-key": "^0.3.2",
-        "slugify": "^1.6.5",
+        "slugify": "^1.6.6",
-        "sodium-universal": "^3.1.0"
+        "sodium-universal": "^4.0.0"
     },
     "devDependencies": {
-        "@lumeweb/relay-types": "https://git.lumeweb.com/LumeWeb/relay-types.git",
+        "@semantic-release/changelog": "^6.0.3",
+        "@semantic-release/exec": "^6.0.3",
+        "@semantic-release/git": "^10.0.1",
         "@types/b4a": "^1.6.0",
         "@types/minimatch": "^3.0.5",
-        "@types/node-fetch": "^2.6.2",
+        "@types/node-fetch": "^2.6.3",
-        "cli-progress": "^3.11.2",
+        "cli-progress": "^3.12.0",
         "hyper-typings": "^1.0.0",
-        "node-gyp": "^9.1.0",
+        "node-gyp": "^9.3.1",
-        "pkg": "^5.8.0",
+        "patch-package": "^6.5.1",
+        "pkg": "^5.8.1",
         "prebuildify": "^5.0.1",
-        "prettier": "^2.7.1",
+        "prettier": "^2.8.7",
-        "rollup": "^2.77.0",
+        "semantic-release": "21",
-        "supports-color": "https://github.com/LumeWeb/supports-color.git",
+        "supports-color": "github:LumeWeb/supports-color",
-        "typescript": "^4.9.4"
+        "typescript": "^4.9.5"
-    },
-    "packageManager": "yarn@4.0.0-rc.33",
-    "resolutions": {
-        "micro-ed25519-hdkey@npm:^0.1.2": "patch:micro-ed25519-hdkey@npm%3A0.1.2#~/.yarn/patches/micro-ed25519-hdkey-npm-0.1.2-5d1d767d1d.patch"
     }
 }

patches/@lumeweb+interface-relay+0.0.2-develop.3.patch

@@ -0,0 +1,12 @@
+diff --git a/node_modules/@lumeweb/interface-relay/package.json b/node_modules/@lumeweb/interface-relay/package.json
+index 0bfaba0..c503d91 100644
+--- a/node_modules/@lumeweb/interface-relay/package.json
++++ b/node_modules/@lumeweb/interface-relay/package.json
+@@ -2,7 +2,6 @@
+     "name": "@lumeweb/interface-relay",
+     "version": "0.0.2-develop.3",
+     "main": "lib/index.js",
+-    "type": "module",
+     "types": "lib/index.d.ts",
+     "module": "lib/index.mjs",
+     "exports": {

patches/sodium-native+4.0.1.patch

@@ -0,0 +1,20 @@
+diff --git a/node_modules/sodium-native/package.json b/node_modules/sodium-native/package.json
+index bda9dd4..3a5541a 100644
+--- a/node_modules/sodium-native/package.json
++++ b/node_modules/sodium-native/package.json
+@@ -3,15 +3,6 @@
+   "version": "4.0.1",
+   "description": "Low level bindings for libsodium",
+   "main": "index.js",
+-  "files": [
+-    "index.js",
+-    "deps/**",
+-    "modules/**",
+-    "binding.c",
+-    "binding.gyp",
+-    "macros.h",
+-    "prebuilds/**"
+-  ],
+   "dependencies": {
+     "node-gyp-build": "^4.3.0"
+   },

pkg.json

@@ -1,9 +1,5 @@
 {
     "assets": [
-        "node_modules/*/build/Release/*.node",
-        "node_modules/libskynet",
-        "node_modules/libskynetnode",
-        "node_modules/@lumeweb"
     ],
     "outputPath": "dist"
 }

src/config.ts

@@ -1,11 +1,11 @@
 // @ts-ignore
-import Config from "@lumeweb/cfg";
+import { Config } from "@lumeweb/relay-cfg";
 import * as os from "os";
 import * as fs from "fs";
 import path from "path";
 import log from "./log.js";
 
-const config = new Config("lumeweb-relay", "core.confdir");
+const config = new Config("lumeweb-relay", "core.confDir");
 
 let configDir;
 
@@ -26,6 +26,7 @@ switch (os.platform()) {
 config.inject({
   "core.confDir": configDir,
   "core.port": 8080,
+  "core.relayPort": 0,
   "core.appPort": 80,
   "core.logLevel": "info",
   "core.pluginDir": path.resolve(configDir, "..", "plugins"),
@@ -33,7 +34,7 @@ config.inject({
 
 config.load();
 
-configDir = config.str("core.confdir");
+configDir = config.str("core.confDir") as string;
 
 if (fs.existsSync(configDir)) {
   try {

src/lib/seed.ts

@@ -1,4 +1,4 @@
-import { HDKey } from "micro-ed25519-hdkey";
+import { HDKey } from "ed25519-keygen/hdkey";
 import config from "../config";
 import * as bip39 from "@scure/bip39";
 import { wordlist } from "@scure/bip39/wordlists/english";
@@ -8,7 +8,7 @@ import b4a from "b4a";
 const BIP44_PATH = "m/44'/1627'/0'/0'/0'";
 
 export function getSeed() {
-  const seed = config.str("core.seed");
+  const seed = config.str("core.seed") as string;
 
   let valid = bip39.validateMnemonic(seed, wordlist);
   if (!valid) {

src/modules/app.ts

@@ -20,7 +20,10 @@ export async function start() {
 
   await getPluginAPI().emitAsync("core.appServer.buildRoutes");
 
-  await app.listen({ port: config.uint("core.appport"), host: "0.0.0.0" });
+  await app.listen({
+    port: config.uint("core.appPort") as number,
+    host: "0.0.0.0",
+  });
 
   getPluginAPI().emit("core.appServer.started");
 }

src/modules/plugin.ts

@@ -1,14 +1,14 @@
 import config from "../config.js";
 import type { RPCServer } from "./rpc/server.js";
 import { getRpcServer } from "./rpc/server.js";
-import type { Plugin, RPCMethod } from "@lumeweb/relay-types";
+import type { Plugin, RPCMethod } from "@lumeweb/interface-relay";
 import slugify from "slugify";
 import * as fs from "fs";
 import path from "path";
 import type { Logger } from "pino";
 
 import { getHDKey, getSeed } from "../lib/seed.js";
-import type Config from "@lumeweb/cfg";
+import type { Config } from "@lumeweb/relay-cfg";
 import EventEmitter2 from "eventemitter2";
 import log from "../log.js";
 import {
@@ -17,7 +17,7 @@ import {
   ProtocolManager,
 } from "./swarm.js";
 import { get as getApp } from "./app.js";
-import type { HDKey } from "micro-ed25519-hdkey";
+import type { HDKey } from "ed25519-keygen/hdkey";
 import corePlugins from "../plugins";
 import Util from "./plugin/util";
 
@@ -250,7 +250,9 @@ export async function loadPlugins() {
     await apiManager.loadPluginInstance(plugin);
   }
 
-  for (const plugin of [...new Set(config.array("core.plugins", []))] as []) {
+  for (const plugin of [
+    ...new Set(config.array("core.plugins", [] as any)),
+  ] as []) {
     await apiManager.loadPlugin(plugin);
   }
 

src/modules/relay.ts

@@ -16,18 +16,24 @@ import * as http2 from "http2";
 import websocket from "@fastify/websocket";
 
 export async function start() {
-  const dht = getSwarm();
+  const swarm = getSwarm();
 
   let relayServer = fastify({
-    http2: true,
     logger: log.child({ module: "relay-server" }),
   });
 
-  relayServer.register(websocket);
+  await relayServer.register(websocket);
 
   relayServer.get("/", { websocket: true }, (connection) => {
-    relay(dht, new Stream(false, connection.socket));
+    relay(swarm.dht, new Stream(false, connection.socket));
+    connection.socket.binaryType = "nodebuffer";
   });
 
-  await relayServer.listen({ port: config.uint("core.port"), host: "0.0.0.0" });
+  let port = config.uint("core.relayPort");
+
+  if (!port) {
+    port = config.uint("core.port") as number;
+  }
+
+  await relayServer.listen({ port, host: "0.0.0.0" });
 }

src/modules/rpc.ts

@@ -7,16 +7,24 @@ import { errorExit } from "../lib/error.js";
 import stringify from "json-stable-stringify";
 import {
   getRpcServer,
+  RPC_PROTOCOL_ID,
   RPC_PROTOCOL_SYMBOL,
   setupStream,
 } from "./rpc/server.js";
 import { get as getSwarm, SecretStream } from "./swarm.js";
 import b4a from "b4a";
+// @ts-ignore
+import Protomux from "protomux";
 
 export async function start() {
-  getSwarm().on("connection", (stream: SecretStream) =>
+  getSwarm().on("connection", (stream: SecretStream) => {
-    getRpcServer().setup(stream)
+    Protomux.from(stream).pair(
+      { protocol: "protomux-rpc", id: RPC_PROTOCOL_ID },
+      async () => {
+        getRpcServer().setup(stream);
+      }
     );
+  });
 }
 
 export async function getRpcByPeer(peer: Buffer | string) {
@@ -30,15 +38,16 @@ export async function getRpcByPeer(peer: Buffer | string) {
   }
 
   return new Promise((resolve) => {
-    const listener = () => {};
+    const listener = (peer: any, info: any) => {
-    swarm.on("connection", (peer: any, info: any) => {
       if (info.publicKey.toString("hex") !== peer.toString("hex")) {
         return;
       }
       swarm.removeListener("connection", listener);
 
       resolve(setupStream(peer));
-    });
+    };
+
+    swarm.on("connection", listener);
 
     swarm.joinPeer(peer);
   });

src/modules/rpc/server.ts

@@ -3,7 +3,7 @@ import {
   RPCMethod,
   RPCRequest,
   RPCResponse,
-} from "@lumeweb/relay-types";
+} from "@lumeweb/interface-relay";
 import EventEmitter from "events";
 // @ts-ignore
 import ProtomuxRPC from "protomux-rpc";
@@ -21,7 +21,7 @@ import jsonStringify from "json-stringify-deterministic";
 const sodium = require("sodium-universal");
 let server: RPCServer;
 
-const RPC_PROTOCOL_ID = b4a.from("lumeweb");
+export const RPC_PROTOCOL_ID = b4a.from("lumeweb");
 export const RPC_PROTOCOL_SYMBOL = Symbol.for(RPC_PROTOCOL_ID.toString());
 
 export function getRpcServer(): RPCServer {
@@ -43,6 +43,8 @@ export function setupStream(stream: SecretStream) {
     valueEncoding: c.json,
   });
 
+  stream.setKeepAlive(5000);
+
   return stream[RPC_PROTOCOL_SYMBOL];
 }
 

src/modules/swarm.ts

@@ -69,6 +69,9 @@ export class ProtocolManager {
     this._swarm = swarm;
 
     this._swarm.on("connection", (peer: any) => {
+      if (!peer.userData) {
+        peer.userData = null;
+      }
       for (const protocol of this._protocols) {
         Protomux.from(peer).pair(
           { protocol: protocol[0] },

src/plugins/core.ts

@@ -1,8 +1,15 @@
-import { Plugin, PluginAPI } from "@lumeweb/relay-types";
+import { Plugin, PluginAPI } from "@lumeweb/interface-relay";
+
+import defer from "p-defer";
 
 const plugin: Plugin = {
   name: "core",
   async plugin(api: PluginAPI): Promise<void> {
+    const pluginsLoaded = defer();
+    api.once("core.pluginsLoaded", () => {
+      pluginsLoaded.resolve();
+    });
+
     api.registerMethod("ping", {
       cacheable: false,
       async handler(): Promise<any> {
@@ -13,6 +20,8 @@ const plugin: Plugin = {
     api.registerMethod("get_methods", {
       cacheable: false,
       async handler(): Promise<any> {
+        await pluginsLoaded.promise;
+
         return api.rpcServer.getMethods();
       },
     });

src/plugins/dht.ts

@@ -1,4 +1,4 @@
-import { Plugin, PluginAPI } from "@lumeweb/relay-types";
+import { Plugin, PluginAPI } from "@lumeweb/interface-relay";
 import b4a from "b4a";
 
 const plugin: Plugin = {

src/plugins/rpc.ts

@@ -3,12 +3,11 @@ import {
   PluginAPI,
   RPCBroadcastRequest,
   RPCBroadcastResponse,
-  RPCCacheItem,
   RPCRequest,
   RPCResponse,
-} from "@lumeweb/relay-types";
+} from "@lumeweb/interface-relay";
 import { getRpcByPeer } from "../modules/rpc";
-import { get as getSwarm, LUMEWEB_TOPIC_HASH } from "../modules/swarm";
+import { get as getSwarm } from "../modules/swarm";
 import b4a from "b4a";
 import pTimeout, { ClearablePromise } from "p-timeout";
 
@@ -50,91 +49,6 @@ const plugin: Plugin = {
   name: "rpc",
   async plugin(_api: PluginAPI): Promise<void> {
     api = _api;
-    if (api.config.bool("cache")) {
-      api.registerMethod("get_cached_item", {
-        cacheable: false,
-        async handler(req: string): Promise<RPCResponse> {
-          if (typeof req !== "string") {
-            throw new Error("item must be a string");
-          }
-
-          const cache = api.rpcServer.cache?.data;
-
-          if (!cache?.has(req)) {
-            throw new Error("item does not exist");
-          }
-
-          return {
-            data: true,
-            ...cache.get<RPCCacheItem>(req)?.value,
-            signature: cache.get<RPCCacheItem>(req)?.signature,
-          };
-        },
-      });
-      api.registerMethod("clear_cached_item", {
-        cacheable: false,
-        async handler(req: string): Promise<void> {
-          if (typeof req !== "string") {
-            throw new Error("item must be a string");
-          }
-          try {
-            api.rpcServer.cache.deleteItem(req);
-          } catch (e: any) {
-            throw e;
-          }
-        },
-      });
-      api.registerMethod("get_peers", {
-        cacheable: false,
-        async handler(): Promise<string[]> {
-          const pubkey = b4a.from(api.identity.publicKeyRaw).toString("hex");
-
-          const online = api.rpcServer.cache?.dhtCache.online || new Set();
-          if (online.has(pubkey)) {
-            online.delete(pubkey);
-          }
-
-          return [...online];
-        },
-      });
-      if (api.logger.level === "debug") {
-        api.registerMethod("get_direct_peers", {
-          cacheable: false,
-          async handler(): Promise<string[]> {
-            const online = api.rpcServer.cache.dhtCache.online;
-            const pubkey = b4a
-              .from(api.swarm.keyPair.publicKeyRaw())
-              .toString("hex");
-
-            if (online.has(pubkey)) {
-              online.delete(pubkey);
-            }
-
-            const topic = LUMEWEB_TOPIC_HASH.toString("hex");
-            return [...api.swarm.peers.values()]
-              .filter((item: any) =>
-                [...item._seenTopics.keys()].includes(topic)
-              )
-              .map((item: any) => item.publicKey.toString("hex"))
-              .filter((item: any) => online.has(item));
-          },
-        });
-        api.registerMethod("get_bootstrap_info", {
-          cacheable: false,
-          async handler(): Promise<string[]> {
-            // @ts-ignore
-            return api.rpcServer.cache.dhtCache._getBootstrapInfo();
-          },
-        });
-        api.registerMethod("get_connected_peers", {
-          cacheable: false,
-          async handler(): Promise<string[]> {
-            // @ts-ignore
-            return [...api.rpcServer.cache.dhtCache.connectedTo];
-          },
-        });
-      }
-    }
     api.registerMethod("broadcast_request", {
       cacheable: false,
       async handler(req: RPCBroadcastRequest): Promise<RPCBroadcastResponse> {

tsconfig.json

@@ -2,7 +2,7 @@
     "compilerOptions": {
         "target": "es2020",
         "module": "commonjs",
-        "moduleResolution": "node",
+        "moduleResolution": "nodenext",
         "rootDir": "src",
         "outDir": "build",
         "esModuleInterop": true,