Compare commits

..

No commits in common. "develop" and "master" have entirely different histories.

23 changed files with 4797 additions and 11548 deletions

4
.npmrc
View File

@ -1,4 +0,0 @@
public-hoist-pattern[]=udx-native
public-hoist-pattern[]=sodium-native
public-hoist-pattern[]=@hyperswarm/dht
public-hoist-pattern[]=hypercore-crypto

View File

@ -1,38 +0,0 @@
{
"plugins": [
"@semantic-release/commit-analyzer",
"@semantic-release/release-notes-generator",
"@semantic-release/changelog",
[
"@semantic-release/exec",
{
"publishCmd": "git clone https://git.lumeweb.com/LumeWeb/circle-ci-publish.git ci; ./ci/publish.sh \"${nextRelease.version}\""
}
],
[
"@semantic-release/npm",
{
"npmPublish": false
}
],
[
"@semantic-release/git",
{
"assets": [
"package.json"
]
}
]
],
"branches": [
"master",
{
name: "develop",
prerelease: true
},
{
name: "develop-*",
prerelease: true
},
]
}

49
.woodpecker.yml Normal file
View File

@ -0,0 +1,49 @@
pipeline:
build:
image: git.lumeweb.com/lumeweb/ci-node
commands:
- corepack enable
- corepack prepare yarn@stable --activate
- yarn
- yarn build
package:
image: ghcr.io/goreleaser/nfpm
commands:
- nfpm pkg --packager deb
publish_focal:
image: git.lumeweb.com/lumeweb/aptly-publisher
settings:
apt_username:
from_secret: APT_USERNAME
apt_password:
from_secret: APT_PASSWORD
repo: apt.web3relay.io
folder: ubuntu
distro: focal
gpg_password:
from_secret: GPG_PASSWORD
publish_jammy:
image: git.lumeweb.com/lumeweb/aptly-publisher
settings:
apt_username:
from_secret: APT_USERNAME
apt_password:
from_secret: APT_PASSWORD
repo: apt.web3relay.io
folder: ubuntu
distro: jammy
gpg_password:
from_secret: GPG_PASSWORD
purge_do_cdn:
image: git.lumeweb.com/lumeweb/do-cdn-purge
settings:
DIGITALOCEAN_ACCESS_TOKEN:
from_secret: digitalocean_access_token
DIGITALOCEAN_CDN_ID:
from_secret: digitalocean_cdn_id
purge_cf_cdn:
image: git.lumeweb.com/lumeweb/cf-cdn-purge
settings:
CLOUDFLARE_ACCESS_TOKEN:
from_secret: cloudflare_access_token
DOMAIN: web3relay.io

View File

@ -0,0 +1,18 @@
diff --git a/index.js b/index.js
index dc628dab1a201e6679a1716b3d51b56e4723b49d..9daeb44298586d51d7bb6e859a1dd696846e0078 100644
--- a/index.js
+++ b/index.js
@@ -115,11 +115,11 @@ class HDKey {
});
}
sign(message) {
- message = ensureBytes(message, 32);
+ message = ensureBytes(message);
return ed25519.sync.sign(message, this.privateKey);
}
verify(message, signature) {
- message = ensureBytes(message, 32);
+ message = ensureBytes(message);
signature = ensureBytes(signature, 64);
return ed25519.sync.verify(signature, message, this.publicKeyRaw);
}

1
.yarnrc.yml Normal file
View File

@ -0,0 +1 @@
nodeLinker: node-modules

11363
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,75 +1,79 @@
{
"name": "@lumeweb/relay",
"type": "commonjs",
"version": "0.1.0-develop.2",
"version": "0.1.0",
"description": "",
"main": "build/index.js",
"repository": {
"url": "gitea@git.lumeweb.com:LumeWeb/relay.git"
},
"types": "src/types.ts",
"author": {
"name": "Derrick Hammer",
"email": "contact@lumeweb.com"
},
"scripts": {
"semantic-release": "semantic-release",
"compile": "tsc",
"compile": "yarn tsc",
"prebuild": "bash prebuild.sh",
"package": "pkg -c pkg.json build/index.js -t linux --public --no-native-build -C gzip",
"build": "npm run compile && npm run prebuild && npm run package",
"postinstall": "patch-package"
"package": "yarn pkg -c pkg.json build/index.js -t linux --public --no-native-build -C gzip",
"package-debug": "yarn pkg -c pkg.json build/index.js -b -t linux --no-bytecode --public",
"build": "yarn run compile && npm run prebuild && npm run package",
"barebuild": "yarn run compile && npm run package"
},
"dependencies": {
"@fastify/websocket": "^8.1.0",
"@hyperswarm/dht-relay": "^0.4.0",
"@lumeweb/interface-relay": "^0.0.2-develop.3",
"@lumeweb/relay-cfg": "^0.1.15",
"@scure/bip39": "^1.2.0",
"@types/node": "^18.15.11",
"@types/ws": "^8.5.4",
"@fastify/websocket": "^7.1.1",
"@hyperswarm/dht-relay": "^0.3.0",
"@lumeweb/cfg": "https://git.lumeweb.com/LumeWeb/cfg.git",
"@lumeweb/dht-cache": "https://git.lumeweb.com/LumeWeb/dht-cache.git",
"@scure/bip39": "^1.1.0",
"@types/acme-client": "^3.3.0",
"@types/node": "^18.0.0",
"@types/node-cron": "^3.0.2",
"@types/ws": "^8.5.3",
"ajv": "^8.11.0",
"async-mutex": "^0.3.2",
"b4a": "^1.6.3",
"b4a": "^1.6.1",
"compact-encoding": "^2.11.0",
"dotenv": "^16.0.3",
"ed25519-keygen": "github:LumeWeb/ed25519-keygen",
"ethers": "^5.7.2",
"date-fns": "^2.28.0",
"dotenv": "^16.0.1",
"ethers": "^5.6.9",
"eventemitter2": "^6.4.9",
"fastify": "^4.19.2",
"fetch-blob": "github:LumeWeb/fetch-blob",
"hyperswarm": "^4.4.0",
"json-stable-stringify": "^1.0.2",
"json-stringify-deterministic": "^1.0.8",
"loady": "github:LumeWeb/loady",
"msgpackr": "^1.8.5",
"fastify": "^4.10.2",
"fetch-blob": "https://github.com/LumeWeb/fetch-blob.git",
"hyperswarm": "^4.3.5",
"json-stable-stringify": "^1.0.1",
"json-stringify-deterministic": "^1.0.7",
"loady": "https://github.com/LumeWeb/loady.git",
"micro-ed25519-hdkey": "patch:micro-ed25519-hdkey@npm%3A0.1.2#~/.yarn/patches/micro-ed25519-hdkey-npm-0.1.2-5d1d767d1d.patch",
"msgpackr": "^1.6.1",
"node-cache": "^5.1.2",
"node-fetch": "^2.6.9",
"p-defer": "git+https://git.lumeweb.com/LumeWeb/p-defer.git",
"p-timeout": "git+https://git.lumeweb.com/LumeWeb/p-timeout.git",
"pino": "^8.11.0",
"pino-pretty": "^9.4.0",
"node-cron": "^3.0.1",
"node-fetch": "2",
"p-timeout": "https://git.lumeweb.com/LumeWeb/p-timeout.git",
"pino": "^8.8.0",
"pino-pretty": "^9.1.1",
"promise-retry": "^2.0.1",
"protomux": "^3.4.1",
"protomux-rpc": "^1.4.1",
"protomux": "^3.4.0",
"protomux-rpc": "^1.3.0",
"random-access-memory": "^4.1.0",
"random-key": "^0.3.2",
"slugify": "^1.6.6",
"sodium-universal": "^4.0.0"
"slugify": "^1.6.5",
"sodium-universal": "^3.1.0"
},
"devDependencies": {
"@semantic-release/changelog": "^6.0.3",
"@semantic-release/exec": "^6.0.3",
"@semantic-release/git": "^10.0.1",
"@lumeweb/relay-types": "https://git.lumeweb.com/LumeWeb/relay-types.git",
"@types/b4a": "^1.6.0",
"@types/minimatch": "^3.0.5",
"@types/node-fetch": "^2.6.3",
"cli-progress": "^3.12.0",
"@types/node-fetch": "^2.6.2",
"cli-progress": "^3.11.2",
"hyper-typings": "^1.0.0",
"node-gyp": "^9.3.1",
"patch-package": "^6.5.1",
"pkg": "^5.8.1",
"node-gyp": "^9.1.0",
"pkg": "^5.8.0",
"prebuildify": "^5.0.1",
"prettier": "^2.8.7",
"semantic-release": "21",
"supports-color": "github:LumeWeb/supports-color",
"typescript": "^4.9.5"
"prettier": "^2.7.1",
"rollup": "^2.77.0",
"supports-color": "https://github.com/LumeWeb/supports-color.git",
"typescript": "^4.9.4"
},
"packageManager": "yarn@4.0.0-rc.33",
"resolutions": {
"micro-ed25519-hdkey@npm:^0.1.2": "patch:micro-ed25519-hdkey@npm%3A0.1.2#~/.yarn/patches/micro-ed25519-hdkey-npm-0.1.2-5d1d767d1d.patch"
}
}

View File

@ -1,12 +0,0 @@
diff --git a/node_modules/@lumeweb/interface-relay/package.json b/node_modules/@lumeweb/interface-relay/package.json
index 0bfaba0..c503d91 100644
--- a/node_modules/@lumeweb/interface-relay/package.json
+++ b/node_modules/@lumeweb/interface-relay/package.json
@@ -2,7 +2,6 @@
"name": "@lumeweb/interface-relay",
"version": "0.0.2-develop.3",
"main": "lib/index.js",
- "type": "module",
"types": "lib/index.d.ts",
"module": "lib/index.mjs",
"exports": {

View File

@ -1,20 +0,0 @@
diff --git a/node_modules/sodium-native/package.json b/node_modules/sodium-native/package.json
index bda9dd4..3a5541a 100644
--- a/node_modules/sodium-native/package.json
+++ b/node_modules/sodium-native/package.json
@@ -3,15 +3,6 @@
"version": "4.0.1",
"description": "Low level bindings for libsodium",
"main": "index.js",
- "files": [
- "index.js",
- "deps/**",
- "modules/**",
- "binding.c",
- "binding.gyp",
- "macros.h",
- "prebuilds/**"
- ],
"dependencies": {
"node-gyp-build": "^4.3.0"
},

View File

@ -1,5 +1,9 @@
{
"assets": [
"node_modules/*/build/Release/*.node",
"node_modules/libskynet",
"node_modules/libskynetnode",
"node_modules/@lumeweb"
],
"outputPath": "dist"
}

View File

@ -1,11 +1,11 @@
// @ts-ignore
import { Config } from "@lumeweb/relay-cfg";
import Config from "@lumeweb/cfg";
import * as os from "os";
import * as fs from "fs";
import path from "path";
import log from "./log.js";
const config = new Config("lumeweb-relay", "core.confDir");
const config = new Config("lumeweb-relay", "core.confdir");
let configDir;
@ -26,7 +26,6 @@ switch (os.platform()) {
config.inject({
"core.confDir": configDir,
"core.port": 8080,
"core.relayPort": 0,
"core.appPort": 80,
"core.logLevel": "info",
"core.pluginDir": path.resolve(configDir, "..", "plugins"),
@ -34,7 +33,7 @@ config.inject({
config.load();
configDir = config.str("core.confDir") as string;
configDir = config.str("core.confdir");
if (fs.existsSync(configDir)) {
try {

View File

@ -1,4 +1,4 @@
import { HDKey } from "ed25519-keygen/hdkey";
import { HDKey } from "micro-ed25519-hdkey";
import config from "../config";
import * as bip39 from "@scure/bip39";
import { wordlist } from "@scure/bip39/wordlists/english";
@ -8,7 +8,7 @@ import b4a from "b4a";
const BIP44_PATH = "m/44'/1627'/0'/0'/0'";
export function getSeed() {
const seed = config.str("core.seed") as string;
const seed = config.str("core.seed");
let valid = bip39.validateMnemonic(seed, wordlist);
if (!valid) {

View File

@ -20,10 +20,7 @@ export async function start() {
await getPluginAPI().emitAsync("core.appServer.buildRoutes");
await app.listen({
port: config.uint("core.appPort") as number,
host: "0.0.0.0",
});
await app.listen({ port: config.uint("core.appport"), host: "0.0.0.0" });
getPluginAPI().emit("core.appServer.started");
}

View File

@ -1,14 +1,14 @@
import config from "../config.js";
import type { RPCServer } from "./rpc/server.js";
import { getRpcServer } from "./rpc/server.js";
import type { Plugin, RPCMethod } from "@lumeweb/interface-relay";
import type { Plugin, RPCMethod } from "@lumeweb/relay-types";
import slugify from "slugify";
import * as fs from "fs";
import path from "path";
import type { Logger } from "pino";
import { getHDKey, getSeed } from "../lib/seed.js";
import type { Config } from "@lumeweb/relay-cfg";
import type Config from "@lumeweb/cfg";
import EventEmitter2 from "eventemitter2";
import log from "../log.js";
import {
@ -17,7 +17,7 @@ import {
ProtocolManager,
} from "./swarm.js";
import { get as getApp } from "./app.js";
import type { HDKey } from "ed25519-keygen/hdkey";
import type { HDKey } from "micro-ed25519-hdkey";
import corePlugins from "../plugins";
import Util from "./plugin/util";
@ -250,9 +250,7 @@ export async function loadPlugins() {
await apiManager.loadPluginInstance(plugin);
}
for (const plugin of [
...new Set(config.array("core.plugins", [] as any)),
] as []) {
for (const plugin of [...new Set(config.array("core.plugins", []))] as []) {
await apiManager.loadPlugin(plugin);
}

View File

@ -16,24 +16,18 @@ import * as http2 from "http2";
import websocket from "@fastify/websocket";
export async function start() {
const swarm = getSwarm();
const dht = getSwarm();
let relayServer = fastify({
http2: true,
logger: log.child({ module: "relay-server" }),
});
await relayServer.register(websocket);
relayServer.register(websocket);
relayServer.get("/", { websocket: true }, (connection) => {
relay(swarm.dht, new Stream(false, connection.socket));
connection.socket.binaryType = "nodebuffer";
relay(dht, new Stream(false, connection.socket));
});
let port = config.uint("core.relayPort");
if (!port) {
port = config.uint("core.port") as number;
}
await relayServer.listen({ port, host: "0.0.0.0" });
await relayServer.listen({ port: config.uint("core.port"), host: "0.0.0.0" });
}

View File

@ -7,24 +7,16 @@ import { errorExit } from "../lib/error.js";
import stringify from "json-stable-stringify";
import {
getRpcServer,
RPC_PROTOCOL_ID,
RPC_PROTOCOL_SYMBOL,
setupStream,
} from "./rpc/server.js";
import { get as getSwarm, SecretStream } from "./swarm.js";
import b4a from "b4a";
// @ts-ignore
import Protomux from "protomux";
export async function start() {
getSwarm().on("connection", (stream: SecretStream) => {
Protomux.from(stream).pair(
{ protocol: "protomux-rpc", id: RPC_PROTOCOL_ID },
async () => {
getRpcServer().setup(stream);
}
);
});
getSwarm().on("connection", (stream: SecretStream) =>
getRpcServer().setup(stream)
);
}
export async function getRpcByPeer(peer: Buffer | string) {
@ -38,16 +30,15 @@ export async function getRpcByPeer(peer: Buffer | string) {
}
return new Promise((resolve) => {
const listener = (peer: any, info: any) => {
const listener = () => {};
swarm.on("connection", (peer: any, info: any) => {
if (info.publicKey.toString("hex") !== peer.toString("hex")) {
return;
}
swarm.removeListener("connection", listener);
resolve(setupStream(peer));
};
swarm.on("connection", listener);
});
swarm.joinPeer(peer);
});

View File

@ -3,7 +3,7 @@ import {
RPCMethod,
RPCRequest,
RPCResponse,
} from "@lumeweb/interface-relay";
} from "@lumeweb/relay-types";
import EventEmitter from "events";
// @ts-ignore
import ProtomuxRPC from "protomux-rpc";
@ -21,7 +21,7 @@ import jsonStringify from "json-stringify-deterministic";
const sodium = require("sodium-universal");
let server: RPCServer;
export const RPC_PROTOCOL_ID = b4a.from("lumeweb");
const RPC_PROTOCOL_ID = b4a.from("lumeweb");
export const RPC_PROTOCOL_SYMBOL = Symbol.for(RPC_PROTOCOL_ID.toString());
export function getRpcServer(): RPCServer {
@ -43,8 +43,6 @@ export function setupStream(stream: SecretStream) {
valueEncoding: c.json,
});
stream.setKeepAlive(5000);
return stream[RPC_PROTOCOL_SYMBOL];
}

View File

@ -69,9 +69,6 @@ export class ProtocolManager {
this._swarm = swarm;
this._swarm.on("connection", (peer: any) => {
if (!peer.userData) {
peer.userData = null;
}
for (const protocol of this._protocols) {
Protomux.from(peer).pair(
{ protocol: protocol[0] },

View File

@ -1,15 +1,8 @@
import { Plugin, PluginAPI } from "@lumeweb/interface-relay";
import defer from "p-defer";
import { Plugin, PluginAPI } from "@lumeweb/relay-types";
const plugin: Plugin = {
name: "core",
async plugin(api: PluginAPI): Promise<void> {
const pluginsLoaded = defer();
api.once("core.pluginsLoaded", () => {
pluginsLoaded.resolve();
});
api.registerMethod("ping", {
cacheable: false,
async handler(): Promise<any> {
@ -20,8 +13,6 @@ const plugin: Plugin = {
api.registerMethod("get_methods", {
cacheable: false,
async handler(): Promise<any> {
await pluginsLoaded.promise;
return api.rpcServer.getMethods();
},
});

View File

@ -1,4 +1,4 @@
import { Plugin, PluginAPI } from "@lumeweb/interface-relay";
import { Plugin, PluginAPI } from "@lumeweb/relay-types";
import b4a from "b4a";
const plugin: Plugin = {

View File

@ -3,11 +3,12 @@ import {
PluginAPI,
RPCBroadcastRequest,
RPCBroadcastResponse,
RPCCacheItem,
RPCRequest,
RPCResponse,
} from "@lumeweb/interface-relay";
} from "@lumeweb/relay-types";
import { getRpcByPeer } from "../modules/rpc";
import { get as getSwarm } from "../modules/swarm";
import { get as getSwarm, LUMEWEB_TOPIC_HASH } from "../modules/swarm";
import b4a from "b4a";
import pTimeout, { ClearablePromise } from "p-timeout";
@ -49,6 +50,91 @@ const plugin: Plugin = {
name: "rpc",
async plugin(_api: PluginAPI): Promise<void> {
api = _api;
if (api.config.bool("cache")) {
api.registerMethod("get_cached_item", {
cacheable: false,
async handler(req: string): Promise<RPCResponse> {
if (typeof req !== "string") {
throw new Error("item must be a string");
}
const cache = api.rpcServer.cache?.data;
if (!cache?.has(req)) {
throw new Error("item does not exist");
}
return {
data: true,
...cache.get<RPCCacheItem>(req)?.value,
signature: cache.get<RPCCacheItem>(req)?.signature,
};
},
});
api.registerMethod("clear_cached_item", {
cacheable: false,
async handler(req: string): Promise<void> {
if (typeof req !== "string") {
throw new Error("item must be a string");
}
try {
api.rpcServer.cache.deleteItem(req);
} catch (e: any) {
throw e;
}
},
});
api.registerMethod("get_peers", {
cacheable: false,
async handler(): Promise<string[]> {
const pubkey = b4a.from(api.identity.publicKeyRaw).toString("hex");
const online = api.rpcServer.cache?.dhtCache.online || new Set();
if (online.has(pubkey)) {
online.delete(pubkey);
}
return [...online];
},
});
if (api.logger.level === "debug") {
api.registerMethod("get_direct_peers", {
cacheable: false,
async handler(): Promise<string[]> {
const online = api.rpcServer.cache.dhtCache.online;
const pubkey = b4a
.from(api.swarm.keyPair.publicKeyRaw())
.toString("hex");
if (online.has(pubkey)) {
online.delete(pubkey);
}
const topic = LUMEWEB_TOPIC_HASH.toString("hex");
return [...api.swarm.peers.values()]
.filter((item: any) =>
[...item._seenTopics.keys()].includes(topic)
)
.map((item: any) => item.publicKey.toString("hex"))
.filter((item: any) => online.has(item));
},
});
api.registerMethod("get_bootstrap_info", {
cacheable: false,
async handler(): Promise<string[]> {
// @ts-ignore
return api.rpcServer.cache.dhtCache._getBootstrapInfo();
},
});
api.registerMethod("get_connected_peers", {
cacheable: false,
async handler(): Promise<string[]> {
// @ts-ignore
return [...api.rpcServer.cache.dhtCache.connectedTo];
},
});
}
}
api.registerMethod("broadcast_request", {
cacheable: false,
async handler(req: RPCBroadcastRequest): Promise<RPCBroadcastResponse> {

View File

@ -2,7 +2,7 @@
"compilerOptions": {
"target": "es2020",
"module": "commonjs",
"moduleResolution": "nodenext",
"moduleResolution": "node",
"rootDir": "src",
"outDir": "build",
"esModuleInterop": true,

4559
yarn.lock Normal file

File diff suppressed because it is too large Load Diff