*Refactoring

*Reformat
*Fix hsd node bootup
This commit is contained in:
Derrick Hammer 2022-07-19 18:24:53 -04:00
parent ff7ebb0984
commit 070b7825e2
15 changed files with 800 additions and 694 deletions

View File

@ -12,20 +12,21 @@
"@hyperswarm/dht": "^6.0.1",
"@hyperswarm/dht-relay": "^0.3.0",
"@pokt-network/pocket-js": "^0.8.0-rc",
"@root/greenlock": "^4.0.5",
"@solana/web3.js": "^1.47.3",
"@types/acme-client": "^3.3.0",
"@types/node": "^18.0.0",
"@types/node-cron": "^3.0.2",
"@types/ws": "^8.5.3",
"@types/xml2js": "^0.4.11",
"acme-client": "^4.2.5",
"algosdk": "^1.18.1",
"async-mutex": "^0.3.2",
"bcfg": "^0.1.7",
"date-fns": "^2.28.0",
"dotenv": "^16.0.1",
"ethers": "^5.6.9",
"express": "^4.18.1",
"greenlock-express": "^4.0.3",
"hsd": "^4.0.1",
"hsd": "https://github.com/LumeWeb/hsd.git#spv-namestate",
"jayson": "^3.6.6",
"json-stable-stringify": "^1.0.1",
"libskynet": "^0.0.48",
@ -44,6 +45,7 @@
"@types/express": "^4.17.13",
"@types/minimatch": "^3.0.5",
"@types/sprintf-js": "^1.1.2",
"esbuild": "^0.14.49",
"hyper-typings": "^1.0.0",
"prettier": "^2.7.1"
}

View File

@ -1,6 +1,6 @@
// @ts-ignore
import BConfig from "bcfg";
import { errorExit } from "./util.js";
import {errorExit} from "./util.js";
const config = new BConfig("lumeweb-relay");
@ -14,7 +14,8 @@ config.load({
});
try {
config.open("config.conf");
} catch (e) {}
} catch (e) {
}
for (const setting of ["relay-domain", "afraid-username", "relay-seed"]) {
if (!config.get(setting)) {

View File

@ -1,7 +1,8 @@
import { createRequire } from "module";
import {createRequire} from "module";
const require = createRequire(import.meta.url);
const DHT = require("@hyperswarm/dht");
import { errorExit } from "./util.js";
import {errorExit} from "./util.js";
import {
deriveMyskyRootKeypair,
ed25519Keypair,
@ -10,9 +11,15 @@ import {
} from "libskynet";
import config from "./config.js";
let server: {
listen: (arg0: ed25519Keypair) => void;
let node: {
ready: () => any;
createServer: () => any;
defaultKeyPair: any;
on: any;
};
let server: {
listen: (arg0: ed25519Keypair) => any;
on: any;
};
async function start() {
@ -25,17 +32,26 @@ async function start() {
const keyPair = deriveMyskyRootKeypair(seedPhraseToSeed(seed)[0]);
const node = new DHT({ keyPair });
node = new DHT({keyPair});
await node.ready();
return (server = node);
server = node.createServer();
await server.listen(keyPair);
return node;
}
export async function get() {
if (!server) {
return start();
export async function get(
ret: "server" | "dht" = "dht"
): Promise<typeof server | typeof node> {
if (!node) {
await start();
}
if (ret == "server") {
return server;
}
return node;
}

View File

@ -1,20 +1,19 @@
import cron from "node-cron";
import fetch from "node-fetch";
import { get as getDHT } from "./dht.js";
import { overwriteRegistryEntry } from "libskynetnode";
import { Buffer } from "buffer";
import { blake2b } from "libskynet";
import { Parser } from "xml2js";
import { URL } from "url";
import { errorExit } from "./util.js";
import { pack } from "msgpackr";
import {get as getDHT} from "./dht.js";
import {overwriteRegistryEntry} from "libskynetnode";
import {Buffer} from "buffer";
import {Parser} from "xml2js";
import {URL} from "url";
import {errorExit, hashDataKey} from "./util.js";
import {pack} from "msgpackr";
import config from "./config.js";
const { createHash } = await import("crypto");
const {createHash} = await import("crypto");
let activeIp: string;
const REGISTRY_DHT_KEY = "lumeweb-dht-relay";
const REGISTRY_NODE_KEY = "lumeweb-dht-node";
async function ipUpdate() {
let currentIp = await getCurrentIp();
@ -31,44 +30,24 @@ async function ipUpdate() {
}
export async function start() {
const dht = await getDHT();
const dht = (await getDHT()) as any;
await ipUpdate();
await overwriteRegistryEntry(
dht.defaultKeyPair,
hashDataKey(REGISTRY_DHT_KEY),
hashDataKey(REGISTRY_NODE_KEY),
pack(`${config.str("relay-domain")}:${config.uint("relay-port")}`)
);
console.log(
"node pubkey:",
Buffer.from(dht.defaultKeyPair.publicKey).toString("hex")
);
cron.schedule("0 * * * *", ipUpdate);
}
function hashDataKey(dataKey: string): Uint8Array {
return blake2b(encodeUtf8String(dataKey));
}
function encodeUtf8String(str: string): Uint8Array {
const byteArray = stringToUint8ArrayUtf8(str);
const encoded = new Uint8Array(8 + byteArray.length);
encoded.set(encodeNumber(byteArray.length));
encoded.set(byteArray, 8);
return encoded;
}
function stringToUint8ArrayUtf8(str: string): Uint8Array {
return Uint8Array.from(Buffer.from(str, "utf-8"));
}
function encodeNumber(num: number): Uint8Array {
const encoded = new Uint8Array(8);
for (let index = 0; index < encoded.length; index++) {
encoded[index] = num & 0xff;
num = num >> 8;
}
return encoded;
}
async function getDomainInfo() {
const relayDomain = config.str("relay-domain");
const parser = new Parser();
@ -113,10 +92,5 @@ async function getDomainInfo() {
}
async function getCurrentIp(): Promise<string> {
const response = await (await fetch("http://checkip.dyndns.org")).text();
const parser = new Parser();
const html = await parser.parseStringPromise(response.trim());
return html.html.body[0].split(":").pop();
return await (await fetch("http://ip1.dynupdate.no-ip.com/")).text();
}

View File

@ -1,6 +1,6 @@
import { start as startDns } from "./dns.js";
import { start as startRpc } from "./rpc.js";
import { start as startRelay } from "./relay.js";
import {start as startDns} from "./dns.js";
import {start as startRpc} from "./rpc.js";
import {start as startRelay} from "./relay.js";
await startDns();
await startRpc();

View File

@ -1,45 +1,56 @@
import WS from "ws";
// @ts-ignore
import DHT from "@hyperswarm/dht";
// @ts-ignore
import { relay } from "@hyperswarm/dht-relay";
import {relay} from "@hyperswarm/dht-relay";
// @ts-ignore
import Stream from "@hyperswarm/dht-relay/ws";
import { get as getDHT } from "./dht.js";
// @ts-ignore
import GLE from "greenlock-express";
// @ts-ignore
import Greenlock from "@root/greenlock";
import express, {Express} from "express";
import path from "path";
import { fileURLToPath } from "url";
import {fileURLToPath} from "url";
import config from "./config.js";
import * as http from "http";
import * as https from "https";
import * as tls from "tls";
import * as acme from "acme-client";
import {Buffer} from "buffer";
import {intervalToDuration} from "date-fns";
import cron from "node-cron";
import {get as getDHT} from "./dht.js";
import WS from "ws";
// @ts-ignore
import DHT from "@hyperswarm/dht";
import {pack} from "msgpackr";
import {overwriteRegistryEntry} from "libskynetnode";
import {hashDataKey} from "./util.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
let sslCtx: tls.SecureContext = tls.createSecureContext();
const sslParams: tls.SecureContextOptions = {};
const sslConfig = {
packageRoot: path.dirname(__dirname),
maintainerEmail: "contact@lumeweb.com",
configDir: path.resolve(config.prefix, "ssl"),
cluster: false,
agreeToTerms: true,
staging: true,
};
const sslPrivateKey = await acme.forge.createPrivateKey();
const acmeClient = new acme.Client({
accountKey: sslPrivateKey,
directoryUrl: acme.directory.letsencrypt.production,
});
let app: Express;
let router = express.Router();
export async function start() {
const relayDomain = config.str("relay-domain");
const relayPort = config.str("relay-port");
const greenlock = Greenlock.create(sslConfig);
await greenlock.add({
subject: relayDomain,
altnames: [relayDomain],
app = express();
app.use(function (req, res, next) {
router(req, res, next);
});
// @ts-ignore
config.greenlock = greenlock;
GLE.init(sslConfig).ready(async (GLEServer: any) => {
let httpsServer = GLEServer.httpsServer();
var httpServer = GLEServer.httpServer();
let httpsServer = https.createServer({
SNICallback(servername, cb) {
cb(null, sslCtx);
},
});
let httpServer = http.createServer(app);
cron.schedule("0 * * * *", createOrRenewSSl);
await new Promise((resolve) => {
httpServer.listen(80, "0.0.0.0", function () {
@ -47,14 +58,14 @@ export async function start() {
resolve(null);
});
});
const dht = await getDHT();
let wsServer = new WS.Server({ server: httpServer });
let wsServer = new WS.Server({server: httpsServer});
wsServer.on("connection", (socket: any) => {
relay(dht, new Stream(false, socket));
});
await new Promise((resolve) => {
httpsServer.listen(relayPort, "0.0.0.0", function () {
console.info("Relay started on ", httpsServer.address());
@ -62,8 +73,46 @@ export async function start() {
});
});
await greenlock.get({
servername: relayDomain,
});
});
await createOrRenewSSl();
}
async function createOrRenewSSl() {
if (sslParams.cert) {
const expires = (
await acme.forge.readCertificateInfo(sslParams.cert as Buffer)
).notAfter;
let duration = intervalToDuration({start: new Date(), end: expires});
let daysLeft = (duration.months as number) * 30 + (duration.days as number);
if (daysLeft > 30) {
return;
}
}
const [certificateKey, certificateRequest] = await acme.forge.createCsr({
commonName: config.str("relay-domain"),
});
sslParams.cert = await acmeClient.auto({
csr: certificateRequest,
termsOfServiceAgreed: true,
challengeCreateFn: async (authz, challenge, keyAuthorization) => {
router.get(
`/.well-known/acme-challenge/${challenge.token}`,
(req, res) => {
res.send(keyAuthorization);
}
);
},
challengeRemoveFn: async () => {
router = express.Router();
},
challengePriority: ["http-01"],
});
sslParams.key = certificateKey;
sslCtx = tls.createSecureContext(sslParams);
console.log("SSL Certificate Updated");
}

View File

@ -1,20 +1,21 @@
import crypto from "crypto";
import jayson from "jayson/promise/index.js";
import { pack, unpack } from "msgpackr";
import { Mutex } from "async-mutex";
import { createRequire } from "module";
import {pack, unpack} from "msgpackr";
import {Mutex} from "async-mutex";
import {createRequire} from "module";
import NodeCache from "node-cache";
import { get as getDHT } from "./dht.js";
import { rpcMethods } from "./rpc/index.js";
import {get as getDHT} from "./dht.js";
import {rpcMethods} from "./rpc/index.js";
import PocketPKG from "@pokt-network/pocket-js";
const { Configuration, HttpRpcProvider, PocketAAT, Pocket } = PocketPKG;
const {Configuration, HttpRpcProvider, PocketAAT, Pocket} = PocketPKG;
import {
JSONRPCRequest,
JSONRPCResponseWithError,
JSONRPCResponseWithResult,
} from "jayson";
import config, { updateUsePocketGateway, usePocketGateway } from "./config.js";
import { errorExit } from "./util.js";
import config, {updateUsePocketGateway, usePocketGateway} from "./config.js";
import {errorExit} from "./util.js";
const require = createRequire(import.meta.url);
@ -101,6 +102,7 @@ async function processRequest(request: RPCRequest): Promise<RPCResponse> {
method: request.query,
jsonrpc: "2.0",
params: request.data,
id: 1
} as unknown as JSONRPCRequest,
request.chain
);
@ -128,7 +130,7 @@ async function processRequest(request: RPCRequest): Promise<RPCResponse> {
}
dbData.data = error
? { error }
? {error}
: (rpcResp as unknown as JSONRPCResponseWithResult).result;
if (!processedRequests.get(reqId) || request.force) {
@ -147,6 +149,7 @@ export function updateAat(aat: PocketAATObject): void {
export function getAat(): PocketAATObject {
return _aat;
}
export function getPocketServer(): typeof Pocket {
return pocketServer;
}
@ -195,7 +198,7 @@ export async function processRpcRequest(
return new Promise((resolve) => {
jsonServer.call(
request,
{ chain },
{chain},
(
err?: JSONRPCResponseWithError | null,
result?: JSONRPCResponseWithResult
@ -239,9 +242,10 @@ export async function start() {
);
}
jsonServer = new jayson.Server(rpcMethods, { useContext: true });
jsonServer = new jayson.Server(rpcMethods, {useContext: true});
(await getDHT()).on("connection", (socket: any) => {
(await getDHT("server")).on("connection", (socket: any) => {
socket.rawStream._ondestroy = () => false;
socket.on("data", async (data: any) => {
let request: RPCRequest;
try {
@ -254,7 +258,7 @@ export async function start() {
socket.write(pack(await maybeProcessRequest(request)));
} catch (error) {
console.trace(error);
socket.write(pack({ error }));
socket.write(pack({error}));
}
socket.end();
});

View File

@ -1,9 +1,9 @@
import { maybeMapChainId, reverseMapChainId } from "../util.js";
import {maybeMapChainId, reverseMapChainId} from "../util.js";
import minimatch from "minimatch";
// @ts-ignore
import HTTPClient from "algosdk/dist/cjs/src/client/client.js";
import { sprintf } from "sprintf-js";
import { RpcMethodList } from "./index.js";
import {sprintf} from "sprintf-js";
import {RpcMethodList} from "./index.js";
import config from "../config.js";
const allowedEndpoints: { [endpoint: string]: ("GET" | "POST")[] } = {
@ -35,7 +35,7 @@ export function proxyRestMethod(
let query = args.query ?? false;
let fullHeaders = args.fullHeaders ?? {};
fullHeaders = { ...fullHeaders, Referer: "lumeweb_dns_relay" };
fullHeaders = {...fullHeaders, Referer: "lumeweb_dns_relay"};
if (method) {
method = method.toUpperCase();
@ -76,7 +76,7 @@ export function proxyRestMethod(
data = new Uint8Array(Buffer.from(data.data));
}
resp = await client.post(endpoint, data, { ...fullHeaders });
resp = await client.post(endpoint, data, {...fullHeaders});
break;
default:
throw new Error("Method Invalid");

View File

@ -1,9 +1,9 @@
import { ethers } from "ethers";
import { Pocket, PocketAAT } from "@pokt-network/pocket-js";
import { maybeMapChainId, reverseMapChainId } from "../util.js";
import { Connection } from "@solana/web3.js";
import { getAat, getPocketServer } from "../rpc.js";
import config, { usePocketGateway } from "../config.js";
import {ethers} from "ethers";
import {Pocket, PocketAAT} from "@pokt-network/pocket-js";
import {maybeMapChainId, reverseMapChainId} from "../util.js";
import {Connection} from "@solana/web3.js";
import {getAat, getPocketServer} from "../rpc.js";
import config, {usePocketGateway} from "../config.js";
type RpcProviderMethod = (method: string, params: Array<any>) => Promise<any>;

View File

@ -1,10 +1,10 @@
import { isIp } from "../util.js";
import { RpcMethodList } from "./index.js";
import { createRequire } from "module";
import {isIp} from "../util.js";
import {RpcMethodList} from "./index.js";
import {createRequire} from "module";
const require = createRequire(import.meta.url);
const bns = require("bns");
const { StubResolver, RecursiveResolver } = bns;
const {StubResolver, RecursiveResolver} = bns;
const resolverOpt = {
tcp: true,

View File

@ -1,5 +1,5 @@
import { proxyRpcMethod } from "./common.js";
import { RpcMethodList } from "./index.js";
import {proxyRpcMethod} from "./common.js";
import {RpcMethodList} from "./index.js";
const rpcMethods: RpcMethodList = {};

View File

@ -1,13 +1,13 @@
import { RpcMethodList } from "./index.js";
import {RpcMethodList} from "./index.js";
// @ts-ignore
import rand from "random-key";
// @ts-ignore
import SPVNode from "hsd/lib/node/spvnode.js";
import config from "../config.js";
import { createRequire } from "module";
import {createRequire} from "module";
const require = createRequire(import.meta.url);
const { NodeClient } = require("hs-client");
const {NodeClient} = require("hs-client");
let hsdServer: SPVNode;
@ -18,20 +18,53 @@ let clientArgs = {
apiKey: rand.generate(),
};
if (!config.bool("hsd-use-extenal-node")) {
if (!config.bool("hsd-use-external-node")) {
hsdServer = new SPVNode({
config: false,
argv: false,
env: true,
noDns: true,
memory: false,
httpHost: "127.0.0.1",
apiKey: clientArgs.apiKey,
logFile: false,
logConsole: false,
logConsole: true,
logLevel: "info",
workers: true,
network: "main",
});
console.log(`HSD API KEY: ${clientArgs.apiKey}`);
hsdServer.on("abort", async (err: any) => {
const timeout = setTimeout(() => {
console.error("Shutdown is taking a long time. Exiting.");
process.exit(3);
}, 5000);
timeout.unref();
try {
console.error("Shutting down...");
await hsdServer.close();
clearTimeout(timeout);
console.error((err as Error).stack);
process.exit(2);
} catch (e: any) {
console.error(`Error occurred during shutdown: ${(e as Error).message}`);
process.exit(3);
}
});
try {
await hsdServer.ensure();
await hsdServer.open();
await hsdServer.connect();
hsdServer.startSync();
} catch (e: any) {
console.error((e as Error).stack);
}
} else {
clientArgs = {
network: config.str("hsd-network-type"),

View File

@ -2,11 +2,11 @@ export type RpcMethodList = { [name: string]: Function };
export * from "./common.js";
import { default as DnsMethods } from "./dns.js";
import { default as EvmMethods } from "./evm.js";
import { default as HnsMethods } from "./handshake.js";
import { default as SolMethods } from "./solana.js";
import { default as AlgoMethods } from "./algorand.js";
import {default as DnsMethods} from "./dns.js";
import {default as EvmMethods} from "./evm.js";
import {default as HnsMethods} from "./handshake.js";
import {default as SolMethods} from "./solana.js";
import {default as AlgoMethods} from "./algorand.js";
export const rpcMethods: RpcMethodList = Object.assign(
{},

View File

@ -1,6 +1,6 @@
import { proxyRpcMethod } from "./common.js";
import { RpcMethodList } from "./index.js";
import * as chainNetworks from "../networks.json" assert { type: "json" };
import {proxyRpcMethod} from "./common.js";
import {RpcMethodList} from "./index.js";
import * as chainNetworks from "../networks.json" assert {type: "json"};
export default {
getAccountInfo: proxyRpcMethod("getAccountInfo", [

View File

@ -1,4 +1,6 @@
import * as chainNetworks from "./networks.json" assert { type: "json" };
import * as chainNetworks from "./networks.json" assert {type: "json"};
import {Buffer} from "buffer";
import {blake2b} from "libskynet";
type networks = { [net: string]: string };
@ -37,3 +39,28 @@ export function isIp(ip: string) {
ip
);
}
export function hashDataKey(dataKey: string): Uint8Array {
return blake2b(encodeUtf8String(dataKey));
}
function encodeUtf8String(str: string): Uint8Array {
const byteArray = stringToUint8ArrayUtf8(str);
const encoded = new Uint8Array(8 + byteArray.length);
encoded.set(encodeNumber(byteArray.length));
encoded.set(byteArray, 8);
return encoded;
}
function stringToUint8ArrayUtf8(str: string): Uint8Array {
return Uint8Array.from(Buffer.from(str, "utf-8"));
}
function encodeNumber(num: number): Uint8Array {
const encoded = new Uint8Array(8);
for (let index = 0; index < encoded.length; index++) {
encoded[index] = num & 0xff;
num = num >> 8;
}
return encoded;
}