From 88dc104102fcdf2140e02b4e41552ffdb44f9e2a Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Fri, 22 Jul 2022 19:53:11 -0400 Subject: [PATCH] *Refactor SSL support to store to skynet as independent files *Introduce ssl-mode config option to use staging letsencrypt --- package.json | 1 + src/file.ts | 479 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/relay.ts | 181 +++++++++++++++---- 3 files changed, 626 insertions(+), 35 deletions(-) create mode 100644 src/file.ts diff --git a/package.json b/package.json index 49e893a..5cb821a 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "@hyperswarm/dht-relay": "^0.3.0", "@lumeweb/kernel-utils": "https://github.com/LumeWeb/kernel-utils.git", "@pokt-network/pocket-js": "^0.8.0-rc", + "@skynetlabs/skynet-nodejs": "^2.6.0", "@solana/web3.js": "^1.47.3", "@types/acme-client": "^3.3.0", "@types/node": "^18.0.0", diff --git a/src/file.ts b/src/file.ts new file mode 100644 index 0000000..a1b1efc --- /dev/null +++ b/src/file.ts @@ -0,0 +1,479 @@ +import { + addContextToErr, + blake2b, + bufToHex, + Ed25519Keypair, + ed25519Sign, + encodePrefixedBytes, + encodeU64, + Err, +} from "libskynet"; +import { progressiveFetch } from "libskynetnode/dist/progressivefetch.js"; +import { defaultPortalList } from "libskynet/dist/defaultportals.js"; +import { readRegistryEntry } from "libskynetnode/dist/registryread.js"; +import { + bufToB64, + decryptFileSmall, + deriveChildSeed, + deriveRegistryEntryID, + encryptFileSmall, + entryIDToSkylink, + namespaceInode, + skylinkToResolverEntryData, + taggedRegistryEntryKeys, +} from "libskynet"; +import { upload } from "libskynetnode"; +// @ts-ignore +import { SkynetClient } from "@skynetlabs/skynet-nodejs"; + +const ERR_EXISTS = "exists"; +const ERR_NOT_EXISTS = "DNE"; +const STD_FILENAME = "file"; + +type OverwriteDataFn = (newData: Uint8Array) => Promise; + +type ReadDataFn = () => Promise<[Uint8Array, Err]>; + +export interface IndependentFileSmallMetadata { + largestHistoricSize: bigint; +} + +export interface IndependentFileSmall { + dataKey: Uint8Array; + fileData: Uint8Array; + inode: string; + keypair: Ed25519Keypair; + metadata: IndependentFileSmallMetadata; + revision: bigint; + seed: Uint8Array; + + skylink: string; + viewKey: string; + + overwriteData: OverwriteDataFn; + + readData: ReadDataFn; +} + +interface IndependentFileSmallViewer { + fileData: Uint8Array; + skylink: string; + viewKey: string; + + readData: ReadDataFn; +} + +function overwriteRegistryEntry( + keypair: any, + datakey: Uint8Array, + data: Uint8Array, + revision: bigint +): Promise { + return new Promise((resolve, reject) => { + if (data.length > 86) { + reject("provided data is too large to fit in a registry entry"); + return; + } + + let [encodedRevision, errU64] = encodeU64(revision); + if (errU64 !== null) { + reject(addContextToErr(errU64, "unable to encode revision number")); + return; + } + + let datakeyHex = bufToHex(datakey); + let [encodedData, errEPB] = encodePrefixedBytes(data); + if (errEPB !== null) { + reject(addContextToErr(errEPB, "unable to encode the registry data")); + return; + } + let dataToSign = new Uint8Array(32 + 8 + data.length + 8); + dataToSign.set(datakey, 0); + dataToSign.set(encodedData, 32); + dataToSign.set(encodedRevision, 32 + 8 + data.length); + let sigHash = blake2b(dataToSign); + let [sig, errS] = ed25519Sign(sigHash, keypair.secretKey); + if (errS !== null) { + reject(addContextToErr(errS, "unable to produce signature")); + return; + } + + let postBody = { + publickey: { + algorithm: "ed25519", + key: Array.from(keypair.publicKey), + }, + datakey: datakeyHex, + revision: Number(revision), + data: Array.from(data), + signature: Array.from(sig), + }; + let fetchOpts = { + method: "post", + body: JSON.stringify(postBody), + }; + let endpoint = "/skynet/registry"; + + progressiveFetch( + endpoint, + fetchOpts, + defaultPortalList, + verifyRegistryWrite + ).then((result) => { + if (result.success === true) { + resolve(null); + return; + } + reject("unable to write registry entry\n" + JSON.stringify(result)); + }); + }); +} +function verifyRegistryWrite(response: Response): Promise { + return new Promise((resolve) => { + if (!("status" in response)) { + resolve("response did not contain a status"); + return; + } + if (response.status === 204) { + resolve(null); + return; + } + resolve("unrecognized status"); + }); +} + +function createIndependentFileSmall( + seed: Uint8Array, + userInode: string, + fileData: Uint8Array +): Promise<[IndependentFileSmall, Err]> { + return new Promise(async (resolve) => { + let [inode, errNI] = namespaceInode("IndependentFileSmall", userInode); + if (errNI !== null) { + resolve([{} as any, addContextToErr(errNI, "unable to namespace inode")]); + return; + } + + let [keypair, dataKey, errTREK] = taggedRegistryEntryKeys( + seed, + inode, + inode + ); + if (errTREK !== null) { + resolve([ + {} as any, + addContextToErr( + errTREK, + "unable to get registry entry for provided inode" + ), + ]); + return; + } + + let result; + try { + result = await readRegistryEntry(keypair.publicKey, dataKey); + } catch (e) { + result = { exists: false }; + } + if (result.exists === true) { + resolve([{} as any, "exists"]); + return; + } + + let encryptionKey = deriveChildSeed(seed, inode); + let metadata: IndependentFileSmallMetadata = { + largestHistoricSize: BigInt(fileData.length), + }; + + let revisionSeed = new Uint8Array(seed.length + 8); + revisionSeed.set(seed, 0); + let revisionKey = deriveChildSeed(revisionSeed, inode); + let revision = BigInt(revisionKey[0]) * 256n + BigInt(revisionKey[1]); + let [encryptedData, errEF] = encryptFileSmall( + encryptionKey, + inode, + revision, + metadata, + fileData, + metadata.largestHistoricSize + ); + if (errEF !== null) { + resolve([{} as any, addContextToErr(errEF, "unable to encrypt file")]); + return; + } + + let immutableSkylink; + + try { + immutableSkylink = await upload(encryptedData, { + Filename: STD_FILENAME, + }); + } catch (e) { + resolve([{} as any, addContextToErr(e, "upload failed")]); + return; + } + + let [entryData, errSTRED] = skylinkToResolverEntryData(immutableSkylink); + if (errSTRED !== null) { + resolve([ + {} as any, + addContextToErr( + errSTRED, + "couldn't create resovler link from upload skylink" + ), + ]); + return; + } + + try { + await overwriteRegistryEntry(keypair, dataKey, entryData, revision); + } catch (e: any) { + resolve([ + {} as any, + addContextToErr(e, "could not write to registry entry"), + ]); + return; + } + + let [entryID, errDREID] = deriveRegistryEntryID(keypair.publicKey, dataKey); + if (errDREID !== null) { + resolve([ + {} as any, + addContextToErr(errDREID, "could not compute entry id"), + ]); + return; + } + let skylink = entryIDToSkylink(entryID); + + let encStr = bufToB64(encryptionKey); + let viewKey = encStr + inode; + + let ifile: IndependentFileSmall = { + dataKey, + fileData, + inode, + keypair, + metadata, + revision, + seed, + + skylink, + viewKey, + + overwriteData: function (newData: Uint8Array): Promise { + return overwriteIndependentFileSmall(ifile, newData); + }, + readData: function (): Promise<[Uint8Array, Err]> { + return new Promise((resolve) => { + let data = new Uint8Array(ifile.fileData.length); + data.set(ifile.fileData, 0); + resolve([data, null]); + }); + }, + }; + resolve([ifile, null]); + }); +} + +function openIndependentFileSmall( + seed: Uint8Array, + userInode: string +): Promise<[IndependentFileSmall, Err]> { + return new Promise(async (resolve) => { + let [inode, errNI] = namespaceInode("IndependentFileSmall", userInode); + if (errNI !== null) { + resolve([{} as any, addContextToErr(errNI, "unable to namespace inode")]); + return; + } + + let [keypair, dataKey, errTREK] = taggedRegistryEntryKeys( + seed, + inode, + inode + ); + if (errTREK !== null) { + resolve([ + {} as any, + addContextToErr( + errTREK, + "unable to get registry entry for provided inode" + ), + ]); + return; + } + + let result; + try { + result = await readRegistryEntry(keypair.publicKey, dataKey); + } catch (e: any) { + resolve([ + {} as any, + addContextToErr(e, "unable to read registry entry for file"), + ]); + return; + } + if (result.exists !== true) { + resolve([{} as any, ERR_NOT_EXISTS]); + return; + } + + let [entryID, errDREID] = deriveRegistryEntryID(keypair.publicKey, dataKey); + if (errDREID !== null) { + resolve([ + {} as any, + addContextToErr(errDREID, "unable to derive registry entry id"), + ]); + return; + } + let skylink = entryIDToSkylink(entryID); + + const client = new SkynetClient("https://web3portal.com"); + let encryptedData; + try { + encryptedData = await client.downloadData(skylink); + } catch (e: any) { + resolve([{} as any, addContextToErr(e, "unable to download file")]); + return; + } + + let encryptionKey = deriveChildSeed(seed, inode); + let [metadata, fileData, errDF] = decryptFileSmall( + encryptionKey, + inode, + encryptedData + ); + if (errDF !== null) { + resolve([{} as any, addContextToErr(errDF, "unable to decrypt file")]); + return; + } + + let encStr = bufToB64(encryptionKey); + let viewKey = encStr + inode; + + let ifile: IndependentFileSmall = { + dataKey, + fileData, + inode, + keypair, + metadata, + revision: result.revision!, + seed, + + skylink, + viewKey, + + overwriteData: function (newData: Uint8Array): Promise { + return overwriteIndependentFileSmall(ifile, newData); + }, + + readData: function (): Promise<[Uint8Array, Err]> { + return new Promise((resolve) => { + let data = new Uint8Array(ifile.fileData.length); + data.set(ifile.fileData, 0); + resolve([data, null]); + }); + }, + }; + resolve([ifile, null]); + }); +} +function overwriteIndependentFileSmall( + file: IndependentFileSmall, + newData: Uint8Array +): Promise { + return new Promise(async (resolve) => { + // Create a new metadata for the file based on the current file + // metadata. Need to update the largest historic size. + let newMetadata: IndependentFileSmallMetadata = { + largestHistoricSize: BigInt(file.metadata.largestHistoricSize), + }; + if (BigInt(newData.length) > newMetadata.largestHistoricSize) { + newMetadata.largestHistoricSize = BigInt(newData.length); + } + + // Compute the new revision number for the file. This is done + // deterministically using the seed and the current revision number, so + // that multiple concurrent updates will end up with the same revision. + // We use a random number between 1 and 256 for our increment. + let [encodedRevision, errEU64] = encodeU64(file.revision); + if (errEU64 !== null) { + resolve(addContextToErr(errEU64, "unable to encode revision")); + return; + } + let revisionSeed = new Uint8Array( + file.seed.length + encodedRevision.length + ); + revisionSeed.set(file.seed, 0); + revisionSeed.set(encodedRevision, file.seed.length); + let revisionKey = deriveChildSeed(revisionSeed, file.inode); + let newRevision = file.revision + BigInt(revisionKey[0]) + 1n; + + // Get the encryption key. + let encryptionKey = deriveChildSeed(file.seed, file.inode); + + // Create a new encrypted blob for the data. + // + // NOTE: Need to supply the data that would be in place after a + // successful update, which means using the new metadata and revision + // number. + let [encryptedData, errEFS] = encryptFileSmall( + encryptionKey, + file.inode, + newRevision, + newMetadata, + newData, + newMetadata.largestHistoricSize + ); + if (errEFS !== null) { + resolve(addContextToErr(errEFS, "unable to encrypt updated file")); + return; + } + + // Upload the data to get the immutable link. + let skylink; + try { + skylink = await upload(encryptedData, { + Filename: STD_FILENAME, + }); + } catch (e) { + resolve(addContextToErr(e, "new data upload failed")); + return; + } + + // Write to the registry entry. + let [entryData, errSTRED] = skylinkToResolverEntryData(skylink); + if (errSTRED !== null) { + resolve( + addContextToErr( + errSTRED, + "could not create resolver link from upload skylink" + ) + ); + return; + } + try { + await overwriteRegistryEntry( + file.keypair, + file.dataKey, + entryData, + newRevision + ); + } catch (e: any) { + resolve(addContextToErr(e, "could not write to registry entry")); + return; + } + + // File update was successful, update the file metadata. + file.revision = newRevision; + file.metadata = newMetadata; + file.fileData = newData; + resolve(null); + }); +} + +export { + createIndependentFileSmall, + openIndependentFileSmall, + overwriteIndependentFileSmall, +}; diff --git a/src/relay.ts b/src/relay.ts index 4d1b309..890709e 100644 --- a/src/relay.ts +++ b/src/relay.ts @@ -5,8 +5,6 @@ import { relay } from "@hyperswarm/dht-relay"; // @ts-ignore import Stream from "@hyperswarm/dht-relay/ws"; import express, { Express } from "express"; -import path from "path"; -import { fileURLToPath } from "url"; import config from "./config.js"; import * as http from "http"; import * as https from "https"; @@ -18,23 +16,34 @@ import cron from "node-cron"; import { get as getDHT } from "./dht.js"; import WS from "ws"; // @ts-ignore -import DHT from "@hyperswarm/dht"; -import { pack } from "msgpackr"; -import { overwriteRegistryEntry } from "libskynetnode"; -import { hashDataKey } from "./util.js"; +import { + createIndependentFileSmall, + IndependentFileSmall, + openIndependentFileSmall, + overwriteIndependentFileSmall, +} from "./file.js"; +import { seedPhraseToSeed } from "libskynet"; let sslCtx: tls.SecureContext = tls.createSecureContext(); -const sslParams: tls.SecureContextOptions = {}; +const sslParams: tls.SecureContextOptions = { cert: "", key: "" }; const sslPrivateKey = await acme.forge.createPrivateKey(); const acmeClient = new acme.Client({ accountKey: sslPrivateKey, - directoryUrl: acme.directory.letsencrypt.production, + directoryUrl: + config.str("ssl-mode") === "staging" + ? acme.directory.letsencrypt.staging + : acme.directory.letsencrypt.production, }); let app: Express; let router = express.Router(); +const FILE_CERT_NAME = "/lumeweb/relay/ssl.crt"; +const FILE_KEY_NAME = "/lumeweb/relay/ssl.key"; + +type SslData = { crt: IndependentFileSmall; key: IndependentFileSmall }; + export async function start() { const relayPort = config.str("relay-port"); app = express(); @@ -50,7 +59,7 @@ export async function start() { let httpServer = http.createServer(app); - cron.schedule("0 * * * *", createOrRenewSSl); + cron.schedule("0 * * * *", setupSSl); await new Promise((resolve) => { httpServer.listen(80, "0.0.0.0", function () { @@ -73,46 +82,148 @@ export async function start() { }); }); - await createOrRenewSSl(); + await setupSSl(); } -async function createOrRenewSSl() { - if (sslParams.cert) { - const expires = ( - await acme.forge.readCertificateInfo(sslParams.cert as Buffer) - ).notAfter; +async function setupSSl() { + let sslCert = await getSslCert(); + let sslKey = await getSslKey(); + let certInfo; + let exists = false; + let domainValid = false; + let dateValid = false; + if (sslCert && sslKey) { + sslParams.cert = Buffer.from((sslCert as IndependentFileSmall).fileData); + sslParams.key = Buffer.from((sslKey as IndependentFileSmall).fileData); + certInfo = await getCertInfo(); + exists = true; + } + + if (exists) { + const expires = certInfo?.notAfter as Date; let duration = intervalToDuration({ start: new Date(), end: expires }); - let daysLeft = (duration.months as number) * 30 + (duration.days as number); if (daysLeft > 30) { - return; + dateValid = true; + } + + if (certInfo?.domains.commonName === config.str("relay-domain")) { + domainValid = true; } } + if (dateValid && domainValid) { + return; + } + + await createOrRenewSSl( + sslCert as IndependentFileSmall, + sslKey as IndependentFileSmall + ); +} + +async function createOrRenewSSl( + oldCert?: IndependentFileSmall, + oldKey?: IndependentFileSmall +) { const [certificateKey, certificateRequest] = await acme.forge.createCsr({ commonName: config.str("relay-domain"), }); - - sslParams.cert = await acmeClient.auto({ - csr: certificateRequest, - termsOfServiceAgreed: true, - challengeCreateFn: async (authz, challenge, keyAuthorization) => { - router.get( - `/.well-known/acme-challenge/${challenge.token}`, - (req, res) => { - res.send(keyAuthorization); - } - ); - }, - challengeRemoveFn: async () => { - router = express.Router(); - }, - challengePriority: ["http-01"], - }); + try { + sslParams.cert = await acmeClient.auto({ + csr: certificateRequest, + termsOfServiceAgreed: true, + challengeCreateFn: async (authz, challenge, keyAuthorization) => { + router.get( + `/.well-known/acme-challenge/${challenge.token}`, + (req, res) => { + res.send(keyAuthorization); + } + ); + }, + challengeRemoveFn: async () => { + router = express.Router(); + }, + challengePriority: ["http-01"], + }); + sslParams.cert = Buffer.from(sslParams.cert); + } catch (e: any) { + console.error((e as Error).message); + process.exit(1); + } sslParams.key = certificateKey; sslCtx = tls.createSecureContext(sslParams); - console.log("SSL Certificate Updated"); + await saveSsl(oldCert, oldKey); +} + +async function saveSsl( + oldCert?: IndependentFileSmall, + oldKey?: IndependentFileSmall +): Promise { + const seed = getSeed(); + + if (oldCert) { + await overwriteIndependentFileSmall( + oldCert, + Buffer.from(sslParams.cert as any) + ); + } else { + await createIndependentFileSmall( + seed, + FILE_CERT_NAME, + Buffer.from(sslParams.cert as any) + ); + } + + if (oldKey) { + await overwriteIndependentFileSmall( + oldKey, + Buffer.from(sslParams.key as any) + ); + } else { + await createIndependentFileSmall( + seed, + FILE_KEY_NAME, + Buffer.from(sslParams.key as any) + ); + } +} + +async function getCertInfo() { + return acme.forge.readCertificateInfo(sslParams.cert as Buffer); +} + +async function getSslCert(): Promise { + return getSslFile(FILE_CERT_NAME); +} +async function getSslKey(): Promise { + return getSslFile(FILE_KEY_NAME); +} + +async function getSslFile( + name: string +): Promise { + let seed = getSeed(); + + let [file, err] = await openIndependentFileSmall(seed, name); + + if (err) { + return false; + } + + return file; +} + +function getSeed(): Uint8Array { + let [seed, err] = seedPhraseToSeed(config.str("relay-seed")); + + if (err) { + console.error(err); + process.exit(1); + } + + return seed; }