*Refactor SSL support to store to skynet as independent files

*Introduce ssl-mode config option to use staging letsencrypt
This commit is contained in:
Derrick Hammer 2022-07-22 19:53:11 -04:00
parent 330866c1e8
commit 88dc104102
3 changed files with 626 additions and 35 deletions

View File

@ -13,6 +13,7 @@
"@hyperswarm/dht-relay": "^0.3.0",
"@lumeweb/kernel-utils": "https://github.com/LumeWeb/kernel-utils.git",
"@pokt-network/pocket-js": "^0.8.0-rc",
"@skynetlabs/skynet-nodejs": "^2.6.0",
"@solana/web3.js": "^1.47.3",
"@types/acme-client": "^3.3.0",
"@types/node": "^18.0.0",

479
src/file.ts Normal file
View File

@ -0,0 +1,479 @@
import {
addContextToErr,
blake2b,
bufToHex,
Ed25519Keypair,
ed25519Sign,
encodePrefixedBytes,
encodeU64,
Err,
} from "libskynet";
import { progressiveFetch } from "libskynetnode/dist/progressivefetch.js";
import { defaultPortalList } from "libskynet/dist/defaultportals.js";
import { readRegistryEntry } from "libskynetnode/dist/registryread.js";
import {
bufToB64,
decryptFileSmall,
deriveChildSeed,
deriveRegistryEntryID,
encryptFileSmall,
entryIDToSkylink,
namespaceInode,
skylinkToResolverEntryData,
taggedRegistryEntryKeys,
} from "libskynet";
import { upload } from "libskynetnode";
// @ts-ignore
import { SkynetClient } from "@skynetlabs/skynet-nodejs";
const ERR_EXISTS = "exists";
const ERR_NOT_EXISTS = "DNE";
const STD_FILENAME = "file";
type OverwriteDataFn = (newData: Uint8Array) => Promise<Err>;
type ReadDataFn = () => Promise<[Uint8Array, Err]>;
export interface IndependentFileSmallMetadata {
largestHistoricSize: bigint;
}
export interface IndependentFileSmall {
dataKey: Uint8Array;
fileData: Uint8Array;
inode: string;
keypair: Ed25519Keypair;
metadata: IndependentFileSmallMetadata;
revision: bigint;
seed: Uint8Array;
skylink: string;
viewKey: string;
overwriteData: OverwriteDataFn;
readData: ReadDataFn;
}
interface IndependentFileSmallViewer {
fileData: Uint8Array;
skylink: string;
viewKey: string;
readData: ReadDataFn;
}
function overwriteRegistryEntry(
keypair: any,
datakey: Uint8Array,
data: Uint8Array,
revision: bigint
): Promise<null> {
return new Promise((resolve, reject) => {
if (data.length > 86) {
reject("provided data is too large to fit in a registry entry");
return;
}
let [encodedRevision, errU64] = encodeU64(revision);
if (errU64 !== null) {
reject(addContextToErr(errU64, "unable to encode revision number"));
return;
}
let datakeyHex = bufToHex(datakey);
let [encodedData, errEPB] = encodePrefixedBytes(data);
if (errEPB !== null) {
reject(addContextToErr(errEPB, "unable to encode the registry data"));
return;
}
let dataToSign = new Uint8Array(32 + 8 + data.length + 8);
dataToSign.set(datakey, 0);
dataToSign.set(encodedData, 32);
dataToSign.set(encodedRevision, 32 + 8 + data.length);
let sigHash = blake2b(dataToSign);
let [sig, errS] = ed25519Sign(sigHash, keypair.secretKey);
if (errS !== null) {
reject(addContextToErr(errS, "unable to produce signature"));
return;
}
let postBody = {
publickey: {
algorithm: "ed25519",
key: Array.from(keypair.publicKey),
},
datakey: datakeyHex,
revision: Number(revision),
data: Array.from(data),
signature: Array.from(sig),
};
let fetchOpts = {
method: "post",
body: JSON.stringify(postBody),
};
let endpoint = "/skynet/registry";
progressiveFetch(
endpoint,
fetchOpts,
defaultPortalList,
verifyRegistryWrite
).then((result) => {
if (result.success === true) {
resolve(null);
return;
}
reject("unable to write registry entry\n" + JSON.stringify(result));
});
});
}
function verifyRegistryWrite(response: Response): Promise<Err> {
return new Promise((resolve) => {
if (!("status" in response)) {
resolve("response did not contain a status");
return;
}
if (response.status === 204) {
resolve(null);
return;
}
resolve("unrecognized status");
});
}
function createIndependentFileSmall(
seed: Uint8Array,
userInode: string,
fileData: Uint8Array
): Promise<[IndependentFileSmall, Err]> {
return new Promise(async (resolve) => {
let [inode, errNI] = namespaceInode("IndependentFileSmall", userInode);
if (errNI !== null) {
resolve([{} as any, addContextToErr(errNI, "unable to namespace inode")]);
return;
}
let [keypair, dataKey, errTREK] = taggedRegistryEntryKeys(
seed,
inode,
inode
);
if (errTREK !== null) {
resolve([
{} as any,
addContextToErr(
errTREK,
"unable to get registry entry for provided inode"
),
]);
return;
}
let result;
try {
result = await readRegistryEntry(keypair.publicKey, dataKey);
} catch (e) {
result = { exists: false };
}
if (result.exists === true) {
resolve([{} as any, "exists"]);
return;
}
let encryptionKey = deriveChildSeed(seed, inode);
let metadata: IndependentFileSmallMetadata = {
largestHistoricSize: BigInt(fileData.length),
};
let revisionSeed = new Uint8Array(seed.length + 8);
revisionSeed.set(seed, 0);
let revisionKey = deriveChildSeed(revisionSeed, inode);
let revision = BigInt(revisionKey[0]) * 256n + BigInt(revisionKey[1]);
let [encryptedData, errEF] = encryptFileSmall(
encryptionKey,
inode,
revision,
metadata,
fileData,
metadata.largestHistoricSize
);
if (errEF !== null) {
resolve([{} as any, addContextToErr(errEF, "unable to encrypt file")]);
return;
}
let immutableSkylink;
try {
immutableSkylink = await upload(encryptedData, {
Filename: STD_FILENAME,
});
} catch (e) {
resolve([{} as any, addContextToErr(e, "upload failed")]);
return;
}
let [entryData, errSTRED] = skylinkToResolverEntryData(immutableSkylink);
if (errSTRED !== null) {
resolve([
{} as any,
addContextToErr(
errSTRED,
"couldn't create resovler link from upload skylink"
),
]);
return;
}
try {
await overwriteRegistryEntry(keypair, dataKey, entryData, revision);
} catch (e: any) {
resolve([
{} as any,
addContextToErr(e, "could not write to registry entry"),
]);
return;
}
let [entryID, errDREID] = deriveRegistryEntryID(keypair.publicKey, dataKey);
if (errDREID !== null) {
resolve([
{} as any,
addContextToErr(errDREID, "could not compute entry id"),
]);
return;
}
let skylink = entryIDToSkylink(entryID);
let encStr = bufToB64(encryptionKey);
let viewKey = encStr + inode;
let ifile: IndependentFileSmall = {
dataKey,
fileData,
inode,
keypair,
metadata,
revision,
seed,
skylink,
viewKey,
overwriteData: function (newData: Uint8Array): Promise<Err> {
return overwriteIndependentFileSmall(ifile, newData);
},
readData: function (): Promise<[Uint8Array, Err]> {
return new Promise((resolve) => {
let data = new Uint8Array(ifile.fileData.length);
data.set(ifile.fileData, 0);
resolve([data, null]);
});
},
};
resolve([ifile, null]);
});
}
function openIndependentFileSmall(
seed: Uint8Array,
userInode: string
): Promise<[IndependentFileSmall, Err]> {
return new Promise(async (resolve) => {
let [inode, errNI] = namespaceInode("IndependentFileSmall", userInode);
if (errNI !== null) {
resolve([{} as any, addContextToErr(errNI, "unable to namespace inode")]);
return;
}
let [keypair, dataKey, errTREK] = taggedRegistryEntryKeys(
seed,
inode,
inode
);
if (errTREK !== null) {
resolve([
{} as any,
addContextToErr(
errTREK,
"unable to get registry entry for provided inode"
),
]);
return;
}
let result;
try {
result = await readRegistryEntry(keypair.publicKey, dataKey);
} catch (e: any) {
resolve([
{} as any,
addContextToErr(e, "unable to read registry entry for file"),
]);
return;
}
if (result.exists !== true) {
resolve([{} as any, ERR_NOT_EXISTS]);
return;
}
let [entryID, errDREID] = deriveRegistryEntryID(keypair.publicKey, dataKey);
if (errDREID !== null) {
resolve([
{} as any,
addContextToErr(errDREID, "unable to derive registry entry id"),
]);
return;
}
let skylink = entryIDToSkylink(entryID);
const client = new SkynetClient("https://web3portal.com");
let encryptedData;
try {
encryptedData = await client.downloadData(skylink);
} catch (e: any) {
resolve([{} as any, addContextToErr(e, "unable to download file")]);
return;
}
let encryptionKey = deriveChildSeed(seed, inode);
let [metadata, fileData, errDF] = decryptFileSmall(
encryptionKey,
inode,
encryptedData
);
if (errDF !== null) {
resolve([{} as any, addContextToErr(errDF, "unable to decrypt file")]);
return;
}
let encStr = bufToB64(encryptionKey);
let viewKey = encStr + inode;
let ifile: IndependentFileSmall = {
dataKey,
fileData,
inode,
keypair,
metadata,
revision: result.revision!,
seed,
skylink,
viewKey,
overwriteData: function (newData: Uint8Array): Promise<Err> {
return overwriteIndependentFileSmall(ifile, newData);
},
readData: function (): Promise<[Uint8Array, Err]> {
return new Promise((resolve) => {
let data = new Uint8Array(ifile.fileData.length);
data.set(ifile.fileData, 0);
resolve([data, null]);
});
},
};
resolve([ifile, null]);
});
}
function overwriteIndependentFileSmall(
file: IndependentFileSmall,
newData: Uint8Array
): Promise<Err> {
return new Promise(async (resolve) => {
// Create a new metadata for the file based on the current file
// metadata. Need to update the largest historic size.
let newMetadata: IndependentFileSmallMetadata = {
largestHistoricSize: BigInt(file.metadata.largestHistoricSize),
};
if (BigInt(newData.length) > newMetadata.largestHistoricSize) {
newMetadata.largestHistoricSize = BigInt(newData.length);
}
// Compute the new revision number for the file. This is done
// deterministically using the seed and the current revision number, so
// that multiple concurrent updates will end up with the same revision.
// We use a random number between 1 and 256 for our increment.
let [encodedRevision, errEU64] = encodeU64(file.revision);
if (errEU64 !== null) {
resolve(addContextToErr(errEU64, "unable to encode revision"));
return;
}
let revisionSeed = new Uint8Array(
file.seed.length + encodedRevision.length
);
revisionSeed.set(file.seed, 0);
revisionSeed.set(encodedRevision, file.seed.length);
let revisionKey = deriveChildSeed(revisionSeed, file.inode);
let newRevision = file.revision + BigInt(revisionKey[0]) + 1n;
// Get the encryption key.
let encryptionKey = deriveChildSeed(file.seed, file.inode);
// Create a new encrypted blob for the data.
//
// NOTE: Need to supply the data that would be in place after a
// successful update, which means using the new metadata and revision
// number.
let [encryptedData, errEFS] = encryptFileSmall(
encryptionKey,
file.inode,
newRevision,
newMetadata,
newData,
newMetadata.largestHistoricSize
);
if (errEFS !== null) {
resolve(addContextToErr(errEFS, "unable to encrypt updated file"));
return;
}
// Upload the data to get the immutable link.
let skylink;
try {
skylink = await upload(encryptedData, {
Filename: STD_FILENAME,
});
} catch (e) {
resolve(addContextToErr(e, "new data upload failed"));
return;
}
// Write to the registry entry.
let [entryData, errSTRED] = skylinkToResolverEntryData(skylink);
if (errSTRED !== null) {
resolve(
addContextToErr(
errSTRED,
"could not create resolver link from upload skylink"
)
);
return;
}
try {
await overwriteRegistryEntry(
file.keypair,
file.dataKey,
entryData,
newRevision
);
} catch (e: any) {
resolve(addContextToErr(e, "could not write to registry entry"));
return;
}
// File update was successful, update the file metadata.
file.revision = newRevision;
file.metadata = newMetadata;
file.fileData = newData;
resolve(null);
});
}
export {
createIndependentFileSmall,
openIndependentFileSmall,
overwriteIndependentFileSmall,
};

View File

@ -5,8 +5,6 @@ import { relay } from "@hyperswarm/dht-relay";
// @ts-ignore
import Stream from "@hyperswarm/dht-relay/ws";
import express, { Express } from "express";
import path from "path";
import { fileURLToPath } from "url";
import config from "./config.js";
import * as http from "http";
import * as https from "https";
@ -18,23 +16,34 @@ import cron from "node-cron";
import { get as getDHT } from "./dht.js";
import WS from "ws";
// @ts-ignore
import DHT from "@hyperswarm/dht";
import { pack } from "msgpackr";
import { overwriteRegistryEntry } from "libskynetnode";
import { hashDataKey } from "./util.js";
import {
createIndependentFileSmall,
IndependentFileSmall,
openIndependentFileSmall,
overwriteIndependentFileSmall,
} from "./file.js";
import { seedPhraseToSeed } from "libskynet";
let sslCtx: tls.SecureContext = tls.createSecureContext();
const sslParams: tls.SecureContextOptions = {};
const sslParams: tls.SecureContextOptions = { cert: "", key: "" };
const sslPrivateKey = await acme.forge.createPrivateKey();
const acmeClient = new acme.Client({
accountKey: sslPrivateKey,
directoryUrl: acme.directory.letsencrypt.production,
directoryUrl:
config.str("ssl-mode") === "staging"
? acme.directory.letsencrypt.staging
: acme.directory.letsencrypt.production,
});
let app: Express;
let router = express.Router();
const FILE_CERT_NAME = "/lumeweb/relay/ssl.crt";
const FILE_KEY_NAME = "/lumeweb/relay/ssl.key";
type SslData = { crt: IndependentFileSmall; key: IndependentFileSmall };
export async function start() {
const relayPort = config.str("relay-port");
app = express();
@ -50,7 +59,7 @@ export async function start() {
let httpServer = http.createServer(app);
cron.schedule("0 * * * *", createOrRenewSSl);
cron.schedule("0 * * * *", setupSSl);
await new Promise((resolve) => {
httpServer.listen(80, "0.0.0.0", function () {
@ -73,46 +82,148 @@ export async function start() {
});
});
await createOrRenewSSl();
await setupSSl();
}
async function createOrRenewSSl() {
if (sslParams.cert) {
const expires = (
await acme.forge.readCertificateInfo(sslParams.cert as Buffer)
).notAfter;
async function setupSSl() {
let sslCert = await getSslCert();
let sslKey = await getSslKey();
let certInfo;
let exists = false;
let domainValid = false;
let dateValid = false;
if (sslCert && sslKey) {
sslParams.cert = Buffer.from((sslCert as IndependentFileSmall).fileData);
sslParams.key = Buffer.from((sslKey as IndependentFileSmall).fileData);
certInfo = await getCertInfo();
exists = true;
}
if (exists) {
const expires = certInfo?.notAfter as Date;
let duration = intervalToDuration({ start: new Date(), end: expires });
let daysLeft = (duration.months as number) * 30 + (duration.days as number);
if (daysLeft > 30) {
return;
dateValid = true;
}
if (certInfo?.domains.commonName === config.str("relay-domain")) {
domainValid = true;
}
}
if (dateValid && domainValid) {
return;
}
await createOrRenewSSl(
sslCert as IndependentFileSmall,
sslKey as IndependentFileSmall
);
}
async function createOrRenewSSl(
oldCert?: IndependentFileSmall,
oldKey?: IndependentFileSmall
) {
const [certificateKey, certificateRequest] = await acme.forge.createCsr({
commonName: config.str("relay-domain"),
});
sslParams.cert = await acmeClient.auto({
csr: certificateRequest,
termsOfServiceAgreed: true,
challengeCreateFn: async (authz, challenge, keyAuthorization) => {
router.get(
`/.well-known/acme-challenge/${challenge.token}`,
(req, res) => {
res.send(keyAuthorization);
}
);
},
challengeRemoveFn: async () => {
router = express.Router();
},
challengePriority: ["http-01"],
});
try {
sslParams.cert = await acmeClient.auto({
csr: certificateRequest,
termsOfServiceAgreed: true,
challengeCreateFn: async (authz, challenge, keyAuthorization) => {
router.get(
`/.well-known/acme-challenge/${challenge.token}`,
(req, res) => {
res.send(keyAuthorization);
}
);
},
challengeRemoveFn: async () => {
router = express.Router();
},
challengePriority: ["http-01"],
});
sslParams.cert = Buffer.from(sslParams.cert);
} catch (e: any) {
console.error((e as Error).message);
process.exit(1);
}
sslParams.key = certificateKey;
sslCtx = tls.createSecureContext(sslParams);
console.log("SSL Certificate Updated");
await saveSsl(oldCert, oldKey);
}
async function saveSsl(
oldCert?: IndependentFileSmall,
oldKey?: IndependentFileSmall
): Promise<void> {
const seed = getSeed();
if (oldCert) {
await overwriteIndependentFileSmall(
oldCert,
Buffer.from(sslParams.cert as any)
);
} else {
await createIndependentFileSmall(
seed,
FILE_CERT_NAME,
Buffer.from(sslParams.cert as any)
);
}
if (oldKey) {
await overwriteIndependentFileSmall(
oldKey,
Buffer.from(sslParams.key as any)
);
} else {
await createIndependentFileSmall(
seed,
FILE_KEY_NAME,
Buffer.from(sslParams.key as any)
);
}
}
async function getCertInfo() {
return acme.forge.readCertificateInfo(sslParams.cert as Buffer);
}
async function getSslCert(): Promise<IndependentFileSmall | boolean> {
return getSslFile(FILE_CERT_NAME);
}
async function getSslKey(): Promise<IndependentFileSmall | boolean> {
return getSslFile(FILE_KEY_NAME);
}
async function getSslFile(
name: string
): Promise<IndependentFileSmall | boolean> {
let seed = getSeed();
let [file, err] = await openIndependentFileSmall(seed, name);
if (err) {
return false;
}
return file;
}
function getSeed(): Uint8Array {
let [seed, err] = seedPhraseToSeed(config.str("relay-seed"));
if (err) {
console.error(err);
process.exit(1);
}
return seed;
}