From 909e8be59e3fceafaf38b11a9126c1c792dba0dc Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Fri, 5 Aug 2022 09:25:55 -0400 Subject: [PATCH] *Initial version --- build.js | 13 ++ package.json | 37 +++++ src-build/build.ts | 340 ++++++++++++++++++++++++++++++++++++++++++++ src/index.ts | 313 ++++++++++++++++++++++++++++++++++++++++ tsconfig.build.json | 14 ++ tsconfig.json | 13 ++ 6 files changed, 730 insertions(+) create mode 100644 build.js create mode 100644 package.json create mode 100644 src-build/build.ts create mode 100644 src/index.ts create mode 100644 tsconfig.build.json create mode 100644 tsconfig.json diff --git a/build.js b/build.js new file mode 100644 index 0000000..e2353b2 --- /dev/null +++ b/build.js @@ -0,0 +1,13 @@ +import esbuild from "esbuild"; + +esbuild.buildSync({ + entryPoints: ["src/index.ts"], + outfile: "dist/index.js", + format: "esm", + bundle: true, + legalComments: "external", + // minify: true + define: { + global: "self", + } +}); diff --git a/package.json b/package.json new file mode 100644 index 0000000..8e1f8d0 --- /dev/null +++ b/package.json @@ -0,0 +1,37 @@ +{ + "name": "@lumeweb/kernel-ipfs", + "version": "0.1.0", + "type": "module", + "scripts": { + "test": "jest", + "build-script": "tsc --project tsconfig.build.json && mv dist-build/build.js dist-build/build.mjs", + "compile": "npm run build-script && node build.js", + "build": "npm run compile && node ./dist-build/build.mjs dev" + }, + "dependencies": { + "@lumeweb/kernel-dht-client": "https://github.com/LumeWeb/kernel-dht-client.git", + "@lumeweb/kernel-utils": "https://github.com/LumeWeb/kernel-utils.git", + "buffer": "^6.0.3", + "fetch-retry": "^5.0.3", + "is-ipfs": "^6.0.2", + "libkmodule": "^0.2.12", + "libskynet": "^0.0.62", + "msgpackr": "^1.6.2", + "p-queue": "^7.3.0", + "timers-browserify": "^2.0.12" + }, + "devDependencies": { + "@types/events": "^3.0.0", + "@types/node": "^18.0.3", + "@types/read": "^0.0.29", + "esbuild": "^0.14.49", + "libskynetnode": "^0.1.3", + "prettier": "^2.7.1", + "read": "^1.0.7", + "tslib": "^2.4.0", + "typescript": "^4.7.4" + }, + "browser": { + "timers": "timers-browserify" + } +} diff --git a/src-build/build.ts b/src-build/build.ts new file mode 100644 index 0000000..1ca3cad --- /dev/null +++ b/src-build/build.ts @@ -0,0 +1,340 @@ +// This is the standard build script for a kernel module. + +import * as fs from "fs" +import { + addContextToErr, + b64ToBuf, + bufToHex, + deriveRegistryEntryID, + entryIDToSkylink, + generateSeedPhraseDeterministic, + seedPhraseToSeed, + sha512, + taggedRegistryEntryKeys, +} from "libskynet" +import { generateSeedPhraseRandom, overwriteRegistryEntry, upload } from "libskynetnode" +import read from "read" + +// Helper variables to make it easier to return empty values alongside errors. +const nu8 = new Uint8Array(0) +const nkp = { + publicKey: nu8, + secretKey: nu8, +} + +// readFile is a wrapper for fs.readFileSync that handles the try-catch for the +// caller. +function readFile(fileName: string): [string, string | null] { + try { + let data = fs.readFileSync(fileName, "utf8") + return [data, null] + } catch (err) { + return ["", "unable to read file: " + JSON.stringify(err)] + } +} + +// readFileBinary is a wrapper for fs.readFileSync that handles the try-catch +// for the caller. +function readFileBinary(fileName: string): [Uint8Array, string | null] { + try { + let data = fs.readFileSync(fileName, null) + return [data, null] + } catch (err) { + return [nu8, "unable to read file: " + JSON.stringify(err)] + } +} + +// writeFile is a wrapper for fs.writeFileSync which handles the try-catch in a +// non-exception way. +function writeFile(fileName: string, fileData: string): string | null { + try { + fs.writeFileSync(fileName, fileData) + return null + } catch (err) { + return "unable to write file: " + JSON.stringify(err) + } +} + +// hardenedSeedPhrase will take a password, harden it with 100,000 iterations +// of hashing, and then turn it into a seed phrase. +function hardenedSeedPhrase(password: string): [string, string | null] { + let pw = password + // Add some hashing iterations to the password to make it stronger. + for (let i = 0; i < 1000000; i++) { + let passU8 = new TextEncoder().encode(password) + let hashIter = sha512(passU8) + password = bufToHex(hashIter) + } + return generateSeedPhraseDeterministic(password) +} + +// seedPhraseToRegistryKeys will convert a seed phrase to the set of registry +// keys that govern the registry entry where the module is published. +function seedPhraseToRegistryKeys(seedPhrase: string): [any, Uint8Array, string | null] { + let [seed, errVSP] = seedPhraseToSeed(seedPhrase) + if (errVSP !== null) { + return [nkp, nu8, addContextToErr(errVSP, "unable to compute seed phrase")] + } + let [keypair, datakey, errTREK] = taggedRegistryEntryKeys(seed, "module-build", "module-key") + if (errTREK !== null) { + return [nkp, nu8, addContextToErr(errTREK, "unable to compute registry entry keys")] + } + return [keypair, datakey, null] +} + +// seedPhraseToRegistryLink will take a seedPhrase as input and convert it to +// the registry link for the module. +function seedPhraseToRegistryLink(seedPhrase: string): [string, string | null] { + let [keypair, datakey, errSPTRK] = seedPhraseToRegistryKeys(seedPhrase) + if (errSPTRK !== null) { + return ["", addContextToErr(errSPTRK, "unable to compute registry keys")] + } + let [entryID, errDREID] = deriveRegistryEntryID(keypair.publicKey, datakey) + if (errDREID !== null) { + return ["", addContextToErr(errDREID, "unable to compute registry entry id")] + } + let registryLink = entryIDToSkylink(entryID) + return [registryLink, null] +} + +// handlePass handles all portions of the script that occur after the password +// has been requested. If no password needs to be requested, handlePass will be +// called with a null input. We need to structure the code this way because the +// password reader is async and we can only access the password when using a +// callback. +function handlePass(password: string) { + try { + // If we are running prod and the seed file does not exist, we + // need to confirm the password and also warn the user to use a + // secure password. + if (!fs.existsSync(seedFile) && process.argv[2] === "prod") { + // The file does not exist, we need to confirm the + // password. + console.log() + console.log("No production entry found for module. Creating new production module...") + console.log("If someone can guess the password, they can push arbitrary changes to your module.") + console.log("Please use a secure password.") + console.log() + read({ prompt: "Confirm Password: ", silent: true }, function (err: any, confirmPassword: string) { + if (err) { + console.error("unable to fetch password:", err) + process.exit(1) + } + if (password !== confirmPassword) { + console.error("passwords do not match") + process.exit(1) + } + password = password + moduleSalt + handlePassConfirm(password) + }) + } else { + // If the seed file does exist, or if we are using dev, + // there's no need to confirm the password but we do + // need to pass the logic off to the handlePassConfirm + // callback. + password = password + moduleSalt + handlePassConfirm(password) + } + } catch (err) { + console.error("Unable to read seedFile:", err) + process.exit(1) + } +} + +// handlePassConfirm handles the full script after the confirmation password +// has been provided. If not confirmation password is needed, this function +// will be called anyway using the unconfirmed password as input. +function handlePassConfirm(password: string) { + // Create the seedFile if it does not exist. For dev we just save the + // seed to disk outright, because this is a dev build and therefore not + // security sensitive. Also the dev seed does not get pushed to the + // github repo. + // + // For prod, we use the seed to create a new seed (called the shield) + // which allows us to verify that the developer has provided the right + // password when deploying the module. The shield does get pushed to + // the github repo so that the production module is the same on all + // devices. + if (!fs.existsSync(seedFile) && process.argv[2] !== "prod") { + // Generate the seed phrase and write it to the file. + let [seedPhrase, errGSP] = generateSeedPhraseRandom() + if (errGSP !== null) { + console.error("Unable to generate seed phrase:", errGSP) + process.exit(1) + } + let errWF = writeFile(seedFile, seedPhrase) + if (errWF !== null) { + console.error("unable to write file:", errWF) + process.exit(1) + } + } else if (!fs.existsSync(seedFile) && process.argv[2] === "prod") { + // Generate the seed phrase. + let [seedPhrase, errGSP] = hardenedSeedPhrase(password) + if (errGSP !== null) { + console.error("Unable to generate seed phrase:", errGSP) + process.exit(1) + } + let [registryLink, errSPTRL] = seedPhraseToRegistryLink(seedPhrase) + if (errSPTRL !== null) { + console.error("Unable to generate registry link:", errSPTRL) + process.exit(1) + } + + // Write the registry link to the file. + let errWF = writeFile(seedFile, registryLink) + if (errWF !== null) { + console.error("unable to write registry link file:", errWF) + process.exit(1) + } + } + + // Load or verify the seed. If this is prod, the password is used to + // create and verify the seed. If this is dev, we just load the seed + // with no password. + let seedPhrase: string + let registryLink: string + if (process.argv[2] === "prod") { + // Generate the seed phrase from the password. + let [sp, errGSP] = hardenedSeedPhrase(password) + if (errGSP !== null) { + console.error("Unable to generate seed phrase: ", errGSP) + process.exit(1) + } + let [rl, errSPTRL] = seedPhraseToRegistryLink(sp) + registryLink = rl + if (errSPTRL !== null) { + console.error("Unable to generate registry link:", errSPTRL) + process.exit(1) + } + let [registryLinkVerify, errRF] = readFile(seedFile) + if (errRF !== null) { + console.error("unable to read seedFile") + process.exit(1) + } + registryLinkVerify = registryLinkVerify.replace(/\n$/, "") + if (registryLink !== registryLinkVerify) { + console.error("Incorrect password") + process.exit(1) + } + seedPhrase = sp + } else { + let [sp, errRF] = readFile(seedFile) + if (errRF !== null) { + console.error("unable to read seed phrase for dev command from disk") + process.exit(1) + } + let [rl, errSPTRL] = seedPhraseToRegistryLink(sp) + registryLink = rl + if (errSPTRL !== null) { + console.error("Unable to generate registry link:", errSPTRL) + process.exit(1) + } + // Write the registry link to the module skylink dev file. + let errWF = writeFile("build/module-skylink-dev", registryLink) + if (errWF !== null) { + console.error("unable to write registry link file:", errWF) + process.exit(1) + } + seedPhrase = sp + } + + // Upload the module to Skynet. + let [distFile, errRF] = readFileBinary("dist/index.js") + if (errRF !== null) { + console.error("unable to read dist file for module") + process.exit(1) + } + let metadata = { + Filename: "index.js", + } + console.log("Uploading module...") + upload(distFile, metadata) + .then((result) => { + console.log("Updating module's registry entry...") + // Update the v2 skylink. + let [keypair, datakey, errSPTRK] = seedPhraseToRegistryKeys(seedPhrase) + if (errSPTRK !== null) { + return ["", addContextToErr(errSPTRK, "unable to compute registry keys")] + } + let [bufLink, errBTB] = b64ToBuf(result) + if (errBTB !== null) { + return ["", addContextToErr(errBTB, "unable to decode skylink")] + } + overwriteRegistryEntry(keypair, datakey, bufLink) + .then(() => { + console.log("registry entry is updated") + console.log("Immutable Link for Module:", result) + console.log("Resolver Link for Module:", registryLink) + }) + .catch((err: any) => { + console.log("unable to update registry entry:", err) + }) + }) + .catch((err) => { + console.error("unable to upload file", err) + process.exit(1) + }) +} + +// Add a newline for readability. +console.log() + +// Check for a 'dev' or 'prod' input to the script. +if (process.argv.length !== 3) { + console.error("need to provide either 'dev' or 'prod' as an input") + process.exit(1) +} + +// Create the build folder if it does not exist. +if (!fs.existsSync("build")) { + fs.mkdirSync("build") +} + +// Determine the seed file. +let seedFile: string +if (process.argv[2] === "prod") { + seedFile = "module-skylink" +} else if (process.argv[2] === "dev") { + seedFile = "build/dev-seed" +} else { + console.error("need to provide either 'dev' or 'prod' as an input") + process.exit(1) +} + +// If doing a prod deployment, check whether the salt file exists. If it does +// not, create it. +let moduleSalt: string +if (!fs.existsSync(".module-salt")) { + let [ms, errGSPR] = generateSeedPhraseRandom() + if (errGSPR !== null) { + console.error("unable to generate module salt:", errGSPR) + process.exit(1) + } + moduleSalt = ms + let errWF = writeFile(".module-salt", moduleSalt) + if (errWF !== null) { + console.error("unable to write module salt file:", errWF) + process.exit(1) + } +} else { + let [ms, errRF] = readFile(".module-salt") + if (errRF !== null) { + console.error("unable to read moduleSalt") + process.exit(1) + } + ms = ms.replace(/\n$/, "") + moduleSalt = ms +} + +// Need to get a password if this is a prod build. +if (process.argv[2] === "prod") { + read({ prompt: "Password: ", silent: true }, function (err: any, password: string) { + if (err) { + console.error("unable to fetch password:", err) + process.exit(1) + } + handlePass(password) + }) +} else { + handlePass("") +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..9baeae9 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,313 @@ +import { addHandler, handleMessage } from "libkmodule"; +import type { ActiveQuery } from "libkmodule"; +import PQueue from "p-queue"; +import { ipfsPath, ipnsPath } from "is-ipfs"; +import { DHT } from "@lumeweb/kernel-dht-client"; +import { pack, unpack } from "msgpackr"; +import { clearTimeout } from "timers"; + +onmessage = handleMessage; + +interface StatFileResponse { + exists: boolean; + contentType: string | null; + error: any; + directory: boolean; + files: string[]; +} + +let blockingGatewayUpdate = Promise.resolve(); + +let activeRelays = []; +let relays = [ + "25c2a0a833782d64213c08879b95dd5a60af244b44a058f3a7a70d6722f4bda7", +]; + +let dht: DHT; + +addHandler("presentSeed", handlePresentSeed); +addHandler("refreshGatewayList", handleRefreshGatewayList); +addHandler("statIpfs", handleStatIpfs); +addHandler("fetchIpfs", handleFetchIpfs, { se }); +addHandler("statIpns", handleStatIpns); +addHandler("fetchIpns", handleFetchIpns); + +let readyPromiseResolve: any; +let readyPromise = new Promise((resolve) => { + readyPromiseResolve = resolve; +}); + +async function handlePresentSeed(aq: ActiveQuery) { + dht = new DHT(false); + for (const relay of relays) { + await dht.addRelay(relay); + } + await dht.ready(); + refreshGatewayList(); + aq.respond(); + readyPromiseResolve(); +} + +async function handleRefreshGatewayList(aq: ActiveQuery) { + await readyPromise; + await blockingGatewayUpdate; + await refreshGatewayList(); + aq.respond(); +} + +async function handleStatIpfs(aq: ActiveQuery) { + return handleStat(aq, "stat_ipfs", "ipfs"); +} + +async function handleFetchIpfs(aq: ActiveQuery) { + return handleFetch(aq, "fetch_ipfs", "ipfs"); +} + +async function handleStatIpns(aq: ActiveQuery) { + return handleStat(aq, "stat_ipns", "ipfs"); +} + +async function handleFetchIpns(aq: ActiveQuery) { + return handleStat(aq, "fetch_ipns", "ipns"); +} + +async function validateInputs(aq: ActiveQuery, type: "ipns" | "ipfs") { + const { hash = null } = aq.callerInput; + const { path = "" } = aq.callerInput; + if (!hash) { + aq.reject("hash missing"); + return; + } + + if (type === "ipfs" && !ipfsPath(`/ipfs/${hash}`)) { + aq.reject("ipfs hash is invalid"); + return; + } + + if (type === "ipns" && !ipnsPath(`/ipns/${hash}`)) { + aq.reject("ipns hash is invalid"); + return; + } + await readyPromise; + await blockingGatewayUpdate; + + return { hash, path }; +} + +async function handleStat( + aq: ActiveQuery, + method: string, + type: "ipns" | "ipfs" +): Promise { + const valid = await validateInputs(aq, type); + if (!valid) { + return; + } + const { hash, path } = valid; + try { + let resp = (await fetchFromRelays(hash, path, method)) as StatFileResponse; + aq.respond(resp); + } catch (e) { + aq.reject(e); + } +} + +async function handleFetch( + aq: ActiveQuery, + method: string, + type: "ipns" | "ipfs" +): Promise { + const valid = await validateInputs(aq, type); + if (!valid) { + return; + } + const { hash, path } = valid; + + try { + await fetchFromRelays(hash, path, method, aq.sendUpdate); + aq.respond(); + } catch (e) { + aq.reject(e); + } +} + +async function fetchFromRelays( + hash: string, + path: string, + method: string, + stream = undefined +) { + let error = new Error("NOT_FOUND"); + for (const relay of activeRelays) { + let resp; + try { + resp = await rpcCall(relay, "ipfs", method, stream, { + hash, + path, + }); + } catch (e: any) { + if (e instanceof Error) { + error = e; + } else { + error = new Error(e); + } + continue; + } + + if (resp) { + return resp; + } + } + + throw error; +} + +async function relayHasMethods( + methodList: string[], + relay: string +): Promise { + let methods = []; + try { + methods = (await rpcCall(relay, "misc", "get_methods")) as []; + } catch (e) { + return false; + } + + let has = true; + + methodList.forEach((item) => { + if (!methods.includes(item)) { + has = false; + } + }); + + return has; +} + +async function rpcCall( + relay: string, + chain: string, + query: string, + stream?: (data) => void, + data = {} +) { + const socket = await dht.connect(relay); + return new Promise((resolve, reject) => { + let timer: NodeJS.Timeout; + let dataCount = 0; + socket.on("data", (res) => { + clearTimeout(timer as number); + dataCount++; + const response = unpack(res); + if (!response || response.error || (response && response?.data?.error)) { + socket.end(); + reject(response?.error || response?.data?.error); + return; + } + if (!stream && 1 === dataCount) { + socket.end(); + resolve(response?.data); + return; + } + + if (stream) { + stream(response?.data.data); + if (response?.data.done) { + socket.end(); + } + } + }); + socket.write("rpc"); + socket.write( + pack({ + query, + chain, + data, + force: true, + }) + ); + timer = setTimeout(() => { + socket.end(); + reject("timeout"); + }, 5 * 1000) as NodeJS.Timeout; + }); +} + +async function processStream(aq: ActiveQuery, response: Response) { + const reader = response.body.getReader(); + + let aqResp = { + headers: Array.from(response.headers), + status: response.status, + }; + while (true) { + let chunk; + + try { + chunk = await reader.read(); + aq.sendUpdate(chunk.value); + if (chunk.done) { + aq.respond(aqResp); + break; + } + } catch (e) { + aq.respond(aqResp); + break; + } + } +} + +async function refreshGatewayList() { + let processResolve: any; + blockingGatewayUpdate = new Promise((resolve) => { + processResolve = resolve; + }); + const queue = new PQueue({ concurrency: 10 }); + + let latencies = []; + + relays.forEach((item) => { + queue.add(checkRelayLatency(item, latencies)); + }); + + await queue.onIdle(); + + activeRelays = latencies + .sort((a: any[], b: any[]) => { + return a[0] - b[0]; + }) + .map((item: any[]) => item[1]); + processResolve(); +} + +function checkRelayLatency(relay: string, list: any[]) { + return async () => { + const start = Date.now(); + + let resp; + try { + resp = await rpcCall(relay, "misc", "ping", null, {}); + } catch { + return; + } + if (!resp.pong) { + return; + } + + const end = Date.now() - start; + + try { + resp = await relayHasMethods( + ["stat_ipfs", "stat_ipns", "fetch_ipfs", "fetch_ipns"], + relay + ); + if (!resp) { + return; + } + } catch { + return; + } + + list.push([end, relay]); + }; +} diff --git a/tsconfig.build.json b/tsconfig.build.json new file mode 100644 index 0000000..90439ba --- /dev/null +++ b/tsconfig.build.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "esnext", + "module": "esnext", + "moduleResolution": "node", + "allowSyntheticDefaultImports": true, + "declaration": true, + "outDir": "./dist-build", + "strict": true, + "esModuleInterop": true + }, + "include": ["src-build"], + "exclude": ["node_modules", "**/__tests__/*"] +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..c24bef2 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "esnext", + "declaration": true, + "moduleResolution": "node", + "outDir": "./build", + "strict": true, + "allowSyntheticDefaultImports": true, + "esModuleInterop": true + }, + "include": ["src"], + "exclude": ["node_modules", "**/__tests__/*"] +}