*Initial version

build.js

@@ -0,0 +1,13 @@
+import esbuild from "esbuild";
+
+esbuild.buildSync({
+  entryPoints: ["src/index.ts"],
+  outfile: "dist/index.js",
+  format: "esm",
+  bundle: true,
+  legalComments: "external",
+  //  minify: true
+  define: {
+    global: "self",
+  }
+});

package.json

@@ -0,0 +1,37 @@
+{
+    "name": "@lumeweb/kernel-ipfs",
+    "version": "0.1.0",
+    "type": "module",
+    "scripts": {
+        "test": "jest",
+        "build-script": "tsc --project tsconfig.build.json && mv dist-build/build.js dist-build/build.mjs",
+        "compile": "npm run build-script && node build.js",
+        "build": "npm run compile && node ./dist-build/build.mjs dev"
+    },
+    "dependencies": {
+        "@lumeweb/kernel-dht-client": "https://github.com/LumeWeb/kernel-dht-client.git",
+        "@lumeweb/kernel-utils": "https://github.com/LumeWeb/kernel-utils.git",
+        "buffer": "^6.0.3",
+        "fetch-retry": "^5.0.3",
+        "is-ipfs": "^6.0.2",
+        "libkmodule": "^0.2.12",
+        "libskynet": "^0.0.62",
+        "msgpackr": "^1.6.2",
+        "p-queue": "^7.3.0",
+        "timers-browserify": "^2.0.12"
+    },
+    "devDependencies": {
+        "@types/events": "^3.0.0",
+        "@types/node": "^18.0.3",
+        "@types/read": "^0.0.29",
+        "esbuild": "^0.14.49",
+        "libskynetnode": "^0.1.3",
+        "prettier": "^2.7.1",
+        "read": "^1.0.7",
+        "tslib": "^2.4.0",
+        "typescript": "^4.7.4"
+    },
+    "browser": {
+        "timers": "timers-browserify"
+    }
+}

src-build/build.ts

@@ -0,0 +1,340 @@
+// This is the standard build script for a kernel module.
+
+import * as fs from "fs"
+import {
+	addContextToErr,
+	b64ToBuf,
+	bufToHex,
+	deriveRegistryEntryID,
+	entryIDToSkylink,
+	generateSeedPhraseDeterministic,
+	seedPhraseToSeed,
+	sha512,
+	taggedRegistryEntryKeys,
+} from "libskynet"
+import { generateSeedPhraseRandom, overwriteRegistryEntry, upload } from "libskynetnode"
+import read from "read"
+
+// Helper variables to make it easier to return empty values alongside errors.
+const nu8 = new Uint8Array(0)
+const nkp = {
+	publicKey: nu8,
+	secretKey: nu8,
+}
+
+// readFile is a wrapper for fs.readFileSync that handles the try-catch for the
+// caller.
+function readFile(fileName: string): [string, string | null] {
+	try {
+		let data = fs.readFileSync(fileName, "utf8")
+		return [data, null]
+	} catch (err) {
+		return ["", "unable to read file: " + JSON.stringify(err)]
+	}
+}
+
+// readFileBinary is a wrapper for fs.readFileSync that handles the try-catch
+// for the caller.
+function readFileBinary(fileName: string): [Uint8Array, string | null] {
+	try {
+		let data = fs.readFileSync(fileName, null)
+		return [data, null]
+	} catch (err) {
+		return [nu8, "unable to read file: " + JSON.stringify(err)]
+	}
+}
+
+// writeFile is a wrapper for fs.writeFileSync which handles the try-catch in a
+// non-exception way.
+function writeFile(fileName: string, fileData: string): string | null {
+	try {
+		fs.writeFileSync(fileName, fileData)
+		return null
+	} catch (err) {
+		return "unable to write file: " + JSON.stringify(err)
+	}
+}
+
+// hardenedSeedPhrase will take a password, harden it with 100,000 iterations
+// of hashing, and then turn it into a seed phrase.
+function hardenedSeedPhrase(password: string): [string, string | null] {
+	let pw = password
+	// Add some hashing iterations to the password to make it stronger.
+	for (let i = 0; i < 1000000; i++) {
+		let passU8 = new TextEncoder().encode(password)
+		let hashIter = sha512(passU8)
+		password = bufToHex(hashIter)
+	}
+	return generateSeedPhraseDeterministic(password)
+}
+
+// seedPhraseToRegistryKeys will convert a seed phrase to the set of registry
+// keys that govern the registry entry where the module is published.
+function seedPhraseToRegistryKeys(seedPhrase: string): [any, Uint8Array, string | null] {
+	let [seed, errVSP] = seedPhraseToSeed(seedPhrase)
+	if (errVSP !== null) {
+		return [nkp, nu8, addContextToErr(errVSP, "unable to compute seed phrase")]
+	}
+	let [keypair, datakey, errTREK] = taggedRegistryEntryKeys(seed, "module-build", "module-key")
+	if (errTREK !== null) {
+		return [nkp, nu8, addContextToErr(errTREK, "unable to compute registry entry keys")]
+	}
+	return [keypair, datakey, null]
+}
+
+// seedPhraseToRegistryLink will take a seedPhrase as input and convert it to
+// the registry link for the module.
+function seedPhraseToRegistryLink(seedPhrase: string): [string, string | null] {
+	let [keypair, datakey, errSPTRK] = seedPhraseToRegistryKeys(seedPhrase)
+	if (errSPTRK !== null) {
+		return ["", addContextToErr(errSPTRK, "unable to compute registry keys")]
+	}
+	let [entryID, errDREID] = deriveRegistryEntryID(keypair.publicKey, datakey)
+	if (errDREID !== null) {
+		return ["", addContextToErr(errDREID, "unable to compute registry entry id")]
+	}
+	let registryLink = entryIDToSkylink(entryID)
+	return [registryLink, null]
+}
+
+// handlePass handles all portions of the script that occur after the password
+// has been requested. If no password needs to be requested, handlePass will be
+// called with a null input. We need to structure the code this way because the
+// password reader is async and we can only access the password when using a
+// callback.
+function handlePass(password: string) {
+	try {
+		// If we are running prod and the seed file does not exist, we
+		// need to confirm the password and also warn the user to use a
+		// secure password.
+		if (!fs.existsSync(seedFile) && process.argv[2] === "prod") {
+			// The file does not exist, we need to confirm the
+			// password.
+			console.log()
+			console.log("No production entry found for module. Creating new production module...")
+			console.log("If someone can guess the password, they can push arbitrary changes to your module.")
+			console.log("Please use a secure password.")
+			console.log()
+			read({ prompt: "Confirm Password: ", silent: true }, function (err: any, confirmPassword: string) {
+				if (err) {
+					console.error("unable to fetch password:", err)
+					process.exit(1)
+				}
+				if (password !== confirmPassword) {
+					console.error("passwords do not match")
+					process.exit(1)
+				}
+				password = password + moduleSalt
+				handlePassConfirm(password)
+			})
+		} else {
+			// If the seed file does exist, or if we are using dev,
+			// there's no need to confirm the password but we do
+			// need to pass the logic off to the handlePassConfirm
+			// callback.
+			password = password + moduleSalt
+			handlePassConfirm(password)
+		}
+	} catch (err) {
+		console.error("Unable to read seedFile:", err)
+		process.exit(1)
+	}
+}
+
+// handlePassConfirm handles the full script after the confirmation password
+// has been provided. If not confirmation password is needed, this function
+// will be called anyway using the unconfirmed password as input.
+function handlePassConfirm(password: string) {
+	// Create the seedFile if it does not exist. For dev we just save the
+	// seed to disk outright, because this is a dev build and therefore not
+	// security sensitive. Also the dev seed does not get pushed to the
+	// github repo.
+	//
+	// For prod, we use the seed to create a new seed (called the shield)
+	// which allows us to verify that the developer has provided the right
+	// password when deploying the module. The shield does get pushed to
+	// the github repo so that the production module is the same on all
+	// devices.
+	if (!fs.existsSync(seedFile) && process.argv[2] !== "prod") {
+		// Generate the seed phrase and write it to the file.
+		let [seedPhrase, errGSP] = generateSeedPhraseRandom()
+		if (errGSP !== null) {
+			console.error("Unable to generate seed phrase:", errGSP)
+			process.exit(1)
+		}
+		let errWF = writeFile(seedFile, seedPhrase)
+		if (errWF !== null) {
+			console.error("unable to write file:", errWF)
+			process.exit(1)
+		}
+	} else if (!fs.existsSync(seedFile) && process.argv[2] === "prod") {
+		// Generate the seed phrase.
+		let [seedPhrase, errGSP] = hardenedSeedPhrase(password)
+		if (errGSP !== null) {
+			console.error("Unable to generate seed phrase:", errGSP)
+			process.exit(1)
+		}
+		let [registryLink, errSPTRL] = seedPhraseToRegistryLink(seedPhrase)
+		if (errSPTRL !== null) {
+			console.error("Unable to generate registry link:", errSPTRL)
+			process.exit(1)
+		}
+
+		// Write the registry link to the file.
+		let errWF = writeFile(seedFile, registryLink)
+		if (errWF !== null) {
+			console.error("unable to write registry link file:", errWF)
+			process.exit(1)
+		}
+	}
+
+	// Load or verify the seed. If this is prod, the password is used to
+	// create and verify the seed. If this is dev, we just load the seed
+	// with no password.
+	let seedPhrase: string
+	let registryLink: string
+	if (process.argv[2] === "prod") {
+		// Generate the seed phrase from the password.
+		let [sp, errGSP] = hardenedSeedPhrase(password)
+		if (errGSP !== null) {
+			console.error("Unable to generate seed phrase: ", errGSP)
+			process.exit(1)
+		}
+		let [rl, errSPTRL] = seedPhraseToRegistryLink(sp)
+		registryLink = rl
+		if (errSPTRL !== null) {
+			console.error("Unable to generate registry link:", errSPTRL)
+			process.exit(1)
+		}
+		let [registryLinkVerify, errRF] = readFile(seedFile)
+		if (errRF !== null) {
+			console.error("unable to read seedFile")
+			process.exit(1)
+		}
+		registryLinkVerify = registryLinkVerify.replace(/\n$/, "")
+		if (registryLink !== registryLinkVerify) {
+			console.error("Incorrect password")
+			process.exit(1)
+		}
+		seedPhrase = sp
+	} else {
+		let [sp, errRF] = readFile(seedFile)
+		if (errRF !== null) {
+			console.error("unable to read seed phrase for dev command from disk")
+			process.exit(1)
+		}
+		let [rl, errSPTRL] = seedPhraseToRegistryLink(sp)
+		registryLink = rl
+		if (errSPTRL !== null) {
+			console.error("Unable to generate registry link:", errSPTRL)
+			process.exit(1)
+		}
+		// Write the registry link to the module skylink dev file.
+		let errWF = writeFile("build/module-skylink-dev", registryLink)
+		if (errWF !== null) {
+			console.error("unable to write registry link file:", errWF)
+			process.exit(1)
+		}
+		seedPhrase = sp
+	}
+
+	// Upload the module to Skynet.
+	let [distFile, errRF] = readFileBinary("dist/index.js")
+	if (errRF !== null) {
+		console.error("unable to read dist file for module")
+		process.exit(1)
+	}
+	let metadata = {
+		Filename: "index.js",
+	}
+	console.log("Uploading module...")
+	upload(distFile, metadata)
+		.then((result) => {
+			console.log("Updating module's registry entry...")
+			// Update the v2 skylink.
+			let [keypair, datakey, errSPTRK] = seedPhraseToRegistryKeys(seedPhrase)
+			if (errSPTRK !== null) {
+				return ["", addContextToErr(errSPTRK, "unable to compute registry keys")]
+			}
+			let [bufLink, errBTB] = b64ToBuf(result)
+			if (errBTB !== null) {
+				return ["", addContextToErr(errBTB, "unable to decode skylink")]
+			}
+			overwriteRegistryEntry(keypair, datakey, bufLink)
+				.then(() => {
+					console.log("registry entry is updated")
+					console.log("Immutable Link for Module:", result)
+					console.log("Resolver Link for Module:", registryLink)
+				})
+				.catch((err: any) => {
+					console.log("unable to update registry entry:", err)
+				})
+		})
+		.catch((err) => {
+			console.error("unable to upload file", err)
+			process.exit(1)
+		})
+}
+
+// Add a newline for readability.
+console.log()
+
+// Check for a 'dev' or 'prod' input to the script.
+if (process.argv.length !== 3) {
+	console.error("need to provide either 'dev' or 'prod' as an input")
+	process.exit(1)
+}
+
+// Create the build folder if it does not exist.
+if (!fs.existsSync("build")) {
+	fs.mkdirSync("build")
+}
+
+// Determine the seed file.
+let seedFile: string
+if (process.argv[2] === "prod") {
+	seedFile = "module-skylink"
+} else if (process.argv[2] === "dev") {
+	seedFile = "build/dev-seed"
+} else {
+	console.error("need to provide either 'dev' or 'prod' as an input")
+	process.exit(1)
+}
+
+// If doing a prod deployment, check whether the salt file exists. If it does
+// not, create it.
+let moduleSalt: string
+if (!fs.existsSync(".module-salt")) {
+	let [ms, errGSPR] = generateSeedPhraseRandom()
+	if (errGSPR !== null) {
+		console.error("unable to generate module salt:", errGSPR)
+		process.exit(1)
+	}
+	moduleSalt = ms
+	let errWF = writeFile(".module-salt", moduleSalt)
+	if (errWF !== null) {
+		console.error("unable to write module salt file:", errWF)
+		process.exit(1)
+	}
+} else {
+	let [ms, errRF] = readFile(".module-salt")
+	if (errRF !== null) {
+		console.error("unable to read moduleSalt")
+		process.exit(1)
+	}
+	ms = ms.replace(/\n$/, "")
+	moduleSalt = ms
+}
+
+// Need to get a password if this is a prod build.
+if (process.argv[2] === "prod") {
+	read({ prompt: "Password: ", silent: true }, function (err: any, password: string) {
+		if (err) {
+			console.error("unable to fetch password:", err)
+			process.exit(1)
+		}
+		handlePass(password)
+	})
+} else {
+	handlePass("")
+}

src/index.ts

@@ -0,0 +1,313 @@
+import { addHandler, handleMessage } from "libkmodule";
+import type { ActiveQuery } from "libkmodule";
+import PQueue from "p-queue";
+import { ipfsPath, ipnsPath } from "is-ipfs";
+import { DHT } from "@lumeweb/kernel-dht-client";
+import { pack, unpack } from "msgpackr";
+import { clearTimeout } from "timers";
+
+onmessage = handleMessage;
+
+interface StatFileResponse {
+  exists: boolean;
+  contentType: string | null;
+  error: any;
+  directory: boolean;
+  files: string[];
+}
+
+let blockingGatewayUpdate = Promise.resolve();
+
+let activeRelays = [];
+let relays = [
+  "25c2a0a833782d64213c08879b95dd5a60af244b44a058f3a7a70d6722f4bda7",
+];
+
+let dht: DHT;
+
+addHandler("presentSeed", handlePresentSeed);
+addHandler("refreshGatewayList", handleRefreshGatewayList);
+addHandler("statIpfs", handleStatIpfs);
+addHandler("fetchIpfs", handleFetchIpfs, { se });
+addHandler("statIpns", handleStatIpns);
+addHandler("fetchIpns", handleFetchIpns);
+
+let readyPromiseResolve: any;
+let readyPromise = new Promise((resolve) => {
+  readyPromiseResolve = resolve;
+});
+
+async function handlePresentSeed(aq: ActiveQuery) {
+  dht = new DHT(false);
+  for (const relay of relays) {
+    await dht.addRelay(relay);
+  }
+  await dht.ready();
+  refreshGatewayList();
+  aq.respond();
+  readyPromiseResolve();
+}
+
+async function handleRefreshGatewayList(aq: ActiveQuery) {
+  await readyPromise;
+  await blockingGatewayUpdate;
+  await refreshGatewayList();
+  aq.respond();
+}
+
+async function handleStatIpfs(aq: ActiveQuery) {
+  return handleStat(aq, "stat_ipfs", "ipfs");
+}
+
+async function handleFetchIpfs(aq: ActiveQuery) {
+  return handleFetch(aq, "fetch_ipfs", "ipfs");
+}
+
+async function handleStatIpns(aq: ActiveQuery) {
+  return handleStat(aq, "stat_ipns", "ipfs");
+}
+
+async function handleFetchIpns(aq: ActiveQuery) {
+  return handleStat(aq, "fetch_ipns", "ipns");
+}
+
+async function validateInputs(aq: ActiveQuery, type: "ipns" | "ipfs") {
+  const { hash = null } = aq.callerInput;
+  const { path = "" } = aq.callerInput;
+  if (!hash) {
+    aq.reject("hash missing");
+    return;
+  }
+
+  if (type === "ipfs" && !ipfsPath(`/ipfs/${hash}`)) {
+    aq.reject("ipfs hash is invalid");
+    return;
+  }
+
+  if (type === "ipns" && !ipnsPath(`/ipns/${hash}`)) {
+    aq.reject("ipns hash is invalid");
+    return;
+  }
+  await readyPromise;
+  await blockingGatewayUpdate;
+
+  return { hash, path };
+}
+
+async function handleStat(
+  aq: ActiveQuery,
+  method: string,
+  type: "ipns" | "ipfs"
+): Promise<void> {
+  const valid = await validateInputs(aq, type);
+  if (!valid) {
+    return;
+  }
+  const { hash, path } = valid;
+  try {
+    let resp = (await fetchFromRelays(hash, path, method)) as StatFileResponse;
+    aq.respond(resp);
+  } catch (e) {
+    aq.reject(e);
+  }
+}
+
+async function handleFetch(
+  aq: ActiveQuery,
+  method: string,
+  type: "ipns" | "ipfs"
+): Promise<void> {
+  const valid = await validateInputs(aq, type);
+  if (!valid) {
+    return;
+  }
+  const { hash, path } = valid;
+
+  try {
+    await fetchFromRelays(hash, path, method, aq.sendUpdate);
+    aq.respond();
+  } catch (e) {
+    aq.reject(e);
+  }
+}
+
+async function fetchFromRelays(
+  hash: string,
+  path: string,
+  method: string,
+  stream = undefined
+) {
+  let error = new Error("NOT_FOUND");
+  for (const relay of activeRelays) {
+    let resp;
+    try {
+      resp = await rpcCall(relay, "ipfs", method, stream, {
+        hash,
+        path,
+      });
+    } catch (e: any) {
+      if (e instanceof Error) {
+        error = e;
+      } else {
+        error = new Error(e);
+      }
+      continue;
+    }
+
+    if (resp) {
+      return resp;
+    }
+  }
+
+  throw error;
+}
+
+async function relayHasMethods(
+  methodList: string[],
+  relay: string
+): Promise<boolean> {
+  let methods = [];
+  try {
+    methods = (await rpcCall(relay, "misc", "get_methods")) as [];
+  } catch (e) {
+    return false;
+  }
+
+  let has = true;
+
+  methodList.forEach((item) => {
+    if (!methods.includes(item)) {
+      has = false;
+    }
+  });
+
+  return has;
+}
+
+async function rpcCall(
+  relay: string,
+  chain: string,
+  query: string,
+  stream?: (data) => void,
+  data = {}
+) {
+  const socket = await dht.connect(relay);
+  return new Promise((resolve, reject) => {
+    let timer: NodeJS.Timeout;
+    let dataCount = 0;
+    socket.on("data", (res) => {
+      clearTimeout(timer as number);
+      dataCount++;
+      const response = unpack(res);
+      if (!response || response.error || (response && response?.data?.error)) {
+        socket.end();
+        reject(response?.error || response?.data?.error);
+        return;
+      }
+      if (!stream && 1 === dataCount) {
+        socket.end();
+        resolve(response?.data);
+        return;
+      }
+
+      if (stream) {
+        stream(response?.data.data);
+        if (response?.data.done) {
+          socket.end();
+        }
+      }
+    });
+    socket.write("rpc");
+    socket.write(
+      pack({
+        query,
+        chain,
+        data,
+        force: true,
+      })
+    );
+    timer = setTimeout(() => {
+      socket.end();
+      reject("timeout");
+    }, 5 * 1000) as NodeJS.Timeout;
+  });
+}
+
+async function processStream(aq: ActiveQuery, response: Response) {
+  const reader = response.body.getReader();
+
+  let aqResp = {
+    headers: Array.from(response.headers),
+    status: response.status,
+  };
+  while (true) {
+    let chunk;
+
+    try {
+      chunk = await reader.read();
+      aq.sendUpdate(chunk.value);
+      if (chunk.done) {
+        aq.respond(aqResp);
+        break;
+      }
+    } catch (e) {
+      aq.respond(aqResp);
+      break;
+    }
+  }
+}
+
+async function refreshGatewayList() {
+  let processResolve: any;
+  blockingGatewayUpdate = new Promise((resolve) => {
+    processResolve = resolve;
+  });
+  const queue = new PQueue({ concurrency: 10 });
+
+  let latencies = [];
+
+  relays.forEach((item) => {
+    queue.add(checkRelayLatency(item, latencies));
+  });
+
+  await queue.onIdle();
+
+  activeRelays = latencies
+    .sort((a: any[], b: any[]) => {
+      return a[0] - b[0];
+    })
+    .map((item: any[]) => item[1]);
+  processResolve();
+}
+
+function checkRelayLatency(relay: string, list: any[]) {
+  return async () => {
+    const start = Date.now();
+
+    let resp;
+    try {
+      resp = await rpcCall(relay, "misc", "ping", null, {});
+    } catch {
+      return;
+    }
+    if (!resp.pong) {
+      return;
+    }
+
+    const end = Date.now() - start;
+
+    try {
+      resp = await relayHasMethods(
+        ["stat_ipfs", "stat_ipns", "fetch_ipfs", "fetch_ipns"],
+        relay
+      );
+      if (!resp) {
+        return;
+      }
+    } catch {
+      return;
+    }
+
+    list.push([end, relay]);
+  };
+}

tsconfig.build.json

@@ -0,0 +1,14 @@
+{
+	"compilerOptions": {
+		"target": "esnext",
+		"module": "esnext",
+		"moduleResolution": "node",
+		"allowSyntheticDefaultImports": true,
+		"declaration": true,
+		"outDir": "./dist-build",
+		"strict": true,
+        "esModuleInterop": true
+	},
+	"include": ["src-build"],
+	"exclude": ["node_modules", "**/__tests__/*"]
+}

tsconfig.json

@@ -0,0 +1,13 @@
+{
+	"compilerOptions": {
+		"target": "esnext",
+		"declaration": true,
+		"moduleResolution": "node",
+		"outDir": "./build",
+		"strict": true,
+        "allowSyntheticDefaultImports": true,
+        "esModuleInterop": true
+    },
+	"include": ["src"],
+	"exclude": ["node_modules", "**/__tests__/*"]
+}