LumeWeb
/
kernel-ipfs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

*Initial version

This commit is contained in:
Derrick Hammer 2022-08-05 09:25:55 -04:00
parent 814075400e
commit 909e8be59e
Signed by: pcfreak30
GPG Key ID: C997C339BE476FF2
6 changed files with 730 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
build.js Normal file
View File

@ -0,0 +1,13 @@
import esbuild from "esbuild";
esbuild.buildSync({
entryPoints: ["src/index.ts"],
outfile: "dist/index.js",
format: "esm",
bundle: true,
legalComments: "external",
// minify: true
define: {
global: "self",
}
});

37
package.json Normal file
View File

@ -0,0 +1,37 @@
{
"name": "@lumeweb/kernel-ipfs",
"version": "0.1.0",
"type": "module",
"scripts": {
"test": "jest",
"build-script": "tsc --project tsconfig.build.json && mv dist-build/build.js dist-build/build.mjs",
"compile": "npm run build-script && node build.js",
"build": "npm run compile && node ./dist-build/build.mjs dev"
},
"dependencies": {
"@lumeweb/kernel-dht-client": "https://github.com/LumeWeb/kernel-dht-client.git",
"@lumeweb/kernel-utils": "https://github.com/LumeWeb/kernel-utils.git",
"buffer": "^6.0.3",
"fetch-retry": "^5.0.3",
"is-ipfs": "^6.0.2",
"libkmodule": "^0.2.12",
"libskynet": "^0.0.62",
"msgpackr": "^1.6.2",
"p-queue": "^7.3.0",
"timers-browserify": "^2.0.12"
},
"devDependencies": {
"@types/events": "^3.0.0",
"@types/node": "^18.0.3",
"@types/read": "^0.0.29",
"esbuild": "^0.14.49",
"libskynetnode": "^0.1.3",
"prettier": "^2.7.1",
"read": "^1.0.7",
"tslib": "^2.4.0",
"typescript": "^4.7.4"
},
"browser": {
"timers": "timers-browserify"
}
}

340
src-build/build.ts Normal file
View File

@ -0,0 +1,340 @@
// This is the standard build script for a kernel module.
import * as fs from "fs"
import {
addContextToErr,
b64ToBuf,
bufToHex,
deriveRegistryEntryID,
entryIDToSkylink,
generateSeedPhraseDeterministic,
seedPhraseToSeed,
sha512,
taggedRegistryEntryKeys,
} from "libskynet"
import { generateSeedPhraseRandom, overwriteRegistryEntry, upload } from "libskynetnode"
import read from "read"
// Helper variables to make it easier to return empty values alongside errors.
const nu8 = new Uint8Array(0)
const nkp = {
publicKey: nu8,
secretKey: nu8,
}
// readFile is a wrapper for fs.readFileSync that handles the try-catch for the
// caller.
function readFile(fileName: string): [string, string | null] {
try {
let data = fs.readFileSync(fileName, "utf8")
return [data, null]
} catch (err) {
return ["", "unable to read file: " + JSON.stringify(err)]
}
}
// readFileBinary is a wrapper for fs.readFileSync that handles the try-catch
// for the caller.
function readFileBinary(fileName: string): [Uint8Array, string | null] {
try {
let data = fs.readFileSync(fileName, null)
return [data, null]
} catch (err) {
return [nu8, "unable to read file: " + JSON.stringify(err)]
}
}
// writeFile is a wrapper for fs.writeFileSync which handles the try-catch in a
// non-exception way.
function writeFile(fileName: string, fileData: string): string | null {
try {
fs.writeFileSync(fileName, fileData)
return null
} catch (err) {
return "unable to write file: " + JSON.stringify(err)
}
}
// hardenedSeedPhrase will take a password, harden it with 100,000 iterations
// of hashing, and then turn it into a seed phrase.
function hardenedSeedPhrase(password: string): [string, string | null] {
let pw = password
// Add some hashing iterations to the password to make it stronger.
for (let i = 0; i < 1000000; i++) {
let passU8 = new TextEncoder().encode(password)
let hashIter = sha512(passU8)
password = bufToHex(hashIter)
}
return generateSeedPhraseDeterministic(password)
}
// seedPhraseToRegistryKeys will convert a seed phrase to the set of registry
// keys that govern the registry entry where the module is published.
function seedPhraseToRegistryKeys(seedPhrase: string): [any, Uint8Array, string | null] {
let [seed, errVSP] = seedPhraseToSeed(seedPhrase)
if (errVSP !== null) {
return [nkp, nu8, addContextToErr(errVSP, "unable to compute seed phrase")]
}
let [keypair, datakey, errTREK] = taggedRegistryEntryKeys(seed, "module-build", "module-key")
if (errTREK !== null) {
return [nkp, nu8, addContextToErr(errTREK, "unable to compute registry entry keys")]
}
return [keypair, datakey, null]
}
// seedPhraseToRegistryLink will take a seedPhrase as input and convert it to
// the registry link for the module.
function seedPhraseToRegistryLink(seedPhrase: string): [string, string | null] {
let [keypair, datakey, errSPTRK] = seedPhraseToRegistryKeys(seedPhrase)
if (errSPTRK !== null) {
return ["", addContextToErr(errSPTRK, "unable to compute registry keys")]
}
let [entryID, errDREID] = deriveRegistryEntryID(keypair.publicKey, datakey)
if (errDREID !== null) {
return ["", addContextToErr(errDREID, "unable to compute registry entry id")]
}
let registryLink = entryIDToSkylink(entryID)
return [registryLink, null]
}
// handlePass handles all portions of the script that occur after the password
// has been requested. If no password needs to be requested, handlePass will be
// called with a null input. We need to structure the code this way because the
// password reader is async and we can only access the password when using a
// callback.
function handlePass(password: string) {
try {
// If we are running prod and the seed file does not exist, we
// need to confirm the password and also warn the user to use a
// secure password.
if (!fs.existsSync(seedFile) && process.argv[2] === "prod") {
// The file does not exist, we need to confirm the
// password.
console.log()
console.log("No production entry found for module. Creating new production module...")
console.log("If someone can guess the password, they can push arbitrary changes to your module.")
console.log("Please use a secure password.")
console.log()
read({ prompt: "Confirm Password: ", silent: true }, function (err: any, confirmPassword: string) {
if (err) {
console.error("unable to fetch password:", err)
process.exit(1)
}
if (password !== confirmPassword) {
console.error("passwords do not match")
process.exit(1)
}
password = password + moduleSalt
handlePassConfirm(password)
})
} else {
// If the seed file does exist, or if we are using dev,
// there's no need to confirm the password but we do
// need to pass the logic off to the handlePassConfirm
// callback.
password = password + moduleSalt
handlePassConfirm(password)
}
} catch (err) {
console.error("Unable to read seedFile:", err)
process.exit(1)
}
}
// handlePassConfirm handles the full script after the confirmation password
// has been provided. If not confirmation password is needed, this function
// will be called anyway using the unconfirmed password as input.
function handlePassConfirm(password: string) {
// Create the seedFile if it does not exist. For dev we just save the
// seed to disk outright, because this is a dev build and therefore not
// security sensitive. Also the dev seed does not get pushed to the
// github repo.
//
// For prod, we use the seed to create a new seed (called the shield)
// which allows us to verify that the developer has provided the right
// password when deploying the module. The shield does get pushed to
// the github repo so that the production module is the same on all
// devices.
if (!fs.existsSync(seedFile) && process.argv[2] !== "prod") {
// Generate the seed phrase and write it to the file.
let [seedPhrase, errGSP] = generateSeedPhraseRandom()
if (errGSP !== null) {
console.error("Unable to generate seed phrase:", errGSP)
process.exit(1)
}
let errWF = writeFile(seedFile, seedPhrase)
if (errWF !== null) {
console.error("unable to write file:", errWF)
process.exit(1)
}
} else if (!fs.existsSync(seedFile) && process.argv[2] === "prod") {
// Generate the seed phrase.
let [seedPhrase, errGSP] = hardenedSeedPhrase(password)
if (errGSP !== null) {
console.error("Unable to generate seed phrase:", errGSP)
process.exit(1)
}
let [registryLink, errSPTRL] = seedPhraseToRegistryLink(seedPhrase)
if (errSPTRL !== null) {
console.error("Unable to generate registry link:", errSPTRL)
process.exit(1)
}
// Write the registry link to the file.
let errWF = writeFile(seedFile, registryLink)
if (errWF !== null) {
console.error("unable to write registry link file:", errWF)
process.exit(1)
}
}
// Load or verify the seed. If this is prod, the password is used to
// create and verify the seed. If this is dev, we just load the seed
// with no password.
let seedPhrase: string
let registryLink: string
if (process.argv[2] === "prod") {
// Generate the seed phrase from the password.
let [sp, errGSP] = hardenedSeedPhrase(password)
if (errGSP !== null) {
console.error("Unable to generate seed phrase: ", errGSP)
process.exit(1)
}
let [rl, errSPTRL] = seedPhraseToRegistryLink(sp)
registryLink = rl
if (errSPTRL !== null) {
console.error("Unable to generate registry link:", errSPTRL)
process.exit(1)
}
let [registryLinkVerify, errRF] = readFile(seedFile)
if (errRF !== null) {
console.error("unable to read seedFile")
process.exit(1)
}
registryLinkVerify = registryLinkVerify.replace(/\n$/, "")
if (registryLink !== registryLinkVerify) {
console.error("Incorrect password")
process.exit(1)
}
seedPhrase = sp
} else {
let [sp, errRF] = readFile(seedFile)
if (errRF !== null) {
console.error("unable to read seed phrase for dev command from disk")
process.exit(1)
}
let [rl, errSPTRL] = seedPhraseToRegistryLink(sp)
registryLink = rl
if (errSPTRL !== null) {
console.error("Unable to generate registry link:", errSPTRL)
process.exit(1)
}
// Write the registry link to the module skylink dev file.
let errWF = writeFile("build/module-skylink-dev", registryLink)
if (errWF !== null) {
console.error("unable to write registry link file:", errWF)
process.exit(1)
}
seedPhrase = sp
}
// Upload the module to Skynet.
let [distFile, errRF] = readFileBinary("dist/index.js")
if (errRF !== null) {
console.error("unable to read dist file for module")
process.exit(1)
}
let metadata = {
Filename: "index.js",
}
console.log("Uploading module...")
upload(distFile, metadata)
.then((result) => {
console.log("Updating module's registry entry...")
// Update the v2 skylink.
let [keypair, datakey, errSPTRK] = seedPhraseToRegistryKeys(seedPhrase)
if (errSPTRK !== null) {
return ["", addContextToErr(errSPTRK, "unable to compute registry keys")]
}
let [bufLink, errBTB] = b64ToBuf(result)
if (errBTB !== null) {
return ["", addContextToErr(errBTB, "unable to decode skylink")]
}
overwriteRegistryEntry(keypair, datakey, bufLink)
.then(() => {
console.log("registry entry is updated")
console.log("Immutable Link for Module:", result)
console.log("Resolver Link for Module:", registryLink)
})
.catch((err: any) => {
console.log("unable to update registry entry:", err)
})
})
.catch((err) => {
console.error("unable to upload file", err)
process.exit(1)
})
}
// Add a newline for readability.
console.log()
// Check for a 'dev' or 'prod' input to the script.
if (process.argv.length !== 3) {
console.error("need to provide either 'dev' or 'prod' as an input")
process.exit(1)
}
// Create the build folder if it does not exist.
if (!fs.existsSync("build")) {
fs.mkdirSync("build")
}
// Determine the seed file.
let seedFile: string
if (process.argv[2] === "prod") {
seedFile = "module-skylink"
} else if (process.argv[2] === "dev") {
seedFile = "build/dev-seed"
} else {
console.error("need to provide either 'dev' or 'prod' as an input")
process.exit(1)
}
// If doing a prod deployment, check whether the salt file exists. If it does
// not, create it.
let moduleSalt: string
if (!fs.existsSync(".module-salt")) {
let [ms, errGSPR] = generateSeedPhraseRandom()
if (errGSPR !== null) {
console.error("unable to generate module salt:", errGSPR)
process.exit(1)
}
moduleSalt = ms
let errWF = writeFile(".module-salt", moduleSalt)
if (errWF !== null) {
console.error("unable to write module salt file:", errWF)
process.exit(1)
}
} else {
let [ms, errRF] = readFile(".module-salt")
if (errRF !== null) {
console.error("unable to read moduleSalt")
process.exit(1)
}
ms = ms.replace(/\n$/, "")
moduleSalt = ms
}
// Need to get a password if this is a prod build.
if (process.argv[2] === "prod") {
read({ prompt: "Password: ", silent: true }, function (err: any, password: string) {
if (err) {
console.error("unable to fetch password:", err)
process.exit(1)
}
handlePass(password)
})
} else {
handlePass("")
}

313
src/index.ts Normal file
View File

@ -0,0 +1,313 @@
import { addHandler, handleMessage } from "libkmodule";
import type { ActiveQuery } from "libkmodule";
import PQueue from "p-queue";
import { ipfsPath, ipnsPath } from "is-ipfs";
import { DHT } from "@lumeweb/kernel-dht-client";
import { pack, unpack } from "msgpackr";
import { clearTimeout } from "timers";
onmessage = handleMessage;
interface StatFileResponse {
exists: boolean;
contentType: string | null;
error: any;
directory: boolean;
files: string[];
}
let blockingGatewayUpdate = Promise.resolve();
let activeRelays = [];
let relays = [
"25c2a0a833782d64213c08879b95dd5a60af244b44a058f3a7a70d6722f4bda7",
];
let dht: DHT;
addHandler("presentSeed", handlePresentSeed);
addHandler("refreshGatewayList", handleRefreshGatewayList);
addHandler("statIpfs", handleStatIpfs);
addHandler("fetchIpfs", handleFetchIpfs, { se });
addHandler("statIpns", handleStatIpns);
addHandler("fetchIpns", handleFetchIpns);
let readyPromiseResolve: any;
let readyPromise = new Promise((resolve) => {
readyPromiseResolve = resolve;
});
async function handlePresentSeed(aq: ActiveQuery) {
dht = new DHT(false);
for (const relay of relays) {
await dht.addRelay(relay);
}
await dht.ready();
refreshGatewayList();
aq.respond();
readyPromiseResolve();
}
async function handleRefreshGatewayList(aq: ActiveQuery) {
await readyPromise;
await blockingGatewayUpdate;
await refreshGatewayList();
aq.respond();
}
async function handleStatIpfs(aq: ActiveQuery) {
return handleStat(aq, "stat_ipfs", "ipfs");
}
async function handleFetchIpfs(aq: ActiveQuery) {
return handleFetch(aq, "fetch_ipfs", "ipfs");
}
async function handleStatIpns(aq: ActiveQuery) {
return handleStat(aq, "stat_ipns", "ipfs");
}
async function handleFetchIpns(aq: ActiveQuery) {
return handleStat(aq, "fetch_ipns", "ipns");
}
async function validateInputs(aq: ActiveQuery, type: "ipns" | "ipfs") {
const { hash = null } = aq.callerInput;
const { path = "" } = aq.callerInput;
if (!hash) {
aq.reject("hash missing");
return;
}
if (type === "ipfs" && !ipfsPath(`/ipfs/${hash}`)) {
aq.reject("ipfs hash is invalid");
return;
}
if (type === "ipns" && !ipnsPath(`/ipns/${hash}`)) {
aq.reject("ipns hash is invalid");
return;
}
await readyPromise;
await blockingGatewayUpdate;
return { hash, path };
}
async function handleStat(
aq: ActiveQuery,
method: string,
type: "ipns" | "ipfs"
): Promise<void> {
const valid = await validateInputs(aq, type);
if (!valid) {
return;
}
const { hash, path } = valid;
try {
let resp = (await fetchFromRelays(hash, path, method)) as StatFileResponse;
aq.respond(resp);
} catch (e) {
aq.reject(e);
}
}
async function handleFetch(
aq: ActiveQuery,
method: string,
type: "ipns" | "ipfs"
): Promise<void> {
const valid = await validateInputs(aq, type);
if (!valid) {
return;
}
const { hash, path } = valid;
try {
await fetchFromRelays(hash, path, method, aq.sendUpdate);
aq.respond();
} catch (e) {
aq.reject(e);
}
}
async function fetchFromRelays(
hash: string,
path: string,
method: string,
stream = undefined
) {
let error = new Error("NOT_FOUND");
for (const relay of activeRelays) {
let resp;
try {
resp = await rpcCall(relay, "ipfs", method, stream, {
hash,
path,
});
} catch (e: any) {
if (e instanceof Error) {
error = e;
} else {
error = new Error(e);
}
continue;
}
if (resp) {
return resp;
}
}
throw error;
}
async function relayHasMethods(
methodList: string[],
relay: string
): Promise<boolean> {
let methods = [];
try {
methods = (await rpcCall(relay, "misc", "get_methods")) as [];
} catch (e) {
return false;
}
let has = true;
methodList.forEach((item) => {
if (!methods.includes(item)) {
has = false;
}
});
return has;
}
async function rpcCall(
relay: string,
chain: string,
query: string,
stream?: (data) => void,
data = {}
) {
const socket = await dht.connect(relay);
return new Promise((resolve, reject) => {
let timer: NodeJS.Timeout;
let dataCount = 0;
socket.on("data", (res) => {
clearTimeout(timer as number);
dataCount++;
const response = unpack(res);
if (!response || response.error || (response && response?.data?.error)) {
socket.end();
reject(response?.error || response?.data?.error);
return;
}
if (!stream && 1 === dataCount) {
socket.end();
resolve(response?.data);
return;
}
if (stream) {
stream(response?.data.data);
if (response?.data.done) {
socket.end();
}
}
});
socket.write("rpc");
socket.write(
pack({
query,
chain,
data,
force: true,
})
);
timer = setTimeout(() => {
socket.end();
reject("timeout");
}, 5 * 1000) as NodeJS.Timeout;
});
}
async function processStream(aq: ActiveQuery, response: Response) {
const reader = response.body.getReader();
let aqResp = {
headers: Array.from(response.headers),
status: response.status,
};
while (true) {
let chunk;
try {
chunk = await reader.read();
aq.sendUpdate(chunk.value);
if (chunk.done) {
aq.respond(aqResp);
break;
}
} catch (e) {
aq.respond(aqResp);
break;
}
}
}
async function refreshGatewayList() {
let processResolve: any;
blockingGatewayUpdate = new Promise((resolve) => {
processResolve = resolve;
});
const queue = new PQueue({ concurrency: 10 });
let latencies = [];
relays.forEach((item) => {
queue.add(checkRelayLatency(item, latencies));
});
await queue.onIdle();
activeRelays = latencies
.sort((a: any[], b: any[]) => {
return a[0] - b[0];
})
.map((item: any[]) => item[1]);
processResolve();
}
function checkRelayLatency(relay: string, list: any[]) {
return async () => {
const start = Date.now();
let resp;
try {
resp = await rpcCall(relay, "misc", "ping", null, {});
} catch {
return;
}
if (!resp.pong) {
return;
}
const end = Date.now() - start;
try {
resp = await relayHasMethods(
["stat_ipfs", "stat_ipns", "fetch_ipfs", "fetch_ipns"],
relay
);
if (!resp) {
return;
}
} catch {
return;
}
list.push([end, relay]);
};
}

14
tsconfig.build.json Normal file
View File

@ -0,0 +1,14 @@
{
"compilerOptions": {
"target": "esnext",
"module": "esnext",
"moduleResolution": "node",
"allowSyntheticDefaultImports": true,
"declaration": true,
"outDir": "./dist-build",
"strict": true,
"esModuleInterop": true
},
"include": ["src-build"],
"exclude": ["node_modules", "**/__tests__/*"]
}

13
tsconfig.json Normal file
View File

@ -0,0 +1,13 @@
{
"compilerOptions": {
"target": "esnext",
"declaration": true,
"moduleResolution": "node",
"outDir": "./build",
"strict": true,
"allowSyntheticDefaultImports": true,
"esModuleInterop": true
},
"include": ["src"],
"exclude": ["node_modules", "**/__tests__/*"]
}