*Large IPFS/IPNS refactor

assets/manifest.json

@@ -2,7 +2,7 @@
     "description": "Lume Web is your decentralized gateway into the web3 internet, the web owned and controlled by its users",
     "manifest_version": 2,
     "name": "Lume Web",
-    "version": "0.3.0",
+    "version": "0.3.0.3",
     "homepage_url": "https://lumeweb.com",
     "icons": {
         "48": "icon.png",
@@ -24,7 +24,7 @@
     "content_scripts": [
         {
             "matches": [
-                "http://kernel.skynet/"
+                "http://kernel.lume/"
             ],
             "js": [
                 "bootloader.js"

build.js

@@ -11,12 +11,13 @@ esbuild.buildSync({
     global: "self",
   },
   inject: ["./polyfill.js"],
+  external: ["node:stream"],
 });
 
 esbuild.buildSync({
   entryPoints: ["src/main/bootloader.ts"],
   outfile: "dist/bootloader.js",
-  format: "esm",
+  format: "iife",
   bundle: true,
   legalComments: "external",
   //  minify: true
@@ -27,14 +28,14 @@ esbuild.buildSync({
 esbuild.buildSync({
   entryPoints: ["src/main/bridge.ts"],
   outfile: "dist/bridge.js",
-  format: "esm",
+  format: "iife",
   bundle: true,
   legalComments: "external",
   //  minify: true
   define: {
     global: "self",
   },
-});
+}); /*
 esbuild.buildSync({
     entryPoints: ["src/main/crypto.ts"],
     outfile: "dist/crypto.js",
@@ -58,3 +59,4 @@ esbuild.buildSync({
         global: "window",
     }
 });
+*/

package.json

@@ -15,45 +15,56 @@
 	"author": "David Vorick",
 	"license": "MIT",
 	"devDependencies": {
-		"@lumeweb/kernel-dns-client": "https://github.com/LumeWeb/kernel-dns-client.git",
+		"@helia/unixfs": "^1.2.1",
-		"@lumeweb/webextension-polyfill": "https://github.com/LumeWeb/webextension-polyfill.git",
 		"@rollup/plugin-node-resolve": "^13.3.0",
-		"@types/ejs": "^3.1.1",
+		"@types/ejs": "^3.1.2",
 		"@types/read": "^0.0.29",
-		"@types/webextension-polyfill": "^0.9.0",
+		"@types/webextension-polyfill": "^0.9.2",
-		"@typescript-eslint/eslint-plugin": "^5.18.0",
+		"@typescript-eslint/eslint-plugin": "^5.57.1",
 		"browserify-fs": "^1.0.0",
-		"cpy-cli": "^4.1.0",
+		"cpy-cli": "^4.2.0",
-		"esbuild": "^0.14.51",
+		"esbuild": "^0.14.54",
-		"eslint": "^8.13.0",
+		"eslint": "^8.38.0",
 		"events": "^3.3.0",
 		"path-browserify": "^1.0.1",
-		"prettier": "^2.6.2",
+		"prettier": "^2.8.7",
 		"process": "^0.11.10",
 		"rimraf": "^3.0.2",
-		"rollup": "^2.75.6",
+		"rollup": "^2.79.1",
 		"stream": "^0.0.2",
-		"typescript": "^4.6.3",
+		"typescript": "^4.9.5",
-		"util": "^0.12.4",
+		"util": "^0.12.5",
 		"webextension-polyfill": "^0.9.0"
 	},
 	"dependencies": {
-		"@lumeweb/kernel-dht-client": "https://github.com/LumeWeb/kernel-dht-client.git",
+		"@lumeweb/kernel-dns-client": "git+https://git.lumeweb.com/LumeWeb/kernel-dns-client.git",
-		"@lumeweb/kernel-ipfs-client": "https://github.com/LumeWeb/kernel-ipfs-client.git",
+		"@lumeweb/kernel-ipfs-client": "git+https://git.lumeweb.com/LumeWeb/kernel-ipfs-client.git",
-		"@lumeweb/libresolver": "https://github.com/LumeWeb/libresolver.git",
+		"@lumeweb/kernel-peer-discovery-client": "git+https://git.lumeweb.com/LumeWeb/kernel-peer-discovery-client.git",
-		"@lumeweb/tld-enum": "https://github.com/LumeWeb/list-of-top-level-domains.git",
+		"@lumeweb/kernel-swarm-client": "git+https://git.lumeweb.com/LumeWeb/kernel-swarm-client.git",
-		"@peculiar/webcrypto": "^1.4.0",
+		"@lumeweb/libresolver": "git+https://git.lumeweb.com/LumeWeb/libresolver.git",
-		"dexie": "^3.2.2",
+		"@lumeweb/tld-enum": "git+https://git.lumeweb.com/LumeWeb/list-of-top-level-domains.git",
-		"ejs": "^3.1.8",
+		"@lumeweb/webextension-polyfill": "git+https://git.lumeweb.com/LumeWeb/webextension-polyfill.git",
+		"@peculiar/webcrypto": "^1.4.3",
+		"@siaweb/libweb": "git+https://git.lumeweb.com/LumeWeb/libsiaweb.git",
+		"buffer": "^6.0.3",
+		"file-type": "^18.2.1",
 		"is-ipfs": "^6.0.2",
-		"libkernel": "https://github.com/LumeWeb/libextension.git",
+		"libkernel": "github:LumeWeb/libextension",
 		"libskynet": "^0.0.62",
-		"node-cache": "^5.1.2"
+		"multiformats": "^11.0.2",
+		"node-cache": "^5.1.2",
+		"p-defer": "^4.0.0"
 	},
 	"browser": {
+		"crypto": "crypto-browserify",
+		"fs": "browserify-fs",
 		"libkmodule": false,
 		"path": "path-browserify",
-		"fs": "browserify-fs",
+		"node:buffer": "buffer"
-		"crypto": "crypto-browserify"
+	},
+	"pnpm": {
+		"overrides": {
+			"libkernel": "github:LumeWeb/libextension"
+		}
 	}
 }

src/clients.ts

@@ -0,0 +1,11 @@
+import { createClient as createDnsClient } from "@lumeweb/kernel-dns-client";
+import { createClient as createIpfsClient } from "@lumeweb/kernel-ipfs-client";
+import { createClient as createSwarmClient } from "@lumeweb/kernel-swarm-client";
+import { createClient as createPeerDiscoveryClient } from "@lumeweb/kernel-peer-discovery-client";
+
+const dnsClient = createDnsClient();
+const ipfsClient = createIpfsClient();
+const swarmClient = createSwarmClient();
+const peerDiscoveryClient = createPeerDiscoveryClient();
+
+export { dnsClient, ipfsClient, swarmClient, peerDiscoveryClient };

src/contentProviders/baseProvider.ts

@@ -52,10 +52,6 @@ export default abstract class BaseProvider {
     const originalUrl = new URL(details.url);
     const hostname = normalizeDomain(originalUrl.hostname);
 
-    if (typeof getAuthStatus === "undefined") {
-      debugger;
-    }
-
     if (getAuthStatus().loginComplete !== true) {
       return false;
     }

src/contentProviders/ipfsProvider.ts

@@ -7,86 +7,22 @@ import {
   OnRequestDetailsType,
   StreamFilter,
 } from "../types.js";
-import { getRelayProxies, streamToArray } from "../util.js";
+import { getRelayProxies } from "../util.js";
-import { ipfsPath, ipnsPath, path } from "is-ipfs";
+import { ipfsPath, ipnsPath, path as checkPath } from "is-ipfs";
-import {
+import { createClient } from "@lumeweb/kernel-ipfs-client";
-  fetchIpfs,
-  fetchIpns,
-  statIpfs,
-  statIpns,
-} from "@lumeweb/kernel-ipfs-client";
-import ejs from "ejs";
-import { cacheDb } from "../databases.js";
 import { DNS_RECORD_TYPE, DNSResult } from "@lumeweb/libresolver";
 import RequestStream from "../requestStream.js";
-import ContentFilterRegistry from "../contentFilterRegistry.js";
+import { UnixFSStats } from "@helia/unixfs";
-
+import * as path from "path";
-const INDEX_HTML_FILES = ["index.html", "index.htm", "index.shtml"];
+import { CID } from "multiformats/cid";
-
+import { fileTypeFromBuffer } from "file-type";
-const DIRECTORY_TEMPLATE = ejs.compile(`
+import extToMimes from "../mimes.js";
-<!DOCTYPE html>
+import NodeCache from "node-cache";
-<html>
-<head>
-  <meta charset="utf-8">
-  <title><%= path %></title>
-  <style></style>
-</head>
-<body>
-  <div id="header" class="row">
-    <div class="col-xs-2">
-      <div id="logo" class="ipfs-logo"></div>
-    </div>
-  </div>
-  <br>
-  <div class="col-xs-12">
-    <div class="panel panel-default">
-      <div class="panel-heading">
-        <strong>Index of <%= path %></strong>
-      </div>
-      <table class="table table-striped">
-        <tbody>
-          <tr>
-            <td class="narrow">
-              <div class="ipfs-icon ipfs-_blank">&nbsp;</div>
-            </td>
-            <td class="padding">
-              <a href="<%= parentHref %>">..</a>
-            </td>
-            <td></td>
-          </tr>
-          <% links.forEach(function (link) { %>
-          <tr>
-            <td><div class="ipfs-icon ipfs-_blank">&nbsp;</div></td>
-            <td><a href="<%= link.link %>"><%= link.name %></a></t>
-            <td><%= link.size %></td>
-          </td>
-          </tr>
-          <% }) %>
-        </tbody>
-      </table>
-    </div>
-  </div>
-</body>
-</html>`);
-
-interface StatFileResponse {
-  exists: boolean;
-  contentType: string | null;
-  error: any;
-  directory: boolean;
-  files: StatFileSubfile[];
-  timeout: boolean;
-  size: number;
-}
-
-interface StatFileSubfile {
-  name: string;
-  size: number;
-}
-
-const MAX_CACHE_SIZE = 1024 * 1024 * 1024 * 50;
 
 export default class IpfsProvider extends BaseProvider {
+  private _ipnsCache = new NodeCache({ stdTTL: 60 * 60 * 24 });
+  private _client = createClient();
+
   async shouldHandleRequest(
     details: OnBeforeRequestDetailsType
   ): Promise<boolean> {
@@ -97,17 +33,17 @@ export default class IpfsProvider extends BaseProvider {
     if (!dnsResult) {
       return false;
     }
-
     let contentRecords = (dnsResult as DNSResult).records.map(
-      (item) => "/" + item.value.replace("://", "/").replace(/^\+/, "/")
+      (item: { value: string }) =>
+        "/" + item.value.replace("://", "/").replace(/^\+/, "/")
     );
 
-    contentRecords = contentRecords.filter((item) => path(item));
+    contentRecords = contentRecords.filter((item) => checkPath(item));
     if (!contentRecords.length) {
       return false;
     }
 
-    this.setData(details, "hash", contentRecords.shift());
+    this.setData(details, "cid", contentRecords.shift());
 
     return true;
   }
@@ -131,114 +67,121 @@ export default class IpfsProvider extends BaseProvider {
   ): Promise<BlockingResponse | boolean> {
     let urlObj = new URL(details.url);
     let urlPath = urlObj.pathname;
-    let hash = this.getData(details, "hash");
+    let cid = this.getData(details, "cid");
-    let resp: StatFileResponse | null = null;
-    let fetchMethod: typeof fetchIpfs | typeof fetchIpns;
     let err;
-    let contentType: string;
+    let stat: UnixFSStats | null = null;
-    let contentSize = 0;
+    const parsedPath = path.parse(urlPath);
-
-    let cachedPage: { contentType: string; data: Blob } | null = null;
-
     try {
-      // @ts-ignore
+      if (ipnsPath(cid)) {
-      cachedPage = await cacheDb.items.where("url").equals(details.url).first();
+        const cidHash = cid.replace("/ipns/", "");
-    } catch {}
+        if (this._ipnsCache.has(cidHash)) {
-
+          cid = this._ipnsCache.get(cidHash);
-    if (!cachedPage) {
-      try {
-        if (ipfsPath(hash)) {
-          hash = hash.replace("/ipfs/", "");
-          resp = await statIpfs(hash.replace("/ipfs/", ""), urlPath);
-          fetchMethod = fetchIpfs;
-        } else if (ipnsPath(hash)) {
-          hash = hash.replace("/ipns/", "");
-          resp = await statIpns(hash.replace("/ipns/", ""), urlPath);
-          fetchMethod = fetchIpns;
         } else {
-          err = "invalid content";
+          cid = await this._client.ipns(cidHash);
+          this._ipnsCache.set(cidHash, cid);
         }
-      } catch (e: any) {
+
+        cid = `/ipfs/${cid}`;
+      }
+
+      if (ipfsPath(cid)) {
+        cid = CID.parse(cid.replace("/ipfs/", "")).toV1().toString();
+        stat = await this._client.stat(cid);
+      }
+    } catch (e) {
       err = (e as Error).message;
     }
 
-      contentType = resp?.contentType as string;
+    if (err) {
-      if (contentType?.includes(";")) {
-        contentType = contentType?.split(";").shift() as string;
-      }
-      contentSize = resp?.size as number;
-    } else {
-      contentType = cachedPage.contentType;
-      contentSize = cachedPage.data.size;
-    }
-
-    if (resp) {
-      if (!resp.exists) {
       err = "404";
     }
-      if (resp.directory) {
+
-        contentType = "text/html";
+    if (!err && stat?.type === "directory") {
-      }
+      if (!parsedPath.base.length || !parsedPath.ext.length) {
+        let found = false;
+        for (const indexFile of ["index.html", "index.htm"]) {
+          try {
+            const subPath = path.join(urlPath, indexFile);
+            await this._client.stat(cid, {
+              path: subPath,
+            });
+            urlPath = subPath;
+            found = true;
+            break;
+          } catch {}
         }
 
-    this.setData(details, "contentType", contentType);
+        if (!found) {
-
+          err = "404";
-    const isSmallFile = contentSize <= MAX_CACHE_SIZE;
+        }
-    const reqStream = new RequestStream(
+      }
-      details,
+    }
-      isSmallFile && ContentFilterRegistry.hasFilters(contentType)
+    const reqStream = new RequestStream(details);
-        ? ContentFilterRegistry.filter(contentType)
-        : undefined
-    );
     reqStream.start();
 
     if (err) {
       reqStream.close();
       return {};
     }
-
-    if (cachedPage) {
-      (
-        cachedPage?.data.stream() as unknown as ReadableStream<Uint8Array>
-      ).pipeThrough(reqStream.stream);
-      return {};
-    }
-    if (resp?.directory) {
-      let indexFiles =
-        resp?.files.filter((item) => INDEX_HTML_FILES.includes(item.name)) ||
-        [];
-
-      if (indexFiles.length > 0) {
-        urlPath += `/${indexFiles[0].name}`;
-      }
-    }
-
-    if (isSmallFile) {
-      streamToArray(reqStream.readableStream).then((data: Uint8Array) => {
-        // @ts-ignore
-        return cacheDb.items.put({
-          url: details.url,
-          contentType,
-          data: new Blob([data.buffer], { type: contentType }),
-          timestamp: Date.now(),
-        });
-      });
-    }
-
     const streamWriter = reqStream.stream.writable.getWriter();
+    const reader = await this._client.cat(cid, { path: urlPath });
 
+    const provider = this;
+
+    let streaming = (async function () {
+      let bufferRead = 0;
+      const fileTypeBufferLength = 4100;
+      const mimeBuffer = [];
+      let checkMime = false;
+
+      try {
         // @ts-ignore
-    fetchMethod?.(hash, urlPath, (data: Buffer) => {
+        for await (const chunk of reader.iterable()) {
-      streamWriter.write(data);
+          streamWriter.write(chunk);
-    })
+          if (bufferRead < fileTypeBufferLength) {
-      .then(() => {
+            if (chunk.length >= fileTypeBufferLength) {
-        streamWriter.releaseLock();
+              mimeBuffer.push(chunk.slice(0, fileTypeBufferLength));
-        return reqStream.close();
+              bufferRead += fileTypeBufferLength;
-      })
+            } else {
-      .catch((e: any) => {
+              mimeBuffer.push(chunk);
+              bufferRead += chunk.length;
+            }
+
+            if (bufferRead >= fileTypeBufferLength) {
+              checkMime = true;
+            }
+          } else {
+            checkMime = true;
+          }
+        }
+      } catch (e) {
         streamWriter.releaseLock();
         reqStream.close();
-      });
+        return;
+      }
+
+      if (checkMime) {
+        const mime = await fileTypeFromBuffer(
+          mimeBuffer.reduce((acc, val) => {
+            return new Uint8Array([...acc, ...val]);
+          }, new Uint8Array())
+        );
+
+        if (mime) {
+          provider.setData(details, "contentType", mime.mime);
+        }
+
+        if (!mime) {
+          const ext = path.parse(urlPath).ext.replace(".", "");
+          if (extToMimes.has(ext)) {
+            provider.setData(details, "contentType", extToMimes.get(ext));
+          }
+        }
+      }
+
+      streamWriter.releaseLock();
+      reqStream.close();
+    })();
 
     return {};
   }
@@ -247,7 +190,6 @@ export default class IpfsProvider extends BaseProvider {
     details: OnHeadersReceivedDetailsType
   ): Promise<BlockingResponse | boolean> {
     let headers = [];
-
     headers.push({
       name: "Content-Type",
       value: this.getData(details, "contentType"),

src/contentProviders/skynetProvider.ts

@@ -1,117 +0,0 @@
-import BaseProvider from "./baseProvider.js";
-import {
-  BlockingResponse,
-  OnBeforeRequestDetailsType,
-  OnHeadersReceivedDetailsType,
-  OnRequestDetailsType,
-} from "../types.js";
-import { validSkylink } from "libskynet";
-import { downloadSkylink, getRelayProxies } from "../util.js";
-import browser from "@lumeweb/webextension-polyfill";
-import { DNS_RECORD_TYPE, DNSResult } from "@lumeweb/libresolver";
-
-export default class SkynetProvider extends BaseProvider {
-  async shouldHandleRequest(
-    details: OnBeforeRequestDetailsType
-  ): Promise<boolean> {
-    let dnsResult: DNSResult | boolean | string = await this.resolveDns(
-      details,
-      [DNS_RECORD_TYPE.CONTENT, DNS_RECORD_TYPE.TEXT]
-    );
-    if (!dnsResult) {
-      return false;
-    }
-    let contentRecords = (dnsResult as DNSResult).records
-      .map((item) => {
-        item.value = item.value.replace("sia://", "");
-        return item;
-      })
-      .filter((item) => {
-        try {
-          return validSkylink(item.value);
-        } catch (e) {
-          return false;
-        }
-      });
-    if (!contentRecords.length) {
-      return false;
-    }
-
-    this.setData(details, "hash", contentRecords.shift()?.value);
-
-    return true;
-  }
-
-  async handleProxy(details: OnRequestDetailsType): Promise<any> {
-    return getRelayProxies();
-  }
-
-  async handleRequest(
-    details: OnBeforeRequestDetailsType
-  ): Promise<BlockingResponse | boolean> {
-    const hash = this.getData(details, "hash");
-    let urlObj = new URL(details.url);
-    let path = urlObj.pathname;
-    let fileData: any, err;
-
-    if (urlObj.protocol == "https") {
-      urlObj.protocol = "http";
-      return { redirectUrl: urlObj.toString() };
-    }
-
-    try {
-      [fileData, err] = await downloadSkylink(hash, path);
-    } catch (e: any) {
-      debugger;
-      this.setData(details, "error", (e as Error).message);
-      return {};
-    }
-
-    if (err) {
-      this.setData(details, "error", err);
-      return {};
-    }
-
-    this.setData(details, "headers", fileData.response?.headers);
-
-    const filter = browser.webRequest.filterResponseData(details.requestId);
-
-    filter.ondata = () => {};
-    filter.onstop = () => {
-      fileData.response.arrayBuffer().then((data: any) => {
-        filter.write(data);
-        filter.close();
-      });
-    };
-
-    return true;
-  }
-
-  async handleHeaders(
-    details: OnHeadersReceivedDetailsType
-  ): Promise<BlockingResponse | boolean> {
-    const err = this.getData(details, "error");
-    let headers: Headers = this.getData(details, "headers") as Headers;
-
-    if (err) {
-      return {
-        responseHeaders: [
-          {
-            name: "Status-Code",
-            value: err == "404" ? "404" : "400",
-          },
-          {
-            name: "Content-Type",
-            value: "text/html; charset=utf8",
-          },
-        ],
-      };
-    }
-
-    return {
-      responseHeaders: Array.from(headers).map((item: string[]) => {
-        return { name: item[0], value: item[1] };
-      }),
-    };
-  }
-}

src/contextMenu.ts

@@ -2,20 +2,12 @@ import browser from "@lumeweb/webextension-polyfill";
 import type WebEngine from "./webEngine.js";
 import type { Menus, Tabs } from "./types.js";
 import IpfsProvider from "./contentProviders/ipfsProvider.js";
-import { cacheDb } from "./databases.js";
 
 export default function setup(engine: WebEngine) {
   browser.menus.create({
     title: "Clear Cache",
     id: "clear-cache",
     onclick: async (info: Menus.OnClickData, tab: Tabs.Tab) => {
-      // @ts-ignore
-      await cacheDb.items
-        .where("url")
-        .startsWithIgnoreCase(
-          `http://${new URL(info.pageUrl as string).hostname}`
-        )
-        .delete();
       browser.tabs.reload(tab.id);
     },
   });

src/databases.ts

@@ -1,7 +0,0 @@
-import Dexie from "dexie";
-
-export const cacheDb = new Dexie("LumeWebIFSCache");
-
-cacheDb.version(1).stores({
-  items: `url,contentType,data,timestamp`,
-});

src/dns.ts

@@ -1,16 +1,12 @@
 import NodeCache from "node-cache";
-import {
-  ready as dnsReady,
-  resolve as resolveDns,
-} from "@lumeweb/kernel-dns-client";
 import {
   DNS_RECORD_TYPE,
-  DNSRecord,
   DNSResult,
   ResolverOptions,
 } from "@lumeweb/libresolver";
-import { blake2b, bufToHex, Err } from "libskynet/dist";
+import { blake2b, bufToHex } from "libskynet/dist";
-import { getDnsSetupPromise } from "./main/vars.js";
+import { getDnsSetupDefer } from "./main/vars.js";
+import { dnsClient } from "./clients.js";
 
 const cache = new NodeCache({ stdTTL: 60 });
 
@@ -28,11 +24,11 @@ export async function resolve(
     return cache.get(cacheId) as DNSResult;
   }
 
-  await getDnsSetupPromise();
+  await getDnsSetupDefer().promise;
 
   let res;
   try {
-    res = await resolveDns(domain, options, bypassCache);
+    res = await dnsClient.resolve(domain, options, bypassCache);
   } catch (e: any) {
     return e as Error;
   }

src/main/background.ts

@@ -1,11 +1,9 @@
 import tldEnum from "@lumeweb/tld-enum";
 import WebEngine from "../webEngine.js";
 import InternalProvider from "../contentProviders/internalProvider.js";
-import SkynetProvider from "../contentProviders/skynetProvider.js";
 import ServerProvider from "../contentProviders/serverProvider.js";
-import { init } from "libkernel";
+import { init, kernelLoaded } from "libkernel";
 import IpfsProvider from "../contentProviders/ipfsProvider.js";
-import { ready as dnsReady } from "@lumeweb/kernel-dns-client";
 import {
   addQuery,
   getAuthStatusResolve,
@@ -32,10 +30,19 @@ import {
   setKernelIframe,
   setOpenPort,
   setTimer,
+  getDnsSetupPromise,
+  getDnsSetupDefer,
 } from "./vars.js";
+// @ts-ignore
 import browser from "@lumeweb/webextension-polyfill";
 import setupContextMenus from "../contextMenu.js";
 import { callModule } from "libkernel";
+import {
+  dnsClient,
+  ipfsClient,
+  peerDiscoveryClient,
+  swarmClient,
+} from "../clients.js";
 
 function logLargeObjects() {
   let queriesLen = Object.keys(getQueries()).length;
@@ -49,6 +56,7 @@ function logLargeObjects() {
   setTimer(getTimer() * 1.25);
   setTimeout(logLargeObjects, getTimer());
 }
+
 setTimeout(logLargeObjects, getTimer());
 
 export function queryKernel(query: any): Promise<any> {
@@ -75,6 +83,7 @@ export function queryKernel(query: any): Promise<any> {
     });
   });
 }
+
 function handleKernelMessage(event: MessageEvent) {
   let data = event.data.data;
 
@@ -174,8 +183,10 @@ function handleBridgeMessage(
     });
     data["domain"] = domain;
   }
+
   getKernelIframe().contentWindow!.postMessage(data, "http://kernel.lume");
 }
+
 function bridgeListener(port: any) {
   let portNonce = getPortsNonce();
   increasePortsNonce();
@@ -206,35 +217,49 @@ async function boot() {
 
   const engine = new WebEngine();
   engine.registerContentProvider(new InternalProvider(engine));
-  engine.registerContentProvider(new SkynetProvider(engine));
+
   engine.registerContentProvider(new IpfsProvider(engine));
   engine.registerContentProvider(new ServerProvider(engine));
 
   setKernelIframe(document.createElement("iframe"));
   getKernelIframe().src = "http://kernel.lume";
-  getKernelIframe().onload = init;
-  document.body.appendChild(getKernelIframe());
 
+  await new Promise((resolve) => {
+    getKernelIframe().onload = () => {
+      init().then(resolve);
+    };
+    document.body.appendChild(getKernelIframe());
+  });
+
+  // @ts-ignore
+  window.callModule = callModule;
+  await kernelLoaded();
+  await swarmClient.addRelay(
+    "fd35779a2dcae738308098e8f6702e25c282a52cce972ff2f96bcc50d5043c99"
+  );
+  await peerDiscoveryClient.register(
+    "_AEPtjxDCq3H4nmLLV7-P0L3D_d_Aude4i9O9S498dXcFw"
+  );
+  await ipfsClient.ready();
   setupContextMenus(engine);
-  setDnsSetupPromise(dnsSetup());
+
+  dnsSetup();
+
+  await getDnsSetupDefer().promise;
+
+  console.log("ready");
 }
 
 async function dnsSetup() {
   const resolvers = [
-    "AQBXtVkPDbZ5Qmjl8dzJ0siSYaFcS3XbDZHapxmZCLfwfg", // icann
+    "_B0tpRWWzAf77qfhiRMx1EGTDURht_2V9VsUmMqIzcpW4Q", // ens
-    "AQAI3TbarrXRxWtrb_5XO-gMYg-UsjVAChue5JEoqywbAw", // eip137
+    // "vAMl33T1TusZqZmJl9mlWJCbYm_Lu1TPjE3aSl2ZFHE_yg", // hns
-    "AQD0s0wZNpZCVg_iO96E6Ff66WxGa2CZst_DCYR_DoQPxw", // solana
-    "AQDtYcJGbquAHA-idtZ-gPOlNBgEVeCZtZUtsyL_J5ZiUA", // algorand
-    "AQDkqoCzCR6s5MP_k6Ee9QWfEwaH5-7XleCKFL1CdxExxQ", // avax
-    "AQC7ALr-OtkFT7qZby2BdMMNbTRXHNMGlpV6r96b35Z79Q", // evmos
-    "AQAmQoZLu1DqIiZaRWRpomvMarQ8Uc3kdHJQBo0r-9uYtg", // handshake
   ];
 
   for (const resolver of resolvers) {
-    await callModule(resolver, "register");
+    await dnsClient.registerResolver(resolver);
   }
 
-  await dnsReady();
+  getDnsSetupDefer().resolve();
 }
-
 boot();

src/main/bootloader.ts

@@ -1,28 +1,14 @@
 import {
   addContextToErr,
-  b64ToBuf,
-  bufToHex,
   bufToStr,
-  computeRegistrySignature,
-  defaultPortalList,
-  deriveChildSeed,
-  deriveRegistryEntryID,
-  downloadSkylink,
-  Ed25519Keypair,
-  entryIDToSkylink,
   Err,
   hexToBuf,
-  progressiveFetch,
-  progressiveFetchResult,
-  taggedRegistryEntryKeys,
   objAsString,
-  verifyRegistryWriteResponse,
+} from "@siaweb/libweb";
-} from "libskynet";
 
-var browser: any; // tsc
+declare var browser: any; // tsc
 
-const defaultKernelResolverLink =
+const defaultKernelLink = "RAC1FocOb2bQw6uwjN0AX__MJ8F-h71F5kvIgQPTKo7fQA";
-  "AQDJDoXMJiiEMBxXodQvUV89qtQHsnXWyV1ViQ9M1pMjUg";
 
 document.title = "kernel.lume";
 let header = document.createElement("h1");
@@ -254,143 +240,44 @@ function downloadKernel(
   kernelSkylink: string
 ): Promise<[kernelCode: string, err: Err]> {
   return new Promise((resolve) => {
-    downloadSkylink(kernelSkylink).then(([fileData, err]) => {
+    fetch(`https://web3portal.com/${kernelSkylink}`).then((result) => {
-      if (err === "404") {
+      if (result.status === 404) {
-        resolve(["", err]);
+        resolve(["", result.status.toString()]);
         return;
       }
-
+      if (!result.ok) {
-      if (err !== null) {
+        resolve(["", result.statusText]);
-        resolve([
-          "",
-          addContextToErr(err, "unable to download the default kernel"),
-        ]);
         return;
       }
+      result
+        .blob()
+        .then((blob) => {
+          return blob.arrayBuffer();
+        })
+        .then((data) => {
+          let [kernelCode, errBBTS] = bufToStr(data);
 
-      let [kernelCode, errBBTS] = bufToStr(fileData);
           if (errBBTS !== null) {
             resolve([
               "",
-          addContextToErr(err, "unable to decode the default kernel"),
+              addContextToErr(null, "unable to decode the default kernel"),
             ]);
             return;
           }
+
           resolve([kernelCode, null]);
         });
     });
+  });
 }
 
 function downloadDefaultKernel(): Promise<[kernelCode: string, err: Err]> {
-  return downloadKernel(defaultKernelResolverLink);
+  return downloadKernel(defaultKernelLink);
 }
 
-function setUserKernelAsDefault(keypair: Ed25519Keypair, dataKey: Uint8Array) {
+async function loadKernel() {
-  log(
+  let [kernelCode, err] = await downloadDefaultKernel();
-    "user kernel not found, setting user kernel to " + defaultKernelResolverLink
-  );
 
-  let [defaultKernelSkylink, err64] = b64ToBuf(defaultKernelResolverLink);
-  if (err64 !== null) {
-    log("unable to convert default kernel link to a Uint8Array");
-    return;
-  }
-
-  let [sig, errCRS] = computeRegistrySignature(
-    keypair.secretKey,
-    dataKey,
-    defaultKernelSkylink,
-    0n
-  );
-  if (errCRS !== null) {
-    log(
-      addContextToErr(
-        errCRS,
-        "unable to compute registry signature to set user kernel"
-      )
-    );
-    return;
-  }
-
-  let dataKeyHex = bufToHex(dataKey);
-  let endpoint = "/skynet/registry";
-  let postBody = {
-    publickey: {
-      algorithm: "ed25519",
-      key: Array.from(keypair.publicKey),
-    },
-    datakey: dataKeyHex,
-    revision: 0,
-    data: Array.from(defaultKernelSkylink),
-    signature: Array.from(sig),
-  };
-  let fetchOpts = {
-    method: "post",
-    body: JSON.stringify(postBody),
-  };
-
-  progressiveFetch(
-    endpoint,
-    fetchOpts,
-    defaultPortalList,
-    verifyRegistryWriteResponse
-  ).then((result: progressiveFetchResult) => {
-    if (result.success !== true) {
-      log("unable to update the user kernel registry entry\n", result.logs);
-      return;
-    }
-    log(
-      "successfully updated the user kernel registry entry to the default kernel"
-    );
-  });
-}
-
-function downloadUserKernel(): Promise<[kernelCode: string, err: Err]> {
-  return new Promise((resolve) => {
-    let kernelEntrySeed = deriveChildSeed(userSeed, "userPreferredKernel2");
-
-    let [keypair, dataKey, errTREK] = taggedRegistryEntryKeys(
-      kernelEntrySeed,
-      "user kernel"
-    );
-    if (errTREK !== null) {
-      resolve([
-        "",
-        addContextToErr(errTREK, "unable to create user kernel registry keys"),
-      ]);
-      return;
-    }
-
-    let [entryID, errREID] = deriveRegistryEntryID(keypair.publicKey, dataKey);
-    if (errREID !== null) {
-      resolve([
-        "",
-        addContextToErr(errREID, "unable to derive registry entry id"),
-      ]);
-      return;
-    }
-    let userKernelSkylink = entryIDToSkylink(entryID);
-
-    downloadKernel(userKernelSkylink).then(([kernelCode, err]) => {
-      if (err === "404") {
-        downloadDefaultKernel().then(([defaultCode, errDefault]) => {
-          if (errDefault === null) {
-            setUserKernelAsDefault(keypair, dataKey);
-          }
-          resolve([defaultCode, errDefault]);
-          return;
-        });
-        return;
-      }
-      log("found user kernel, using: " + userKernelSkylink);
-
-      resolve([kernelCode, err]);
-    });
-  });
-}
-
-function loadKernel() {
-  downloadUserKernel().then(([kernelCode, err]) => {
   if (err !== null) {
     let extErr = addContextToErr(err, "unable to download kernel");
     kernelLoaded = extErr;
@@ -415,7 +302,6 @@ function loadKernel() {
     sendAuthUpdate();
     return;
   }
-  });
 }
 
 let loginComplete = false;
@@ -436,9 +322,9 @@ function sendAuthUpdate() {
 }
 sendAuthUpdate();
 
-let userSeed: Uint8Array;
+var userSeed: Uint8Array;
 function checkForLoadKernel() {
-  let userSeedString = window.localStorage.getItem("v1-seed");
+  let userSeedString = window.localStorage.getItem("v1-key");
   if (userSeedString === null) {
     sendAuthUpdate();
     return;

src/main/bridge.ts

@@ -130,6 +130,12 @@ function handleMessage(event: MessageEvent) {
   // Everything else just gets ignored.
 }
 window.addEventListener("message", handleMessage);
+window.addEventListener("message", (event) => {
+  port.postMessage({
+    method: "log1",
+    data: [event.data, event.origin],
+  });
+});
 port.postMessage({
   method: "bridgeLoaded",
 });

src/main/vars.ts

@@ -1,4 +1,5 @@
 import type { DataFn, KernelAuthStatus } from "libskynet";
+import defer, { DeferredPromise } from "p-defer";
 
 export let queriesNonce = 1;
 export let queries: any = {};
@@ -17,7 +18,7 @@ let blockForBridge = new Promise((resolve) => {
   bridgeLoadedResolve = resolve;
 });
 let kernelFrame: HTMLIFrameElement;
-let blockForDnsSetup: Promise<void>;
+let blockForDnsSetup = defer();
 
 export function getAuthStatusKnown() {
   return authStatusKnown;
@@ -95,10 +96,7 @@ export function setKernelIframe(iframe: HTMLIFrameElement) {
   kernelFrame = iframe;
 }
 
-export function setDnsSetupPromise(p: Promise<void>) {
+export function getDnsSetupDefer(): DeferredPromise<any> {
-  blockForDnsSetup = p;
-}
-export function getDnsSetupPromise(): Promise<void> {
   return blockForDnsSetup;
 }
 export function getAuthStatusResolve(): DataFn {

src/mimes.ts

@@ -0,0 +1,10 @@
+const extToMimes = new Map(
+  Object.entries({
+    html: "text/html",
+    xhtml: "application/xhtml+xml",
+    xml: "application/xml",
+  })
+);
+Object.freeze(extToMimes);
+
+export default extToMimes;

src/util.ts

@@ -1,21 +1,8 @@
-import {
+import { defaultPortalList } from "libskynet";
-  addContextToErr,
-  b64ToBuf,
-  defaultPortalList,
-  Err,
-  objAsString,
-  progressiveFetch,
-  progressiveFetchResult,
-  validSkylink,
-  verifyDownloadResponse,
-} from "libskynet";
-import { DHT } from "@lumeweb/kernel-dht-client";
 
 defaultPortalList.unshift("https://web3portal.com");
 defaultPortalList.pop();
 
-const relayDht = new DHT();
-
 export function isIp(ip: string) {
   return /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/.test(
     ip
@@ -33,12 +20,15 @@ export function normalizeDomain(domain: string): string {
 }
 
 export async function getRelayProxies() {
-  let relays: string[] = await relayDht.getRelayServers();
+  //let relays: string[] = await relayDht.getRelayServers();
-  let proxies = [{ type: "http", host: "localhost", port: 25252 }];
+  let proxies = [
-
+    { type: "http", host: "localhost", port: 25252 },
+    { type: "http", host: "web3portal.com", port: 80 },
+  ];
+  /*
   for (const relay of relays) {
     proxies.push({ type: "http", host: new URL(relay).hostname, port: 25252 });
-  }
+  }*/
 
   return proxies;
 }
@@ -46,9 +36,6 @@ export async function getRelayProxies() {
 export const requestProxies = [
   { type: "http", host: "localhost", port: 25252 },
   { type: "http", host: "web3portal.com", port: 80 },
-  { type: "http", host: "siasky.net", port: 80 },
-  { type: "http", host: "skynetfree.net", port: 80 },
-  { type: "http", host: "skynetpro.net", port: 80 },
 ];
 
 export function getTld(hostname: string): string {
@@ -56,80 +43,6 @@ export function getTld(hostname: string): string {
     ? hostname.split(".")[hostname.split(".").length - 1]
     : hostname;
 }
-
-export type FileDataType = {
-  err: null;
-  response: Response | null;
-  fileData: Uint8Array;
-};
-
-export function downloadSkylink(
-  skylink: string,
-  path?: string
-): Promise<[data: FileDataType, err: Err]> {
-  return new Promise((resolve) => {
-    // Get the Uint8Array of the input skylink.
-    let [u8Link, errBTB] = b64ToBuf(skylink);
-    if (errBTB !== null) {
-      resolve([{} as any, addContextToErr(errBTB, "unable to decode skylink")]);
-      return;
-    }
-    if (!validSkylink(u8Link)) {
-      resolve([{} as any, "skylink appears to be invalid"]);
-      return;
-    }
-
-    // Prepare the download call.
-    let endpoint = "/" + skylink;
-    if (path) {
-      endpoint += path;
-    }
-    let fileDataPtr: FileDataType = {
-      fileData: new Uint8Array(0),
-      err: null,
-      response: null,
-    };
-    let verifyFunction = function (response: Response): Promise<Err> {
-      return verifyDownloadResponse(response, u8Link, fileDataPtr);
-    };
-
-    // Perform the download call.
-    progressiveFetch(endpoint, null, defaultPortalList, () =>
-      Promise.resolve(null)
-    ).then((result: progressiveFetchResult) => {
-      // Return an error if the call failed.
-      if (result.success !== true) {
-        // Check for a 404.
-        for (let i = 0; i < result.responsesFailed.length; i++) {
-          if (result.responsesFailed[i].status === 404) {
-            resolve([{} as any, "404"]);
-            return;
-          }
-        }
-
-        // Error is not a 404, return the logs as the error.
-        let err = objAsString(result.logs);
-        resolve([
-          {} as any,
-          addContextToErr(err, "unable to complete download"),
-        ]);
-        return;
-      }
-      // Check if the portal is honest but the download is corrupt.
-      if (fileDataPtr.err !== null) {
-        resolve([
-          {} as any,
-          addContextToErr(fileDataPtr.err, "download is corrupt"),
-        ]);
-        return;
-      }
-
-      fileDataPtr.response = result.response;
-      resolve([fileDataPtr, null]);
-    });
-  });
-}
-
 export async function* iterateStream(
   stream: ReadableStream<any>
 ): AsyncGenerator<Uint8Array> {