assets/manifest.json

@@ -2,7 +2,7 @@
     "description": "Lume Web is your decentralized gateway into the web3 internet, the web owned and controlled by its users",
     "manifest_version": 2,
     "name": "Lume Web",
-    "version": "0.3.0.3",
+    "version": "0.3.0",
     "homepage_url": "https://lumeweb.com",
     "icons": {
         "48": "icon.png",
@@ -24,7 +24,7 @@
     "content_scripts": [
         {
             "matches": [
-                "http://kernel.lume/"
+                "http://kernel.skynet/"
             ],
             "js": [
                 "bootloader.js"

build.js

@@ -11,13 +11,12 @@ esbuild.buildSync({
     global: "self",
   },
   inject: ["./polyfill.js"],
-  external: ["node:stream"],
 });
 
 esbuild.buildSync({
   entryPoints: ["src/main/bootloader.ts"],
   outfile: "dist/bootloader.js",
-  format: "iife",
+  format: "esm",
   bundle: true,
   legalComments: "external",
   //  minify: true
@@ -28,14 +27,14 @@ esbuild.buildSync({
 esbuild.buildSync({
   entryPoints: ["src/main/bridge.ts"],
   outfile: "dist/bridge.js",
-  format: "iife",
+  format: "esm",
   bundle: true,
   legalComments: "external",
   //  minify: true
   define: {
     global: "self",
   },
-}); /*
+});
 esbuild.buildSync({
     entryPoints: ["src/main/crypto.ts"],
     outfile: "dist/crypto.js",
@@ -59,4 +58,3 @@ esbuild.buildSync({
         global: "window",
     }
 });
-*/

package.json

@@ -15,56 +15,45 @@
 	"author": "David Vorick",
 	"license": "MIT",
 	"devDependencies": {
-		"@helia/unixfs": "^1.2.1",
+		"@lumeweb/kernel-dns-client": "https://github.com/LumeWeb/kernel-dns-client.git",
+		"@lumeweb/webextension-polyfill": "https://github.com/LumeWeb/webextension-polyfill.git",
 		"@rollup/plugin-node-resolve": "^13.3.0",
-		"@types/ejs": "^3.1.2",
+		"@types/ejs": "^3.1.1",
 		"@types/read": "^0.0.29",
-		"@types/webextension-polyfill": "^0.9.2",
+		"@types/webextension-polyfill": "^0.9.0",
-		"@typescript-eslint/eslint-plugin": "^5.57.1",
+		"@typescript-eslint/eslint-plugin": "^5.18.0",
 		"browserify-fs": "^1.0.0",
-		"cpy-cli": "^4.2.0",
+		"cpy-cli": "^4.1.0",
-		"esbuild": "^0.14.54",
+		"esbuild": "^0.14.51",
-		"eslint": "^8.38.0",
+		"eslint": "^8.13.0",
 		"events": "^3.3.0",
 		"path-browserify": "^1.0.1",
-		"prettier": "^2.8.7",
+		"prettier": "^2.6.2",
 		"process": "^0.11.10",
 		"rimraf": "^3.0.2",
-		"rollup": "^2.79.1",
+		"rollup": "^2.75.6",
 		"stream": "^0.0.2",
-		"typescript": "^4.9.5",
+		"typescript": "^4.6.3",
-		"util": "^0.12.5",
+		"util": "^0.12.4",
 		"webextension-polyfill": "^0.9.0"
 	},
 	"dependencies": {
-		"@lumeweb/kernel-dns-client": "git+https://git.lumeweb.com/LumeWeb/kernel-dns-client.git",
+		"@lumeweb/kernel-dht-client": "https://github.com/LumeWeb/kernel-dht-client.git",
-		"@lumeweb/kernel-ipfs-client": "git+https://git.lumeweb.com/LumeWeb/kernel-ipfs-client.git",
+		"@lumeweb/kernel-ipfs-client": "https://github.com/LumeWeb/kernel-ipfs-client.git",
-		"@lumeweb/kernel-peer-discovery-client": "git+https://git.lumeweb.com/LumeWeb/kernel-peer-discovery-client.git",
+		"@lumeweb/libresolver": "https://github.com/LumeWeb/libresolver.git",
-		"@lumeweb/kernel-swarm-client": "git+https://git.lumeweb.com/LumeWeb/kernel-swarm-client.git",
+		"@lumeweb/tld-enum": "https://github.com/LumeWeb/list-of-top-level-domains.git",
-		"@lumeweb/libresolver": "git+https://git.lumeweb.com/LumeWeb/libresolver.git",
+		"@peculiar/webcrypto": "^1.4.0",
-		"@lumeweb/tld-enum": "git+https://git.lumeweb.com/LumeWeb/list-of-top-level-domains.git",
+		"dexie": "^3.2.2",
-		"@lumeweb/webextension-polyfill": "git+https://git.lumeweb.com/LumeWeb/webextension-polyfill.git",
+		"ejs": "^3.1.8",
-		"@peculiar/webcrypto": "^1.4.3",
-		"@siaweb/libweb": "git+https://git.lumeweb.com/LumeWeb/libsiaweb.git",
-		"buffer": "^6.0.3",
-		"file-type": "^18.2.1",
 		"is-ipfs": "^6.0.2",
-		"libkernel": "github:LumeWeb/libextension",
+		"libkernel": "https://github.com/LumeWeb/libextension.git",
 		"libskynet": "^0.0.62",
-		"multiformats": "^11.0.2",
+		"node-cache": "^5.1.2"
-		"node-cache": "^5.1.2",
-		"p-defer": "^4.0.0"
 	},
 	"browser": {
-		"crypto": "crypto-browserify",
-		"fs": "browserify-fs",
 		"libkmodule": false,
 		"path": "path-browserify",
-		"node:buffer": "buffer"
+		"fs": "browserify-fs",
-	},
+		"crypto": "crypto-browserify"
-	"pnpm": {
-		"overrides": {
-			"libkernel": "github:LumeWeb/libextension"
-		}
 	}
 }

src/clients.ts

@@ -1,11 +0,0 @@
-import { createClient as createDnsClient } from "@lumeweb/kernel-dns-client";
-import { createClient as createIpfsClient } from "@lumeweb/kernel-ipfs-client";
-import { createClient as createSwarmClient } from "@lumeweb/kernel-swarm-client";
-import { createClient as createPeerDiscoveryClient } from "@lumeweb/kernel-peer-discovery-client";
-
-const dnsClient = createDnsClient();
-const ipfsClient = createIpfsClient();
-const swarmClient = createSwarmClient();
-const peerDiscoveryClient = createPeerDiscoveryClient();
-
-export { dnsClient, ipfsClient, swarmClient, peerDiscoveryClient };

src/contentProviders/baseProvider.ts

@@ -52,6 +52,10 @@ export default abstract class BaseProvider {
     const originalUrl = new URL(details.url);
     const hostname = normalizeDomain(originalUrl.hostname);
 
+    if (typeof getAuthStatus === "undefined") {
+      debugger;
+    }
+
     if (getAuthStatus().loginComplete !== true) {
       return false;
     }

src/contentProviders/ipfsProvider.ts

@@ -7,22 +7,86 @@ import {
   OnRequestDetailsType,
   StreamFilter,
 } from "../types.js";
-import { getRelayProxies } from "../util.js";
+import { getRelayProxies, streamToArray } from "../util.js";
-import { ipfsPath, ipnsPath, path as checkPath } from "is-ipfs";
+import { ipfsPath, ipnsPath, path } from "is-ipfs";
-import { createClient } from "@lumeweb/kernel-ipfs-client";
+import {
+  fetchIpfs,
+  fetchIpns,
+  statIpfs,
+  statIpns,
+} from "@lumeweb/kernel-ipfs-client";
+import ejs from "ejs";
+import { cacheDb } from "../databases.js";
 import { DNS_RECORD_TYPE, DNSResult } from "@lumeweb/libresolver";
 import RequestStream from "../requestStream.js";
-import { UnixFSStats } from "@helia/unixfs";
+import ContentFilterRegistry from "../contentFilterRegistry.js";
-import * as path from "path";
+
-import { CID } from "multiformats/cid";
+const INDEX_HTML_FILES = ["index.html", "index.htm", "index.shtml"];
-import { fileTypeFromBuffer } from "file-type";
+
-import extToMimes from "../mimes.js";
+const DIRECTORY_TEMPLATE = ejs.compile(`
-import NodeCache from "node-cache";
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title><%= path %></title>
+  <style></style>
+</head>
+<body>
+  <div id="header" class="row">
+    <div class="col-xs-2">
+      <div id="logo" class="ipfs-logo"></div>
+    </div>
+  </div>
+  <br>
+  <div class="col-xs-12">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <strong>Index of <%= path %></strong>
+      </div>
+      <table class="table table-striped">
+        <tbody>
+          <tr>
+            <td class="narrow">
+              <div class="ipfs-icon ipfs-_blank">&nbsp;</div>
+            </td>
+            <td class="padding">
+              <a href="<%= parentHref %>">..</a>
+            </td>
+            <td></td>
+          </tr>
+          <% links.forEach(function (link) { %>
+          <tr>
+            <td><div class="ipfs-icon ipfs-_blank">&nbsp;</div></td>
+            <td><a href="<%= link.link %>"><%= link.name %></a></t>
+            <td><%= link.size %></td>
+          </td>
+          </tr>
+          <% }) %>
+        </tbody>
+      </table>
+    </div>
+  </div>
+</body>
+</html>`);
+
+interface StatFileResponse {
+  exists: boolean;
+  contentType: string | null;
+  error: any;
+  directory: boolean;
+  files: StatFileSubfile[];
+  timeout: boolean;
+  size: number;
+}
+
+interface StatFileSubfile {
+  name: string;
+  size: number;
+}
+
+const MAX_CACHE_SIZE = 1024 * 1024 * 1024 * 50;
 
 export default class IpfsProvider extends BaseProvider {
-  private _ipnsCache = new NodeCache({ stdTTL: 60 * 60 * 24 });
-  private _client = createClient();
-
   async shouldHandleRequest(
     details: OnBeforeRequestDetailsType
   ): Promise<boolean> {
@@ -33,17 +97,17 @@ export default class IpfsProvider extends BaseProvider {
     if (!dnsResult) {
       return false;
     }
+
     let contentRecords = (dnsResult as DNSResult).records.map(
-      (item: { value: string }) =>
+      (item) => "/" + item.value.replace("://", "/").replace(/^\+/, "/")
-        "/" + item.value.replace("://", "/").replace(/^\+/, "/")
     );
 
-    contentRecords = contentRecords.filter((item) => checkPath(item));
+    contentRecords = contentRecords.filter((item) => path(item));
     if (!contentRecords.length) {
       return false;
     }
 
-    this.setData(details, "cid", contentRecords.shift());
+    this.setData(details, "hash", contentRecords.shift());
 
     return true;
   }
@@ -67,121 +131,114 @@ export default class IpfsProvider extends BaseProvider {
   ): Promise<BlockingResponse | boolean> {
     let urlObj = new URL(details.url);
     let urlPath = urlObj.pathname;
-    let cid = this.getData(details, "cid");
+    let hash = this.getData(details, "hash");
+    let resp: StatFileResponse | null = null;
+    let fetchMethod: typeof fetchIpfs | typeof fetchIpns;
     let err;
-    let stat: UnixFSStats | null = null;
+    let contentType: string;
-    const parsedPath = path.parse(urlPath);
+    let contentSize = 0;
+
+    let cachedPage: { contentType: string; data: Blob } | null = null;
+
     try {
-      if (ipnsPath(cid)) {
+      // @ts-ignore
-        const cidHash = cid.replace("/ipns/", "");
+      cachedPage = await cacheDb.items.where("url").equals(details.url).first();
-        if (this._ipnsCache.has(cidHash)) {
+    } catch {}
-          cid = this._ipnsCache.get(cidHash);
+
+    if (!cachedPage) {
+      try {
+        if (ipfsPath(hash)) {
+          hash = hash.replace("/ipfs/", "");
+          resp = await statIpfs(hash.replace("/ipfs/", ""), urlPath);
+          fetchMethod = fetchIpfs;
+        } else if (ipnsPath(hash)) {
+          hash = hash.replace("/ipns/", "");
+          resp = await statIpns(hash.replace("/ipns/", ""), urlPath);
+          fetchMethod = fetchIpns;
         } else {
-          cid = await this._client.ipns(cidHash);
+          err = "invalid content";
-          this._ipnsCache.set(cidHash, cid);
         }
-
+      } catch (e: any) {
-        cid = `/ipfs/${cid}`;
+        err = (e as Error).message;
       }
 
-      if (ipfsPath(cid)) {
+      contentType = resp?.contentType as string;
-        cid = CID.parse(cid.replace("/ipfs/", "")).toV1().toString();
+      if (contentType?.includes(";")) {
-        stat = await this._client.stat(cid);
+        contentType = contentType?.split(";").shift() as string;
       }
-    } catch (e) {
+      contentSize = resp?.size as number;
-      err = (e as Error).message;
+    } else {
+      contentType = cachedPage.contentType;
+      contentSize = cachedPage.data.size;
     }
 
-    if (err) {
+    if (resp) {
-      err = "404";
+      if (!resp.exists) {
-    }
+        err = "404";
-
+      }
-    if (!err && stat?.type === "directory") {
+      if (resp.directory) {
-      if (!parsedPath.base.length || !parsedPath.ext.length) {
+        contentType = "text/html";
-        let found = false;
-        for (const indexFile of ["index.html", "index.htm"]) {
-          try {
-            const subPath = path.join(urlPath, indexFile);
-            await this._client.stat(cid, {
-              path: subPath,
-            });
-            urlPath = subPath;
-            found = true;
-            break;
-          } catch {}
-        }
-
-        if (!found) {
-          err = "404";
-        }
       }
     }
-    const reqStream = new RequestStream(details);
+
+    this.setData(details, "contentType", contentType);
+
+    const isSmallFile = contentSize <= MAX_CACHE_SIZE;
+    const reqStream = new RequestStream(
+      details,
+      isSmallFile && ContentFilterRegistry.hasFilters(contentType)
+        ? ContentFilterRegistry.filter(contentType)
+        : undefined
+    );
     reqStream.start();
 
     if (err) {
       reqStream.close();
       return {};
     }
-    const streamWriter = reqStream.stream.writable.getWriter();
-    const reader = await this._client.cat(cid, { path: urlPath });
 
-    const provider = this;
+    if (cachedPage) {
+      (
+        cachedPage?.data.stream() as unknown as ReadableStream<Uint8Array>
+      ).pipeThrough(reqStream.stream);
+      return {};
+    }
+    if (resp?.directory) {
+      let indexFiles =
+        resp?.files.filter((item) => INDEX_HTML_FILES.includes(item.name)) ||
+        [];
 
-    let streaming = (async function () {
+      if (indexFiles.length > 0) {
-      let bufferRead = 0;
+        urlPath += `/${indexFiles[0].name}`;
-      const fileTypeBufferLength = 4100;
+      }
-      const mimeBuffer = [];
+    }
-      let checkMime = false;
 
-      try {
+    if (isSmallFile) {
+      streamToArray(reqStream.readableStream).then((data: Uint8Array) => {
         // @ts-ignore
-        for await (const chunk of reader.iterable()) {
+        return cacheDb.items.put({
-          streamWriter.write(chunk);
+          url: details.url,
-          if (bufferRead < fileTypeBufferLength) {
+          contentType,
-            if (chunk.length >= fileTypeBufferLength) {
+          data: new Blob([data.buffer], { type: contentType }),
-              mimeBuffer.push(chunk.slice(0, fileTypeBufferLength));
+          timestamp: Date.now(),
-              bufferRead += fileTypeBufferLength;
+        });
-            } else {
+      });
-              mimeBuffer.push(chunk);
+    }
-              bufferRead += chunk.length;
-            }
 
-            if (bufferRead >= fileTypeBufferLength) {
+    const streamWriter = reqStream.stream.writable.getWriter();
-              checkMime = true;
+
-            }
+    // @ts-ignore
-          } else {
+    fetchMethod?.(hash, urlPath, (data: Buffer) => {
-            checkMime = true;
+      streamWriter.write(data);
-          }
+    })
-        }
+      .then(() => {
-      } catch (e) {
+        streamWriter.releaseLock();
+        return reqStream.close();
+      })
+      .catch((e: any) => {
         streamWriter.releaseLock();
         reqStream.close();
-        return;
+      });
-      }
-
-      if (checkMime) {
-        const mime = await fileTypeFromBuffer(
-          mimeBuffer.reduce((acc, val) => {
-            return new Uint8Array([...acc, ...val]);
-          }, new Uint8Array())
-        );
-
-        if (mime) {
-          provider.setData(details, "contentType", mime.mime);
-        }
-
-        if (!mime) {
-          const ext = path.parse(urlPath).ext.replace(".", "");
-          if (extToMimes.has(ext)) {
-            provider.setData(details, "contentType", extToMimes.get(ext));
-          }
-        }
-      }
-
-      streamWriter.releaseLock();
-      reqStream.close();
-    })();
 
     return {};
   }
@@ -190,6 +247,7 @@ export default class IpfsProvider extends BaseProvider {
     details: OnHeadersReceivedDetailsType
   ): Promise<BlockingResponse | boolean> {
     let headers = [];
+
     headers.push({
       name: "Content-Type",
       value: this.getData(details, "contentType"),

src/contentProviders/skynetProvider.ts

@@ -0,0 +1,117 @@
+import BaseProvider from "./baseProvider.js";
+import {
+  BlockingResponse,
+  OnBeforeRequestDetailsType,
+  OnHeadersReceivedDetailsType,
+  OnRequestDetailsType,
+} from "../types.js";
+import { validSkylink } from "libskynet";
+import { downloadSkylink, getRelayProxies } from "../util.js";
+import browser from "@lumeweb/webextension-polyfill";
+import { DNS_RECORD_TYPE, DNSResult } from "@lumeweb/libresolver";
+
+export default class SkynetProvider extends BaseProvider {
+  async shouldHandleRequest(
+    details: OnBeforeRequestDetailsType
+  ): Promise<boolean> {
+    let dnsResult: DNSResult | boolean | string = await this.resolveDns(
+      details,
+      [DNS_RECORD_TYPE.CONTENT, DNS_RECORD_TYPE.TEXT]
+    );
+    if (!dnsResult) {
+      return false;
+    }
+    let contentRecords = (dnsResult as DNSResult).records
+      .map((item) => {
+        item.value = item.value.replace("sia://", "");
+        return item;
+      })
+      .filter((item) => {
+        try {
+          return validSkylink(item.value);
+        } catch (e) {
+          return false;
+        }
+      });
+    if (!contentRecords.length) {
+      return false;
+    }
+
+    this.setData(details, "hash", contentRecords.shift()?.value);
+
+    return true;
+  }
+
+  async handleProxy(details: OnRequestDetailsType): Promise<any> {
+    return getRelayProxies();
+  }
+
+  async handleRequest(
+    details: OnBeforeRequestDetailsType
+  ): Promise<BlockingResponse | boolean> {
+    const hash = this.getData(details, "hash");
+    let urlObj = new URL(details.url);
+    let path = urlObj.pathname;
+    let fileData: any, err;
+
+    if (urlObj.protocol == "https") {
+      urlObj.protocol = "http";
+      return { redirectUrl: urlObj.toString() };
+    }
+
+    try {
+      [fileData, err] = await downloadSkylink(hash, path);
+    } catch (e: any) {
+      debugger;
+      this.setData(details, "error", (e as Error).message);
+      return {};
+    }
+
+    if (err) {
+      this.setData(details, "error", err);
+      return {};
+    }
+
+    this.setData(details, "headers", fileData.response?.headers);
+
+    const filter = browser.webRequest.filterResponseData(details.requestId);
+
+    filter.ondata = () => {};
+    filter.onstop = () => {
+      fileData.response.arrayBuffer().then((data: any) => {
+        filter.write(data);
+        filter.close();
+      });
+    };
+
+    return true;
+  }
+
+  async handleHeaders(
+    details: OnHeadersReceivedDetailsType
+  ): Promise<BlockingResponse | boolean> {
+    const err = this.getData(details, "error");
+    let headers: Headers = this.getData(details, "headers") as Headers;
+
+    if (err) {
+      return {
+        responseHeaders: [
+          {
+            name: "Status-Code",
+            value: err == "404" ? "404" : "400",
+          },
+          {
+            name: "Content-Type",
+            value: "text/html; charset=utf8",
+          },
+        ],
+      };
+    }
+
+    return {
+      responseHeaders: Array.from(headers).map((item: string[]) => {
+        return { name: item[0], value: item[1] };
+      }),
+    };
+  }
+}

src/contextMenu.ts

@@ -2,12 +2,20 @@ import browser from "@lumeweb/webextension-polyfill";
 import type WebEngine from "./webEngine.js";
 import type { Menus, Tabs } from "./types.js";
 import IpfsProvider from "./contentProviders/ipfsProvider.js";
+import { cacheDb } from "./databases.js";
 
 export default function setup(engine: WebEngine) {
   browser.menus.create({
     title: "Clear Cache",
     id: "clear-cache",
     onclick: async (info: Menus.OnClickData, tab: Tabs.Tab) => {
+      // @ts-ignore
+      await cacheDb.items
+        .where("url")
+        .startsWithIgnoreCase(
+          `http://${new URL(info.pageUrl as string).hostname}`
+        )
+        .delete();
       browser.tabs.reload(tab.id);
     },
   });

src/databases.ts

@@ -0,0 +1,7 @@
+import Dexie from "dexie";
+
+export const cacheDb = new Dexie("LumeWebIFSCache");
+
+cacheDb.version(1).stores({
+  items: `url,contentType,data,timestamp`,
+});

src/dns.ts

@@ -1,12 +1,16 @@
 import NodeCache from "node-cache";
+import {
+  ready as dnsReady,
+  resolve as resolveDns,
+} from "@lumeweb/kernel-dns-client";
 import {
   DNS_RECORD_TYPE,
+  DNSRecord,
   DNSResult,
   ResolverOptions,
 } from "@lumeweb/libresolver";
-import { blake2b, bufToHex } from "libskynet/dist";
+import { blake2b, bufToHex, Err } from "libskynet/dist";
-import { getDnsSetupDefer } from "./main/vars.js";
+import { getDnsSetupPromise } from "./main/vars.js";
-import { dnsClient } from "./clients.js";
 
 const cache = new NodeCache({ stdTTL: 60 });
 
@@ -24,11 +28,11 @@ export async function resolve(
     return cache.get(cacheId) as DNSResult;
   }
 
-  await getDnsSetupDefer().promise;
+  await getDnsSetupPromise();
 
   let res;
   try {
-    res = await dnsClient.resolve(domain, options, bypassCache);
+    res = await resolveDns(domain, options, bypassCache);
   } catch (e: any) {
     return e as Error;
   }

src/main/background.ts

@@ -1,9 +1,11 @@
 import tldEnum from "@lumeweb/tld-enum";
 import WebEngine from "../webEngine.js";
 import InternalProvider from "../contentProviders/internalProvider.js";
+import SkynetProvider from "../contentProviders/skynetProvider.js";
 import ServerProvider from "../contentProviders/serverProvider.js";
-import { init, kernelLoaded } from "libkernel";
+import { init } from "libkernel";
 import IpfsProvider from "../contentProviders/ipfsProvider.js";
+import { ready as dnsReady } from "@lumeweb/kernel-dns-client";
 import {
   addQuery,
   getAuthStatusResolve,
@@ -30,19 +32,10 @@ import {
   setKernelIframe,
   setOpenPort,
   setTimer,
-  getDnsSetupPromise,
-  getDnsSetupDefer,
 } from "./vars.js";
-// @ts-ignore
 import browser from "@lumeweb/webextension-polyfill";
 import setupContextMenus from "../contextMenu.js";
 import { callModule } from "libkernel";
-import {
-  dnsClient,
-  ipfsClient,
-  peerDiscoveryClient,
-  swarmClient,
-} from "../clients.js";
 
 function logLargeObjects() {
   let queriesLen = Object.keys(getQueries()).length;
@@ -56,7 +49,6 @@ function logLargeObjects() {
   setTimer(getTimer() * 1.25);
   setTimeout(logLargeObjects, getTimer());
 }
-
 setTimeout(logLargeObjects, getTimer());
 
 export function queryKernel(query: any): Promise<any> {
@@ -83,7 +75,6 @@ export function queryKernel(query: any): Promise<any> {
     });
   });
 }
-
 function handleKernelMessage(event: MessageEvent) {
   let data = event.data.data;
 
@@ -183,10 +174,8 @@ function handleBridgeMessage(
     });
     data["domain"] = domain;
   }
-
   getKernelIframe().contentWindow!.postMessage(data, "http://kernel.lume");
 }
-
 function bridgeListener(port: any) {
   let portNonce = getPortsNonce();
   increasePortsNonce();
@@ -217,49 +206,35 @@ async function boot() {
 
   const engine = new WebEngine();
   engine.registerContentProvider(new InternalProvider(engine));
-
+  engine.registerContentProvider(new SkynetProvider(engine));
   engine.registerContentProvider(new IpfsProvider(engine));
   engine.registerContentProvider(new ServerProvider(engine));
 
   setKernelIframe(document.createElement("iframe"));
   getKernelIframe().src = "http://kernel.lume";
+  getKernelIframe().onload = init;
+  document.body.appendChild(getKernelIframe());
 
-  await new Promise((resolve) => {
-    getKernelIframe().onload = () => {
-      init().then(resolve);
-    };
-    document.body.appendChild(getKernelIframe());
-  });
-
-  // @ts-ignore
-  window.callModule = callModule;
-  await kernelLoaded();
-  await swarmClient.addRelay(
-    "fd35779a2dcae738308098e8f6702e25c282a52cce972ff2f96bcc50d5043c99"
-  );
-  await peerDiscoveryClient.register(
-    "_AEPtjxDCq3H4nmLLV7-P0L3D_d_Aude4i9O9S498dXcFw"
-  );
-  await ipfsClient.ready();
   setupContextMenus(engine);
-
+  setDnsSetupPromise(dnsSetup());
-  dnsSetup();
-
-  await getDnsSetupDefer().promise;
-
-  console.log("ready");
 }
 
 async function dnsSetup() {
   const resolvers = [
-    "_B0tpRWWzAf77qfhiRMx1EGTDURht_2V9VsUmMqIzcpW4Q", // ens
+    "AQBXtVkPDbZ5Qmjl8dzJ0siSYaFcS3XbDZHapxmZCLfwfg", // icann
-    // "vAMl33T1TusZqZmJl9mlWJCbYm_Lu1TPjE3aSl2ZFHE_yg", // hns
+    "AQAI3TbarrXRxWtrb_5XO-gMYg-UsjVAChue5JEoqywbAw", // eip137
+    "AQD0s0wZNpZCVg_iO96E6Ff66WxGa2CZst_DCYR_DoQPxw", // solana
+    "AQDtYcJGbquAHA-idtZ-gPOlNBgEVeCZtZUtsyL_J5ZiUA", // algorand
+    "AQDkqoCzCR6s5MP_k6Ee9QWfEwaH5-7XleCKFL1CdxExxQ", // avax
+    "AQC7ALr-OtkFT7qZby2BdMMNbTRXHNMGlpV6r96b35Z79Q", // evmos
+    "AQAmQoZLu1DqIiZaRWRpomvMarQ8Uc3kdHJQBo0r-9uYtg", // handshake
   ];
 
   for (const resolver of resolvers) {
-    await dnsClient.registerResolver(resolver);
+    await callModule(resolver, "register");
   }
 
-  getDnsSetupDefer().resolve();
+  await dnsReady();
 }
+
 boot();

src/main/bootloader.ts

@@ -1,14 +1,28 @@
 import {
   addContextToErr,
+  b64ToBuf,
+  bufToHex,
   bufToStr,
+  computeRegistrySignature,
+  defaultPortalList,
+  deriveChildSeed,
+  deriveRegistryEntryID,
+  downloadSkylink,
+  Ed25519Keypair,
+  entryIDToSkylink,
   Err,
   hexToBuf,
+  progressiveFetch,
+  progressiveFetchResult,
+  taggedRegistryEntryKeys,
   objAsString,
-} from "@siaweb/libweb";
+  verifyRegistryWriteResponse,
+} from "libskynet";
 
-declare var browser: any; // tsc
+var browser: any; // tsc
 
-const defaultKernelLink = "RAC1FocOb2bQw6uwjN0AX__MJ8F-h71F5kvIgQPTKo7fQA";
+const defaultKernelResolverLink =
+  "AQDJDoXMJiiEMBxXodQvUV89qtQHsnXWyV1ViQ9M1pMjUg";
 
 document.title = "kernel.lume";
 let header = document.createElement("h1");
@@ -240,68 +254,168 @@ function downloadKernel(
   kernelSkylink: string
 ): Promise<[kernelCode: string, err: Err]> {
   return new Promise((resolve) => {
-    fetch(`https://web3portal.com/${kernelSkylink}`).then((result) => {
+    downloadSkylink(kernelSkylink).then(([fileData, err]) => {
-      if (result.status === 404) {
+      if (err === "404") {
-        resolve(["", result.status.toString()]);
+        resolve(["", err]);
         return;
       }
-      if (!result.ok) {
+
-        resolve(["", result.statusText]);
+      if (err !== null) {
+        resolve([
+          "",
+          addContextToErr(err, "unable to download the default kernel"),
+        ]);
         return;
       }
-      result
-        .blob()
-        .then((blob) => {
-          return blob.arrayBuffer();
-        })
-        .then((data) => {
-          let [kernelCode, errBBTS] = bufToStr(data);
 
-          if (errBBTS !== null) {
+      let [kernelCode, errBBTS] = bufToStr(fileData);
-            resolve([
+      if (errBBTS !== null) {
-              "",
+        resolve([
-              addContextToErr(null, "unable to decode the default kernel"),
+          "",
-            ]);
+          addContextToErr(err, "unable to decode the default kernel"),
-            return;
+        ]);
-          }
+        return;
-
+      }
-          resolve([kernelCode, null]);
+      resolve([kernelCode, null]);
-        });
     });
   });
 }
 
 function downloadDefaultKernel(): Promise<[kernelCode: string, err: Err]> {
-  return downloadKernel(defaultKernelLink);
+  return downloadKernel(defaultKernelResolverLink);
 }
 
-async function loadKernel() {
+function setUserKernelAsDefault(keypair: Ed25519Keypair, dataKey: Uint8Array) {
-  let [kernelCode, err] = await downloadDefaultKernel();
+  log(
+    "user kernel not found, setting user kernel to " + defaultKernelResolverLink
+  );
 
-  if (err !== null) {
+  let [defaultKernelSkylink, err64] = b64ToBuf(defaultKernelResolverLink);
-    let extErr = addContextToErr(err, "unable to download kernel");
+  if (err64 !== null) {
-    kernelLoaded = extErr;
+    log("unable to convert default kernel link to a Uint8Array");
-    logErr(extErr);
-    sendAuthUpdate();
     return;
   }
 
-  try {
+  let [sig, errCRS] = computeRegistrySignature(
-    eval(kernelCode);
+    keypair.secretKey,
-    kernelLoaded = "success";
+    dataKey,
-    sendAuthUpdate();
+    defaultKernelSkylink,
-    log("kernel successfully loaded");
+    0n
-    return;
+  );
-  } catch (err: any) {
+  if (errCRS !== null) {
-    let extErr = addContextToErr(err, "unable to eval kernel");
+    log(
-    kernelLoaded = extErr;
+      addContextToErr(
-    logErr(extErr);
+        errCRS,
-    logErr(err.toString());
+        "unable to compute registry signature to set user kernel"
-    console.error(extErr);
+      )
-    console.error(err);
+    );
-    sendAuthUpdate();
     return;
   }
+
+  let dataKeyHex = bufToHex(dataKey);
+  let endpoint = "/skynet/registry";
+  let postBody = {
+    publickey: {
+      algorithm: "ed25519",
+      key: Array.from(keypair.publicKey),
+    },
+    datakey: dataKeyHex,
+    revision: 0,
+    data: Array.from(defaultKernelSkylink),
+    signature: Array.from(sig),
+  };
+  let fetchOpts = {
+    method: "post",
+    body: JSON.stringify(postBody),
+  };
+
+  progressiveFetch(
+    endpoint,
+    fetchOpts,
+    defaultPortalList,
+    verifyRegistryWriteResponse
+  ).then((result: progressiveFetchResult) => {
+    if (result.success !== true) {
+      log("unable to update the user kernel registry entry\n", result.logs);
+      return;
+    }
+    log(
+      "successfully updated the user kernel registry entry to the default kernel"
+    );
+  });
+}
+
+function downloadUserKernel(): Promise<[kernelCode: string, err: Err]> {
+  return new Promise((resolve) => {
+    let kernelEntrySeed = deriveChildSeed(userSeed, "userPreferredKernel2");
+
+    let [keypair, dataKey, errTREK] = taggedRegistryEntryKeys(
+      kernelEntrySeed,
+      "user kernel"
+    );
+    if (errTREK !== null) {
+      resolve([
+        "",
+        addContextToErr(errTREK, "unable to create user kernel registry keys"),
+      ]);
+      return;
+    }
+
+    let [entryID, errREID] = deriveRegistryEntryID(keypair.publicKey, dataKey);
+    if (errREID !== null) {
+      resolve([
+        "",
+        addContextToErr(errREID, "unable to derive registry entry id"),
+      ]);
+      return;
+    }
+    let userKernelSkylink = entryIDToSkylink(entryID);
+
+    downloadKernel(userKernelSkylink).then(([kernelCode, err]) => {
+      if (err === "404") {
+        downloadDefaultKernel().then(([defaultCode, errDefault]) => {
+          if (errDefault === null) {
+            setUserKernelAsDefault(keypair, dataKey);
+          }
+          resolve([defaultCode, errDefault]);
+          return;
+        });
+        return;
+      }
+      log("found user kernel, using: " + userKernelSkylink);
+
+      resolve([kernelCode, err]);
+    });
+  });
+}
+
+function loadKernel() {
+  downloadUserKernel().then(([kernelCode, err]) => {
+    if (err !== null) {
+      let extErr = addContextToErr(err, "unable to download kernel");
+      kernelLoaded = extErr;
+      logErr(extErr);
+      sendAuthUpdate();
+      return;
+    }
+
+    try {
+      eval(kernelCode);
+      kernelLoaded = "success";
+      sendAuthUpdate();
+      log("kernel successfully loaded");
+      return;
+    } catch (err: any) {
+      let extErr = addContextToErr(err, "unable to eval kernel");
+      kernelLoaded = extErr;
+      logErr(extErr);
+      logErr(err.toString());
+      console.error(extErr);
+      console.error(err);
+      sendAuthUpdate();
+      return;
+    }
+  });
 }
 
 let loginComplete = false;
@@ -322,9 +436,9 @@ function sendAuthUpdate() {
 }
 sendAuthUpdate();
 
-var userSeed: Uint8Array;
+let userSeed: Uint8Array;
 function checkForLoadKernel() {
-  let userSeedString = window.localStorage.getItem("v1-key");
+  let userSeedString = window.localStorage.getItem("v1-seed");
   if (userSeedString === null) {
     sendAuthUpdate();
     return;

src/main/bridge.ts

@@ -130,12 +130,6 @@ function handleMessage(event: MessageEvent) {
   // Everything else just gets ignored.
 }
 window.addEventListener("message", handleMessage);
-window.addEventListener("message", (event) => {
-  port.postMessage({
-    method: "log1",
-    data: [event.data, event.origin],
-  });
-});
 port.postMessage({
   method: "bridgeLoaded",
 });

src/main/vars.ts

@@ -1,5 +1,4 @@
 import type { DataFn, KernelAuthStatus } from "libskynet";
-import defer, { DeferredPromise } from "p-defer";
 
 export let queriesNonce = 1;
 export let queries: any = {};
@@ -18,7 +17,7 @@ let blockForBridge = new Promise((resolve) => {
   bridgeLoadedResolve = resolve;
 });
 let kernelFrame: HTMLIFrameElement;
-let blockForDnsSetup = defer();
+let blockForDnsSetup: Promise<void>;
 
 export function getAuthStatusKnown() {
   return authStatusKnown;
@@ -96,7 +95,10 @@ export function setKernelIframe(iframe: HTMLIFrameElement) {
   kernelFrame = iframe;
 }
 
-export function getDnsSetupDefer(): DeferredPromise<any> {
+export function setDnsSetupPromise(p: Promise<void>) {
+  blockForDnsSetup = p;
+}
+export function getDnsSetupPromise(): Promise<void> {
   return blockForDnsSetup;
 }
 export function getAuthStatusResolve(): DataFn {

src/mimes.ts

@@ -1,10 +0,0 @@
-const extToMimes = new Map(
-  Object.entries({
-    html: "text/html",
-    xhtml: "application/xhtml+xml",
-    xml: "application/xml",
-  })
-);
-Object.freeze(extToMimes);
-
-export default extToMimes;

src/util.ts

@@ -1,8 +1,21 @@
-import { defaultPortalList } from "libskynet";
+import {
+  addContextToErr,
+  b64ToBuf,
+  defaultPortalList,
+  Err,
+  objAsString,
+  progressiveFetch,
+  progressiveFetchResult,
+  validSkylink,
+  verifyDownloadResponse,
+} from "libskynet";
+import { DHT } from "@lumeweb/kernel-dht-client";
 
 defaultPortalList.unshift("https://web3portal.com");
 defaultPortalList.pop();
 
+const relayDht = new DHT();
+
 export function isIp(ip: string) {
   return /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/.test(
     ip
@@ -20,15 +33,12 @@ export function normalizeDomain(domain: string): string {
 }
 
 export async function getRelayProxies() {
-  //let relays: string[] = await relayDht.getRelayServers();
+  let relays: string[] = await relayDht.getRelayServers();
-  let proxies = [
+  let proxies = [{ type: "http", host: "localhost", port: 25252 }];
-    { type: "http", host: "localhost", port: 25252 },
+
-    { type: "http", host: "web3portal.com", port: 80 },
-  ];
-  /*
   for (const relay of relays) {
     proxies.push({ type: "http", host: new URL(relay).hostname, port: 25252 });
-  }*/
+  }
 
   return proxies;
 }
@@ -36,6 +46,9 @@ export async function getRelayProxies() {
 export const requestProxies = [
   { type: "http", host: "localhost", port: 25252 },
   { type: "http", host: "web3portal.com", port: 80 },
+  { type: "http", host: "siasky.net", port: 80 },
+  { type: "http", host: "skynetfree.net", port: 80 },
+  { type: "http", host: "skynetpro.net", port: 80 },
 ];
 
 export function getTld(hostname: string): string {
@@ -43,6 +56,80 @@ export function getTld(hostname: string): string {
     ? hostname.split(".")[hostname.split(".").length - 1]
     : hostname;
 }
+
+export type FileDataType = {
+  err: null;
+  response: Response | null;
+  fileData: Uint8Array;
+};
+
+export function downloadSkylink(
+  skylink: string,
+  path?: string
+): Promise<[data: FileDataType, err: Err]> {
+  return new Promise((resolve) => {
+    // Get the Uint8Array of the input skylink.
+    let [u8Link, errBTB] = b64ToBuf(skylink);
+    if (errBTB !== null) {
+      resolve([{} as any, addContextToErr(errBTB, "unable to decode skylink")]);
+      return;
+    }
+    if (!validSkylink(u8Link)) {
+      resolve([{} as any, "skylink appears to be invalid"]);
+      return;
+    }
+
+    // Prepare the download call.
+    let endpoint = "/" + skylink;
+    if (path) {
+      endpoint += path;
+    }
+    let fileDataPtr: FileDataType = {
+      fileData: new Uint8Array(0),
+      err: null,
+      response: null,
+    };
+    let verifyFunction = function (response: Response): Promise<Err> {
+      return verifyDownloadResponse(response, u8Link, fileDataPtr);
+    };
+
+    // Perform the download call.
+    progressiveFetch(endpoint, null, defaultPortalList, () =>
+      Promise.resolve(null)
+    ).then((result: progressiveFetchResult) => {
+      // Return an error if the call failed.
+      if (result.success !== true) {
+        // Check for a 404.
+        for (let i = 0; i < result.responsesFailed.length; i++) {
+          if (result.responsesFailed[i].status === 404) {
+            resolve([{} as any, "404"]);
+            return;
+          }
+        }
+
+        // Error is not a 404, return the logs as the error.
+        let err = objAsString(result.logs);
+        resolve([
+          {} as any,
+          addContextToErr(err, "unable to complete download"),
+        ]);
+        return;
+      }
+      // Check if the portal is honest but the download is corrupt.
+      if (fileDataPtr.err !== null) {
+        resolve([
+          {} as any,
+          addContextToErr(fileDataPtr.err, "download is corrupt"),
+        ]);
+        return;
+      }
+
+      fileDataPtr.response = result.response;
+      resolve([fileDataPtr, null]);
+    });
+  });
+}
+
 export async function* iterateStream(
   stream: ReadableStream<any>
 ): AsyncGenerator<Uint8Array> {