*Add IPFS caching with IndexedDb

package.json

@@ -41,6 +41,7 @@
 		"@lumeweb/kernel-dht-client": "https://github.com/LumeWeb/kernel-dht-client.git",
 		"@lumeweb/kernel-ipfs-client": "https://github.com/LumeWeb/kernel-ipfs-client.git",
 		"@lumeweb/tld-enum": "https://github.com/LumeWeb/list-of-top-level-domains.git",
+		"dexie": "^3.2.2",
 		"ejs": "^3.1.8",
 		"is-ipfs": "^6.0.2",
 		"libkernel": "https://github.com/LumeWeb/libextension.git",

src/contentProviders/ipfsProvider.ts

@@ -22,6 +22,7 @@ import {
   CONTENT_MODE_CHUNKED,
   contentModes,
 } from "../mimes.js";
+import Dexie from "dexie";
 
 const INDEX_HTML_FILES = ["index.html", "index.htm", "index.shtml"];
 
@@ -86,6 +87,14 @@ interface StatFileSubfile {
   size: number;
 }
 
+const cacheDb = new Dexie("LumeWebIFSCache");
+
+cacheDb.version(1).stores({
+  items: `url,contentType,data,timestamp`,
+});
+
+const MAX_CACHE_SIZE = 1024 * 1024 * 1024 * 50;
+
 export default class IpfsProvider extends BaseProvider {
   async shouldHandleRequest(
     details: OnBeforeRequestDetailsType
@@ -127,12 +136,20 @@ export default class IpfsProvider extends BaseProvider {
     let resp: StatFileResponse | null = null;
     let fetchMethod: typeof fetchIpfs | typeof fetchIpns;
     let err;
-
+    let contentType: string;
     if (urlObj.protocol == "https") {
       urlObj.protocol = "http";
       return { redirectUrl: urlObj.toString() };
     }
 
+    let cachedPage: { contentType: string; data: Blob } | null = null;
+
+    try {
+      // @ts-ignore
+      cachedPage = await cacheDb.items.where("url").equals(details.url).first();
+    } catch {}
+
+    if (!cachedPage) {
       try {
         if (ipfsPath(hash)) {
           hash = hash.replace("/ipfs/", "");
@@ -149,15 +166,16 @@ export default class IpfsProvider extends BaseProvider {
         err = (e as Error).message;
       }
 
-    let contentType = resp?.contentType;
+      contentType = resp?.contentType as string;
-    let contentLength = resp?.size;
+      if (contentType?.includes(";")) {
+        contentType = contentType?.split(";").shift() as string;
+      }
+    } else {
+      contentType = cachedPage.contentType;
+    }
 
     let status = "200";
 
-    if (contentType?.includes(";")) {
-      contentType = contentType?.split(";").shift();
-    }
-
     if (resp) {
       if (!resp.exists) {
         err = "404";
@@ -188,7 +206,6 @@ export default class IpfsProvider extends BaseProvider {
     }
 
     this.setData(details, "status", status);
-    this.setData(details, "contentLength", contentLength);
 
     let filterPromiseResolve: any;
     let filterPromise = new Promise((resolve) => {
@@ -203,9 +220,13 @@ export default class IpfsProvider extends BaseProvider {
       filterPromiseResolve();
     };
 
-    const buffer: Uint8Array[] = [];
+    let buffer: Uint8Array[] = [];
+    let cacheBuffer: Uint8Array[] | Uint8Array = [];
 
     const receiveUpdate = (chunk: Uint8Array) => {
+      if (!chunk.length && !chunk.byteLength) {
+        return filterPromise;
+      }
       if (
         Object.keys(contentModes).includes(contentType as string) &&
         [CONTENT_MODE_CHUNKED, CONTENT_MODE_BUFFERED].includes(
@@ -213,12 +234,20 @@ export default class IpfsProvider extends BaseProvider {
         )
       ) {
         buffer.push(chunk);
-        return;
+        resp = resp as StatFileResponse;
+        cacheBuffer = cacheBuffer as Uint8Array[];
+        if (!cachedPage && resp.size <= MAX_CACHE_SIZE) {
+          cacheBuffer.push(chunk);
         }
 
-      filterPromise.then(() => {
+        return filterPromise;
+      }
+
+      return filterPromise.then(() => {
         streamPromise = streamPromise.then(() => {
           filter.write(chunk);
+          cacheBuffer = cacheBuffer as Uint8Array[];
+          cacheBuffer.push(chunk);
         });
       });
     };
@@ -238,23 +267,25 @@ export default class IpfsProvider extends BaseProvider {
         urlPath += `/${indexFiles[0].name}`;
       } else {
         const renderedDirectory = DIRECTORY_TEMPLATE(resp?.files);
-        contentLength = renderedDirectory.length;
         receiveUpdate(new TextEncoder().encode(renderedDirectory));
         filterPromise
           .then(() => streamPromise)
           .then(() => {
             filter.close();
           });
-        this.setData(details, "contentLength", contentLength);
         return {};
       }
     }
-    // @ts-ignore
+
-    fetchMethod?.(hash, urlPath, receiveUpdate)
+    const handleBuffer = () => {
-      .then(() => streamPromise)
-      .then(() => {
       if (buffer.length) {
         let mode = contentModes[contentType as string];
+        buffer = buffer.map((item: Uint8Array | ArrayBuffer) => {
+          if (item instanceof ArrayBuffer) {
+            return new Uint8Array(item);
+          }
+          return item;
+        });
         if (mode === CONTENT_MODE_BUFFERED) {
           let data: string | Uint8Array = Uint8Array.from(
             buffer.reduce(
@@ -263,14 +294,6 @@ export default class IpfsProvider extends BaseProvider {
               }
             )
           );
-            if (contentType === "application/javascript") {
-              data = new TextDecoder("utf-8", { fatal: true }).decode(data);
-              data = data.replace(
-                /\/\/#\s*sourceMappingURL=([^\.]+)\.js.map/,
-                ""
-              );
-              data = new TextEncoder().encode(data);
-            }
           /*            if (contentType === "text/html") {
                   data = new TextDecoder("utf-8", { fatal: true }).decode(data);
                   let htmlDoc = new DOMParser().parseFromString(
@@ -293,7 +316,43 @@ export default class IpfsProvider extends BaseProvider {
           buffer.forEach((data) => filter.write(data));
         }
       }
-        filter.close();
+    };
+    if (cachedPage) {
+      // @ts-ignore
+      cachedPage.data.arrayBuffer().then((data: ArrayBuffer) => {
+        // @ts-ignore
+        receiveUpdate(data)?.then(() => {
+          handleBuffer();
+          filterPromise.then(() => filter.close());
+        });
+      });
+
+      return {};
+    }
+
+    // @ts-ignore
+    fetchMethod?.(hash, urlPath, receiveUpdate)
+      .then(() => streamPromise)
+      .then(() => {
+        handleBuffer();
+        resp = resp as StatFileResponse;
+        if (resp.size <= MAX_CACHE_SIZE) {
+          cacheBuffer = Uint8Array.from(
+            (cacheBuffer as Uint8Array[]).reduce(
+              (previousValue: Uint8Array, currentValue: Uint8Array) => {
+                return Uint8Array.from([...previousValue, ...currentValue]);
+              }
+            )
+          );
+
+          // @ts-ignore
+          return cacheDb.items.put({
+            url: details.url,
+            contentType,
+            data: new Blob([cacheBuffer.buffer], { type: contentType }),
+            timestamp: Date.now(),
+          });
+        }
       })
       .catch((e) => {
         console.error("page error", urlPath, e.message);

yarn.lock

@@ -598,6 +598,11 @@ define-properties@^1.1.3, define-properties@^1.1.4:
     has-property-descriptors "^1.0.0"
     object-keys "^1.1.1"
 
+dexie@^3.2.2:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/dexie/-/dexie-3.2.2.tgz#fa6f2a3c0d6ed0766f8d97a03720056f88fe0e01"
+  integrity sha512-q5dC3HPmir2DERlX+toCBbHQXW5MsyrFqPFcovkH9N2S/UW/H3H5AWAB6iEOExeraAu+j+zRDG+zg/D7YhH0qg==
+
 dir-glob@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f"