*Add IPFS caching with IndexedDb

package.json

@@ -41,6 +41,7 @@
 		"@lumeweb/kernel-dht-client": "https://github.com/LumeWeb/kernel-dht-client.git",
 		"@lumeweb/kernel-ipfs-client": "https://github.com/LumeWeb/kernel-ipfs-client.git",
 		"@lumeweb/tld-enum": "https://github.com/LumeWeb/list-of-top-level-domains.git",
+		"dexie": "^3.2.2",
 		"ejs": "^3.1.8",
 		"is-ipfs": "^6.0.2",
 		"libkernel": "https://github.com/LumeWeb/libextension.git",

src/contentProviders/ipfsProvider.ts

@@ -22,6 +22,7 @@ import {
   CONTENT_MODE_CHUNKED,
   contentModes,
 } from "../mimes.js";
+import Dexie from "dexie";
 
 const INDEX_HTML_FILES = ["index.html", "index.htm", "index.shtml"];
 
@@ -86,6 +87,14 @@ interface StatFileSubfile {
   size: number;
 }
 
+const cacheDb = new Dexie("LumeWebIFSCache");
+
+cacheDb.version(1).stores({
+  items: `url,contentType,data,timestamp`,
+});
+
+const MAX_CACHE_SIZE = 1024 * 1024 * 1024 * 50;
+
 export default class IpfsProvider extends BaseProvider {
   async shouldHandleRequest(
     details: OnBeforeRequestDetailsType
@@ -127,37 +136,46 @@ export default class IpfsProvider extends BaseProvider {
     let resp: StatFileResponse | null = null;
     let fetchMethod: typeof fetchIpfs | typeof fetchIpns;
     let err;
-
+    let contentType: string;
     if (urlObj.protocol == "https") {
       urlObj.protocol = "http";
       return { redirectUrl: urlObj.toString() };
     }
 
-    try {
+    let cachedPage: { contentType: string; data: Blob } | null = null;
-      if (ipfsPath(hash)) {
-        hash = hash.replace("/ipfs/", "");
-        resp = await statIpfs(hash.replace("/ipfs/", ""), urlPath);
-        fetchMethod = fetchIpfs;
-      } else if (ipnsPath(hash)) {
-        hash = hash.replace("/ipns/", "");
-        resp = await statIpns(hash.replace("/ipns/", ""), urlPath);
-        fetchMethod = fetchIpns;
-      } else {
-        err = "invalid content";
-      }
-    } catch (e: any) {
-      err = (e as Error).message;
-    }
 
-    let contentType = resp?.contentType;
+    try {
-    let contentLength = resp?.size;
+      // @ts-ignore
+      cachedPage = await cacheDb.items.where("url").equals(details.url).first();
+    } catch {}
+
+    if (!cachedPage) {
+      try {
+        if (ipfsPath(hash)) {
+          hash = hash.replace("/ipfs/", "");
+          resp = await statIpfs(hash.replace("/ipfs/", ""), urlPath);
+          fetchMethod = fetchIpfs;
+        } else if (ipnsPath(hash)) {
+          hash = hash.replace("/ipns/", "");
+          resp = await statIpns(hash.replace("/ipns/", ""), urlPath);
+          fetchMethod = fetchIpns;
+        } else {
+          err = "invalid content";
+        }
+      } catch (e: any) {
+        err = (e as Error).message;
+      }
+
+      contentType = resp?.contentType as string;
+      if (contentType?.includes(";")) {
+        contentType = contentType?.split(";").shift() as string;
+      }
+    } else {
+      contentType = cachedPage.contentType;
+    }
 
     let status = "200";
 
-    if (contentType?.includes(";")) {
-      contentType = contentType?.split(";").shift();
-    }
-
     if (resp) {
       if (!resp.exists) {
         err = "404";
@@ -188,7 +206,6 @@ export default class IpfsProvider extends BaseProvider {
     }
 
     this.setData(details, "status", status);
-    this.setData(details, "contentLength", contentLength);
 
     let filterPromiseResolve: any;
     let filterPromise = new Promise((resolve) => {
@@ -203,9 +220,13 @@ export default class IpfsProvider extends BaseProvider {
       filterPromiseResolve();
     };
 
-    const buffer: Uint8Array[] = [];
+    let buffer: Uint8Array[] = [];
+    let cacheBuffer: Uint8Array[] | Uint8Array = [];
 
     const receiveUpdate = (chunk: Uint8Array) => {
+      if (!chunk.length && !chunk.byteLength) {
+        return filterPromise;
+      }
       if (
         Object.keys(contentModes).includes(contentType as string) &&
         [CONTENT_MODE_CHUNKED, CONTENT_MODE_BUFFERED].includes(
@@ -213,12 +234,20 @@ export default class IpfsProvider extends BaseProvider {
         )
       ) {
         buffer.push(chunk);
-        return;
+        resp = resp as StatFileResponse;
+        cacheBuffer = cacheBuffer as Uint8Array[];
+        if (!cachedPage && resp.size <= MAX_CACHE_SIZE) {
+          cacheBuffer.push(chunk);
+        }
+
+        return filterPromise;
       }
 
-      filterPromise.then(() => {
+      return filterPromise.then(() => {
         streamPromise = streamPromise.then(() => {
           filter.write(chunk);
+          cacheBuffer = cacheBuffer as Uint8Array[];
+          cacheBuffer.push(chunk);
         });
       });
     };
@@ -238,62 +267,92 @@ export default class IpfsProvider extends BaseProvider {
         urlPath += `/${indexFiles[0].name}`;
       } else {
         const renderedDirectory = DIRECTORY_TEMPLATE(resp?.files);
-        contentLength = renderedDirectory.length;
         receiveUpdate(new TextEncoder().encode(renderedDirectory));
         filterPromise
           .then(() => streamPromise)
           .then(() => {
             filter.close();
           });
-        this.setData(details, "contentLength", contentLength);
         return {};
       }
     }
+
+    const handleBuffer = () => {
+      if (buffer.length) {
+        let mode = contentModes[contentType as string];
+        buffer = buffer.map((item: Uint8Array | ArrayBuffer) => {
+          if (item instanceof ArrayBuffer) {
+            return new Uint8Array(item);
+          }
+          return item;
+        });
+        if (mode === CONTENT_MODE_BUFFERED) {
+          let data: string | Uint8Array = Uint8Array.from(
+            buffer.reduce(
+              (previousValue: Uint8Array, currentValue: Uint8Array) => {
+                return Uint8Array.from([...previousValue, ...currentValue]);
+              }
+            )
+          );
+          /*            if (contentType === "text/html") {
+                  data = new TextDecoder("utf-8", { fatal: true }).decode(data);
+                  let htmlDoc = new DOMParser().parseFromString(
+                    data as string,
+                    contentType
+                  );
+                  let found = htmlDoc.documentElement.querySelectorAll(
+                    'meta[http-equiv="Content-Security-Policy"]'
+                  );
+
+                  if (found.length) {
+                    found.forEach((item) => item.remove());
+                    data = htmlDoc.documentElement.outerHTML;
+                  }
+
+                  data = new TextEncoder().encode(data);
+                }*/
+          filter.write(data);
+        } else if (mode == CONTENT_MODE_CHUNKED) {
+          buffer.forEach((data) => filter.write(data));
+        }
+      }
+    };
+    if (cachedPage) {
+      // @ts-ignore
+      cachedPage.data.arrayBuffer().then((data: ArrayBuffer) => {
+        // @ts-ignore
+        receiveUpdate(data)?.then(() => {
+          handleBuffer();
+          filterPromise.then(() => filter.close());
+        });
+      });
+
+      return {};
+    }
+
     // @ts-ignore
     fetchMethod?.(hash, urlPath, receiveUpdate)
       .then(() => streamPromise)
       .then(() => {
-        if (buffer.length) {
+        handleBuffer();
-          let mode = contentModes[contentType as string];
+        resp = resp as StatFileResponse;
-          if (mode === CONTENT_MODE_BUFFERED) {
+        if (resp.size <= MAX_CACHE_SIZE) {
-            let data: string | Uint8Array = Uint8Array.from(
+          cacheBuffer = Uint8Array.from(
-              buffer.reduce(
+            (cacheBuffer as Uint8Array[]).reduce(
-                (previousValue: Uint8Array, currentValue: Uint8Array) => {
+              (previousValue: Uint8Array, currentValue: Uint8Array) => {
-                  return Uint8Array.from([...previousValue, ...currentValue]);
+                return Uint8Array.from([...previousValue, ...currentValue]);
-                }
-              )
-            );
-            if (contentType === "application/javascript") {
-              data = new TextDecoder("utf-8", { fatal: true }).decode(data);
-              data = data.replace(
-                /\/\/#\s*sourceMappingURL=([^\.]+)\.js.map/,
-                ""
-              );
-              data = new TextEncoder().encode(data);
-            }
-            /*            if (contentType === "text/html") {
-              data = new TextDecoder("utf-8", { fatal: true }).decode(data);
-              let htmlDoc = new DOMParser().parseFromString(
-                data as string,
-                contentType
-              );
-              let found = htmlDoc.documentElement.querySelectorAll(
-                'meta[http-equiv="Content-Security-Policy"]'
-              );
-
-              if (found.length) {
-                found.forEach((item) => item.remove());
-                data = htmlDoc.documentElement.outerHTML;
               }
+            )
+          );
 
-              data = new TextEncoder().encode(data);
+          // @ts-ignore
-            }*/
+          return cacheDb.items.put({
-            filter.write(data);
+            url: details.url,
-          } else if (mode == CONTENT_MODE_CHUNKED) {
+            contentType,
-            buffer.forEach((data) => filter.write(data));
+            data: new Blob([cacheBuffer.buffer], { type: contentType }),
-          }
+            timestamp: Date.now(),
+          });
         }
-        filter.close();
       })
       .catch((e) => {
         console.error("page error", urlPath, e.message);

yarn.lock

@@ -598,6 +598,11 @@ define-properties@^1.1.3, define-properties@^1.1.4:
     has-property-descriptors "^1.0.0"
     object-keys "^1.1.1"
 
+dexie@^3.2.2:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/dexie/-/dexie-3.2.2.tgz#fa6f2a3c0d6ed0766f8d97a03720056f88fe0e01"
+  integrity sha512-q5dC3HPmir2DERlX+toCBbHQXW5MsyrFqPFcovkH9N2S/UW/H3H5AWAB6iEOExeraAu+j+zRDG+zg/D7YhH0qg==
+
 dir-glob@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f"