From 8a6ec42fa6370ec86ce167ee2e76b65caa7101eb Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Mon, 15 Aug 2022 09:02:25 -0400 Subject: [PATCH] *Add IPFS caching with IndexedDb --- package.json | 1 + src/contentProviders/ipfsProvider.ts | 191 ++++++++++++++++++--------- yarn.lock | 5 + 3 files changed, 131 insertions(+), 66 deletions(-) diff --git a/package.json b/package.json index 3e6761f..7140076 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "@lumeweb/kernel-dht-client": "https://github.com/LumeWeb/kernel-dht-client.git", "@lumeweb/kernel-ipfs-client": "https://github.com/LumeWeb/kernel-ipfs-client.git", "@lumeweb/tld-enum": "https://github.com/LumeWeb/list-of-top-level-domains.git", + "dexie": "^3.2.2", "ejs": "^3.1.8", "is-ipfs": "^6.0.2", "libkernel": "https://github.com/LumeWeb/libextension.git", diff --git a/src/contentProviders/ipfsProvider.ts b/src/contentProviders/ipfsProvider.ts index 3ab5a66..dca0d11 100644 --- a/src/contentProviders/ipfsProvider.ts +++ b/src/contentProviders/ipfsProvider.ts @@ -22,6 +22,7 @@ import { CONTENT_MODE_CHUNKED, contentModes, } from "../mimes.js"; +import Dexie from "dexie"; const INDEX_HTML_FILES = ["index.html", "index.htm", "index.shtml"]; @@ -86,6 +87,14 @@ interface StatFileSubfile { size: number; } +const cacheDb = new Dexie("LumeWebIFSCache"); + +cacheDb.version(1).stores({ + items: `url,contentType,data,timestamp`, +}); + +const MAX_CACHE_SIZE = 1024 * 1024 * 1024 * 50; + export default class IpfsProvider extends BaseProvider { async shouldHandleRequest( details: OnBeforeRequestDetailsType @@ -127,37 +136,46 @@ export default class IpfsProvider extends BaseProvider { let resp: StatFileResponse | null = null; let fetchMethod: typeof fetchIpfs | typeof fetchIpns; let err; - + let contentType: string; if (urlObj.protocol == "https") { urlObj.protocol = "http"; return { redirectUrl: urlObj.toString() }; } - try { - if (ipfsPath(hash)) { - hash = hash.replace("/ipfs/", ""); - resp = await statIpfs(hash.replace("/ipfs/", ""), urlPath); - fetchMethod = fetchIpfs; - } else if (ipnsPath(hash)) { - hash = hash.replace("/ipns/", ""); - resp = await statIpns(hash.replace("/ipns/", ""), urlPath); - fetchMethod = fetchIpns; - } else { - err = "invalid content"; - } - } catch (e: any) { - err = (e as Error).message; - } + let cachedPage: { contentType: string; data: Blob } | null = null; - let contentType = resp?.contentType; - let contentLength = resp?.size; + try { + // @ts-ignore + cachedPage = await cacheDb.items.where("url").equals(details.url).first(); + } catch {} + + if (!cachedPage) { + try { + if (ipfsPath(hash)) { + hash = hash.replace("/ipfs/", ""); + resp = await statIpfs(hash.replace("/ipfs/", ""), urlPath); + fetchMethod = fetchIpfs; + } else if (ipnsPath(hash)) { + hash = hash.replace("/ipns/", ""); + resp = await statIpns(hash.replace("/ipns/", ""), urlPath); + fetchMethod = fetchIpns; + } else { + err = "invalid content"; + } + } catch (e: any) { + err = (e as Error).message; + } + + contentType = resp?.contentType as string; + if (contentType?.includes(";")) { + contentType = contentType?.split(";").shift() as string; + } + } else { + contentType = cachedPage.contentType; + } let status = "200"; - if (contentType?.includes(";")) { - contentType = contentType?.split(";").shift(); - } - if (resp) { if (!resp.exists) { err = "404"; @@ -188,7 +206,6 @@ export default class IpfsProvider extends BaseProvider { } this.setData(details, "status", status); - this.setData(details, "contentLength", contentLength); let filterPromiseResolve: any; let filterPromise = new Promise((resolve) => { @@ -203,9 +220,13 @@ export default class IpfsProvider extends BaseProvider { filterPromiseResolve(); }; - const buffer: Uint8Array[] = []; + let buffer: Uint8Array[] = []; + let cacheBuffer: Uint8Array[] | Uint8Array = []; const receiveUpdate = (chunk: Uint8Array) => { + if (!chunk.length && !chunk.byteLength) { + return filterPromise; + } if ( Object.keys(contentModes).includes(contentType as string) && [CONTENT_MODE_CHUNKED, CONTENT_MODE_BUFFERED].includes( @@ -213,12 +234,20 @@ export default class IpfsProvider extends BaseProvider { ) ) { buffer.push(chunk); - return; + resp = resp as StatFileResponse; + cacheBuffer = cacheBuffer as Uint8Array[]; + if (!cachedPage && resp.size <= MAX_CACHE_SIZE) { + cacheBuffer.push(chunk); + } + + return filterPromise; } - filterPromise.then(() => { + return filterPromise.then(() => { streamPromise = streamPromise.then(() => { filter.write(chunk); + cacheBuffer = cacheBuffer as Uint8Array[]; + cacheBuffer.push(chunk); }); }); }; @@ -238,62 +267,92 @@ export default class IpfsProvider extends BaseProvider { urlPath += `/${indexFiles[0].name}`; } else { const renderedDirectory = DIRECTORY_TEMPLATE(resp?.files); - contentLength = renderedDirectory.length; receiveUpdate(new TextEncoder().encode(renderedDirectory)); filterPromise .then(() => streamPromise) .then(() => { filter.close(); }); - this.setData(details, "contentLength", contentLength); return {}; } } + + const handleBuffer = () => { + if (buffer.length) { + let mode = contentModes[contentType as string]; + buffer = buffer.map((item: Uint8Array | ArrayBuffer) => { + if (item instanceof ArrayBuffer) { + return new Uint8Array(item); + } + return item; + }); + if (mode === CONTENT_MODE_BUFFERED) { + let data: string | Uint8Array = Uint8Array.from( + buffer.reduce( + (previousValue: Uint8Array, currentValue: Uint8Array) => { + return Uint8Array.from([...previousValue, ...currentValue]); + } + ) + ); + /* if (contentType === "text/html") { + data = new TextDecoder("utf-8", { fatal: true }).decode(data); + let htmlDoc = new DOMParser().parseFromString( + data as string, + contentType + ); + let found = htmlDoc.documentElement.querySelectorAll( + 'meta[http-equiv="Content-Security-Policy"]' + ); + + if (found.length) { + found.forEach((item) => item.remove()); + data = htmlDoc.documentElement.outerHTML; + } + + data = new TextEncoder().encode(data); + }*/ + filter.write(data); + } else if (mode == CONTENT_MODE_CHUNKED) { + buffer.forEach((data) => filter.write(data)); + } + } + }; + if (cachedPage) { + // @ts-ignore + cachedPage.data.arrayBuffer().then((data: ArrayBuffer) => { + // @ts-ignore + receiveUpdate(data)?.then(() => { + handleBuffer(); + filterPromise.then(() => filter.close()); + }); + }); + + return {}; + } + // @ts-ignore fetchMethod?.(hash, urlPath, receiveUpdate) .then(() => streamPromise) .then(() => { - if (buffer.length) { - let mode = contentModes[contentType as string]; - if (mode === CONTENT_MODE_BUFFERED) { - let data: string | Uint8Array = Uint8Array.from( - buffer.reduce( - (previousValue: Uint8Array, currentValue: Uint8Array) => { - return Uint8Array.from([...previousValue, ...currentValue]); - } - ) - ); - if (contentType === "application/javascript") { - data = new TextDecoder("utf-8", { fatal: true }).decode(data); - data = data.replace( - /\/\/#\s*sourceMappingURL=([^\.]+)\.js.map/, - "" - ); - data = new TextEncoder().encode(data); - } - /* if (contentType === "text/html") { - data = new TextDecoder("utf-8", { fatal: true }).decode(data); - let htmlDoc = new DOMParser().parseFromString( - data as string, - contentType - ); - let found = htmlDoc.documentElement.querySelectorAll( - 'meta[http-equiv="Content-Security-Policy"]' - ); - - if (found.length) { - found.forEach((item) => item.remove()); - data = htmlDoc.documentElement.outerHTML; + handleBuffer(); + resp = resp as StatFileResponse; + if (resp.size <= MAX_CACHE_SIZE) { + cacheBuffer = Uint8Array.from( + (cacheBuffer as Uint8Array[]).reduce( + (previousValue: Uint8Array, currentValue: Uint8Array) => { + return Uint8Array.from([...previousValue, ...currentValue]); } + ) + ); - data = new TextEncoder().encode(data); - }*/ - filter.write(data); - } else if (mode == CONTENT_MODE_CHUNKED) { - buffer.forEach((data) => filter.write(data)); - } + // @ts-ignore + return cacheDb.items.put({ + url: details.url, + contentType, + data: new Blob([cacheBuffer.buffer], { type: contentType }), + timestamp: Date.now(), + }); } - filter.close(); }) .catch((e) => { console.error("page error", urlPath, e.message); diff --git a/yarn.lock b/yarn.lock index a831aeb..4ceb611 100644 --- a/yarn.lock +++ b/yarn.lock @@ -598,6 +598,11 @@ define-properties@^1.1.3, define-properties@^1.1.4: has-property-descriptors "^1.0.0" object-keys "^1.1.1" +dexie@^3.2.2: + version "3.2.2" + resolved "https://registry.yarnpkg.com/dexie/-/dexie-3.2.2.tgz#fa6f2a3c0d6ed0766f8d97a03720056f88fe0e01" + integrity sha512-q5dC3HPmir2DERlX+toCBbHQXW5MsyrFqPFcovkH9N2S/UW/H3H5AWAB6iEOExeraAu+j+zRDG+zg/D7YhH0qg== + dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f"