import { json, LoaderFunction, ActionFunction, ActionFunctionArgs, } from "@remix-run/node"; import { S5Client } from "@lumeweb/s5-js"; import xml2js from "xml2js"; import { prisma } from "@/lib/prisma"; import slugify from "slugify"; import path from "path"; import { getAvailableSites } from "@/utils.js"; import { CID } from "@lumeweb/libs5"; // Action function for POST requests export async function action({ request }: ActionFunctionArgs) { const client = new S5Client("https://s5.web3portal.com"); const data = await request.json(); const site = data.site; const sites = getAvailableSites(); if (!(site in sites)) { throw new Response("Site does not exist", { status: 404 }); } const siteInfo = sites[site]; const meta = (await client.getMetadata(data.cid as string)) as any; const fileMeta = meta.metadata as any; const paths = fileMeta.paths as { [file: string]: { cid: string; }; }; // Check if the RSS feed path exists in the paths if (!(siteInfo.rss in paths)) { throw new Response("RSS feed not found", { status: 404 }); } // Download and parse the RSS feed const rssData = await client.downloadData(paths[siteInfo.rss].cid); const rss = await xml2js.parseStringPromise(rssData); // Process each item in the RSS feed for (const item of rss.rss.channel[0].item) { const url = item.link[0]; const title = item.title[0]; // Title is directly available from the feed let pathname = new URL(url).pathname; // Normalize and remove leading and trailing slashes from the path pathname = path.normalize(pathname).replace(/^\/|\/$/g, ""); // Function to determine if a URL path represents a directory const isDirectory = (pathname: string) => { return !paths.hasOwnProperty(pathname); }; // Check if the path is a directory and look for a directory index if (isDirectory(pathname)) { for (const file of fileMeta.tryFiles) { const indexPath = path.join(pathname, file); if (paths.hasOwnProperty(indexPath)) { pathname = indexPath; break; } } } const cid = paths[pathname]?.cid; if (cid) { const exists = await prisma.article.findUnique({ where: { cid }, }); if (!exists) { const record = { title, url, cid: CID.decode(cid).toString(), createdAt: new Date(), updatedAt: new Date(), slug: slugify(new URL(url).pathname), siteKey: data.site, }; // Insert a new record into the database await prisma.article.create({ data: record, }); } } } return new Response("", { status: 200 }); }