ipfs

docker-compose.yml

@@ -78,6 +78,7 @@ services:
       - caddy
       - handshake-api
       - dnslink-api
+      - ipfs-api
       - website
 
   website:
@@ -174,3 +175,31 @@ services:
       - 3100
     depends_on:
       - caddy
+
+  ipfs-api:
+    build:
+      context: ./packages/ipfs-api
+      dockerfile: Dockerfile
+    container_name: ipfs-api
+    restart: unless-stopped
+    logging: *default-logging
+    networks:
+      shared:
+        ipv4_address: 10.10.10.58
+    expose:
+      - 3100
+    depends_on:
+      - ipfs-mongo
+
+  ipfs-mongo:
+    image: mongo:4.4.1
+    container_name: ipfs-mongo
+    restart: unless-stopped
+    logging: *default-logging
+    volumes:
+      - ./docker/data/ipfs-mongo/db:/data/db
+    networks:
+      shared:
+        ipv4_address: 10.10.10.59
+    expose:
+      - 27017

docker/nginx/conf.d/server/server.api

@@ -92,6 +92,12 @@ location /hns {
     include /etc/nginx/conf.d/include/location-hns;
 }
 
+location /ipfs {
+    include /etc/nginx/conf.d/include/cors;
+
+    proxy_pass http://ipfs-api:3100;
+}
+
 location /hnsres {
     include /etc/nginx/conf.d/include/cors;
 

packages/ipfs-api/.gitignore

@@ -0,0 +1 @@
+dist

packages/ipfs-api/Dockerfile

@@ -0,0 +1,19 @@
+FROM node:16.9.0-alpine
+
+WORKDIR /usr/app
+
+# copy package configuration and install node modules
+COPY package.json yarn.lock ./
+RUN yarn --frozen-lockfile
+
+# copy over source files
+COPY src src
+COPY tsconfig.json ./
+
+# build application
+ENV NODE_ENV production
+RUN yarn build
+
+EXPOSE 3100
+
+CMD ["yarn", "start"]

packages/ipfs-api/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "ipfs-to-skynet",
+  "version": "1.0.0",
+  "description": "",
+  "main": "src/server.ts",
+  "scripts": {
+    "build": "rimraf dist && tsc",
+    "lint": "tslint --project tsconfig.json",
+    "watch": "tsnodemon -x 'ts-node src/server.ts'",
+    "start": "ts-node src/server.ts"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@skynetlabs/skynet-nodejs": "2.2.0",
+    "axios": "0.21.4",
+    "cors": "2.8.5",
+    "dotenv": "10.0.0",
+    "express": "^4.17.1",
+    "got": "11.8.2",
+    "mime-types": "2.1.32",
+    "mongodb": "4.1.2",
+    "skynet-js": "4.0.13-beta",
+    "tar-fs": "2.1.1"
+  },
+  "devDependencies": {
+    "@types/express": "4.17.13",
+    "@types/got": "9.6.12",
+    "@types/mime-types": "2.1.1",
+    "@types/node": "16.9.2",
+    "rimraf": "3.0.2",
+    "ts-node": "10.2.1",
+    "tslint": "6.1.3",
+    "tsnodemon": "^1.2.2",
+    "typescript": "4.4.3"
+  }
+}

packages/ipfs-api/src/consts.ts

@@ -0,0 +1,12 @@
+export const API_HOSTNAME = process.env.API_HOSTNAME || "0.0.0.0";
+export const API_PORT = process.env.API_PORT || "3100";
+export const UPLOAD_PATH = process.env.UPLOAD_PATH || "/tmp";
+export const IPFS_INFURA_API =
+  process.env.IPFS_INFURA_API || "https://ipfs.infura.io:5001";
+export const IPFS_GATEWAY =
+  process.env.IPFS_GATEWAY || "https://cloudflare-ipfs.com/ipfs/";
+export const SKYNET_PORTAL = process.env.SKYNET_PORTAL || "https://siasky.net";
+export const MONGO_CONNECTIONSTRING =
+  process.env.MONGO_CONNECTIONSTRING ||
+  "mongodb://root:password@ipfs-mongo:27017";
+export const MONGO_DBNAME = process.env.MONGO_DBNAME || "ipfs-to-skynet";

packages/ipfs-api/src/mongodb.ts

@@ -0,0 +1,69 @@
+import {
+  Collection,
+  CreateIndexesOptions,
+  Db,
+  MongoClient,
+  MongoClientOptions,
+} from "mongodb";
+
+export class MongoDB {
+  private db: Db;
+  private client: MongoClient;
+
+  public constructor(
+    private connectionString: string,
+    private databaseName: string
+  ) {}
+
+  public async connect() {
+    const options: MongoClientOptions = {
+      sslValidate: true,
+      keepAlive: true,
+      keepAliveInitialDelay: 1000,
+      connectTimeoutMS: 1000,
+    };
+
+    this.client = await MongoClient.connect(this.connectionString, options);
+    this.db = this.client.db(this.databaseName);
+  }
+
+  public async createCollection<T>(
+    collectionName: string
+  ): Promise<Collection<T>> {
+    return await this.db.createCollection<T>(collectionName);
+  }
+
+  public async dropCollection(collectionName: string): Promise<void> {
+    await this.db.dropCollection(collectionName);
+  }
+
+  public async ensureCollection<T>(
+    collectionName: string
+  ): Promise<Collection<T>> {
+    const collections = await this.db
+      .listCollections({ name: collectionName })
+      .toArray();
+
+    const collection = collections.length
+      ? (this.db.collection(collectionName) as Collection<T>)
+      : await this.createCollection<T>(collectionName);
+
+    return collection;
+  }
+
+  public async ensureIndex(
+    collectionName: string,
+    fieldOrSpec: any,
+    options?: CreateIndexesOptions
+  ): Promise<string> {
+    const collection = await this.ensureCollection(collectionName);
+    const ensured = await collection.createIndex(fieldOrSpec, options);
+    return ensured;
+  }
+
+  public async getCollection<T>(
+    collectionName: string
+  ): Promise<Collection<T>> {
+    return this.ensureCollection<T>(collectionName);
+  }
+}

packages/ipfs-api/src/server.ts

@@ -0,0 +1,138 @@
+import cors from "cors";
+import express, { Request, Response } from "express";
+import { rmdirSync, unlinkSync } from "fs";
+import { extension as toExtension } from "mime-types";
+import { Collection } from "mongodb";
+import {
+  API_HOSTNAME,
+  API_PORT,
+  MONGO_CONNECTIONSTRING,
+  MONGO_DBNAME,
+  UPLOAD_PATH,
+} from "./consts";
+import { MongoDB } from "./mongodb";
+import { IRecord } from "./types";
+import {
+  contentType,
+  download,
+  extractArchive,
+  isDirectory,
+  uploadDirectory,
+  uploadFile,
+} from "./utils";
+
+require("dotenv").config();
+
+(async () => {
+  // create the database connection
+  console.log("creating database connection");
+  const mongo = new MongoDB(MONGO_CONNECTIONSTRING, MONGO_DBNAME);
+  await mongo.connect();
+
+  // ensure the database model (ensureIndex will ensure the collection exists)
+  console.log("ensuring database model");
+  const recordsDB = await mongo.getCollection<IRecord>("records");
+  await mongo.ensureIndex("records", "cid", { unique: true });
+  await recordsDB.deleteMany({});
+
+  // create the server
+  console.log("creating express server");
+  const app = express();
+  app.use(express.json());
+  app.use(cors());
+
+  // create the routes
+  app.get("/migrate/:cid", (req: Request, res: Response) => {
+    return handleGetLink(req, res, recordsDB);
+  });
+
+  // start the server
+  app.listen(parseInt(API_PORT, 10), API_HOSTNAME, () => {
+    console.log(`IPFS to Skynet API listening at ${API_HOSTNAME}:${API_PORT}`);
+  });
+})();
+
+async function handleGetLink(
+  req: Request,
+  res: Response,
+  recordsDB: Collection<IRecord>
+) {
+  try {
+    const { cid } = req.params;
+
+    const record = await recordsDB.findOne({ cid });
+    if (record && record.skylink) {
+      res.status(200).send({ skylink: record.skylink });
+      return;
+    }
+
+    if (record && !record.skylink) {
+      // TODO: we can retry the IPFS download and skynet upload here if
+      // time.Since(createdAt) exceeds some threshold
+      res.status(200).send({ error: "processing" });
+      return;
+    }
+
+    // insert an empty record for the cid
+    await recordsDB.insertOne({ cid, createdAt: new Date(), skylink: "" });
+
+    // reupload the cid and return the skylink
+    const skylink = await reuploadFile(cid, recordsDB);
+    res.status(200).send({ skylink });
+    return;
+  } catch (error) {
+    res.status(500).send(error);
+    return;
+  }
+}
+
+async function reuploadFile(
+  cid: string,
+  recordsDB: Collection<IRecord>
+): Promise<string> {
+  // get the content type
+  const ct = await contentType(cid);
+  const ext = toExtension(ct);
+
+  // find out whether it's a directory
+  const isDir = await isDirectory(cid);
+
+  // upload directory
+  if (isDir) {
+    // download cid as archive
+    const tarPath = `${UPLOAD_PATH}/${cid}.tar`;
+    await download(cid, tarPath, isDir);
+
+    // untar the archive
+    const dirPath = `${UPLOAD_PATH}/${cid}`;
+    await extractArchive(tarPath, dirPath);
+
+    // upload the directory
+    const dirPathExtracted = `${UPLOAD_PATH}/${cid}/${cid}`;
+    const skylink = await uploadDirectory(dirPathExtracted);
+
+    // cleanup files
+    unlinkSync(tarPath);
+    rmdirSync(dirPath, { recursive: true });
+
+    // update record
+    await recordsDB.updateOne({ cid }, { $set: { skylink } });
+
+    return skylink;
+  }
+
+  // download cid as file
+  const filePath = `${UPLOAD_PATH}/${cid}.${ext}`;
+  await download(cid, filePath, isDir);
+
+  // upload the file
+  const skylink = await uploadFile(filePath);
+
+  // cleanup files
+  unlinkSync(filePath);
+
+  // update record
+  await recordsDB.updateOne({ cid }, { $set: { skylink } });
+
+  return skylink;
+}

packages/ipfs-api/src/types.ts

@@ -0,0 +1,5 @@
+export interface IRecord {
+  cid: string;
+  createdAt: Date;
+  skylink: string;
+}

packages/ipfs-api/src/utils.ts

@@ -0,0 +1,82 @@
+import { SkynetClient } from "@skynetlabs/skynet-nodejs";
+import { createReadStream, createWriteStream } from "fs";
+import got from "got";
+import { extract } from "tar-fs";
+import { IPFS_GATEWAY, IPFS_INFURA_API, SKYNET_PORTAL } from "./consts";
+
+const client = new SkynetClient(SKYNET_PORTAL);
+
+export async function contentType(cid: string): Promise<string> {
+  const url = `${IPFS_GATEWAY}/${cid}`;
+  const response = await got.head(url);
+  return response.headers["content-type"];
+}
+
+export async function isDirectory(cid: string): Promise<boolean> {
+  const url = `${IPFS_INFURA_API}/api/v0/object/get?arg=${cid}&encoding=json`;
+  const json = await got.get(url).json();
+  return Boolean(json["Links"].length);
+}
+
+export async function download(
+  cid: string,
+  destination: string,
+  directory: boolean
+): Promise<boolean> {
+  const url = directory
+    ? `${IPFS_INFURA_API}/api/v0/get?arg=${cid}&archive=true`
+    : `${IPFS_GATEWAY}/${cid}`;
+
+  console.log(url);
+
+  return new Promise((resolve, reject) => {
+    const downloadStream = got.stream(url);
+    downloadStream.on("error", (error) => {
+      console.error(`Download failed: ${error.message}`);
+    });
+
+    const fileWriterStream = createWriteStream(destination);
+    fileWriterStream
+      .on("error", (error) => {
+        console.error(`Could not write file to system: ${error.message}`);
+        reject(error);
+      })
+      .on("finish", () => {
+        console.log(`File downloaded to ${destination}`);
+        resolve(true);
+      });
+
+    downloadStream.pipe(fileWriterStream);
+  });
+}
+
+export async function extractArchive(src: string, dst: string) {
+  return new Promise((resolve, reject) => {
+    createReadStream(src)
+      .pipe(extract(dst))
+      .on("finish", resolve)
+      .on("error", reject);
+  });
+}
+
+export async function sleep(ms: number) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
+export async function uploadFile(filePath: string): Promise<string> {
+  const response = await client.uploadFile(filePath);
+  if (response.startsWith("sia://")) {
+    return response.slice("sia://".length);
+  }
+  throw new Error("upload failed, skylink not found");
+}
+
+export async function uploadDirectory(dirPath: string): Promise<string> {
+  const response = await client.uploadDirectory(dirPath);
+  if (response.startsWith("sia://")) {
+    return response.slice("sia://".length);
+  }
+  throw new Error("upload failed, skylink not found");
+}

packages/ipfs-api/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "module": "CommonJS",
+    "target": "es5",
+    "noImplicitReturns": true,
+    "noUnusedLocals": true,
+    "outDir": "dist",
+    "sourceMap": true,
+    "strict": false,
+    "lib": ["es2020"],
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "typeRoots": ["./types", "./node_modules/@types"]
+  },
+  "include": ["./src"],
+  "exclude": ["./node_modules", "./dist"],
+  "compileOnSave": true
+}