digital ocean spaces/aws S3 integratoin

.env.sample

@@ -1,4 +1,12 @@
-NEXTAUTH_SECRET=very_sensitive_secret2
+NEXTAUTH_SECRET=very_sensitive_secret
 DATABASE_URL=postgresql://user:password@localhost:5432/linkwarden
 NEXTAUTH_URL=http://localhost:3000
 PAGINATION_TAKE_COUNT=20
+STORAGE_FOLDER=data
+
+# Linkwarden Cloud specific configs (Ignore - Not applicable for self-hosted version)
+IS_CLOUD_INSTANCE=
+SPACES_KEY=
+SPACES_SECRET=
+SPACES_ENDPOINT=
+SPACES_REGION=

.github/SECURITY.md

@@ -0,0 +1,17 @@
+# Security Policy
+
+## Supported Versions
+
+| Version | Supported |
+| ------- | --------- |
+| 1.x.x   | ✅        |
+
+## Reporting a Vulnerability
+
+First off, we really appreciate the time you spent!
+
+If you found a vulnerability, these are the ways you can reach us:
+
+Email: [hello@linkwarden.app](mailto:hello@daniel31x13.io)
+
+Or you can directly reach me via Twitter: [@daniel31x13](https://twitter.com/Daniel31X13).

.gitignore

@@ -36,3 +36,6 @@ next-env.d.ts
 
 # generated files and folders
 /data
+
+# tests
+/tests

lib/api/archive.ts

@@ -3,15 +3,13 @@ import { prisma } from "@/lib/api/db";
 import puppeteer from "puppeteer-extra";
 import AdblockerPlugin from "puppeteer-extra-plugin-adblocker";
 import StealthPlugin from "puppeteer-extra-plugin-stealth";
-import fs from "fs";
+import createFile from "@/lib/api/storage/createFile";
 
 export default async function archive(
   url: string,
   collectionId: number,
   linkId: number
 ) {
-  const archivePath = `data/archives/${collectionId}/${linkId}`;
-
   const browser = await puppeteer.launch();
 
   try {
@@ -42,11 +40,14 @@ export default async function archive(
         fullPage: true,
       });
 
-      fs.writeFile(archivePath + ".pdf", pdf, function (err) {
-        console.log(err);
+      createFile({
+        data: screenshot,
+        filePath: `archives/${collectionId}/${linkId}.png`,
       });
-      fs.writeFile(archivePath + ".png", screenshot, function (err) {
-        console.log(err);
+
+      createFile({
+        data: pdf,
+        filePath: `archives/${collectionId}/${linkId}.pdf`,
       });
     }
 

lib/api/controllers/collections/deleteCollection.ts

@@ -1,7 +1,7 @@
 import { prisma } from "@/lib/api/db";
 import getPermission from "@/lib/api/getPermission";
 import { Collection, UsersAndCollections } from "@prisma/client";
-import fs from "fs";
+import removeFolder from "@/lib/api/storage/removeFolder";
 
 export default async function deleteCollection(
   collection: { id: number },
@@ -56,13 +56,7 @@ export default async function deleteCollection(
       },
     });
 
-    try {
-      fs.rmdirSync(`data/archives/${collectionId}`, { recursive: true });
-    } catch (error) {
-      console.log(
-        "Collection's archive directory wasn't deleted most likely because it didn't exist..."
-      );
-    }
+    removeFolder({ filePath: `archives/${collectionId}` });
 
     return await prisma.collection.delete({
       where: {

lib/api/controllers/collections/postCollection.ts

@@ -1,6 +1,6 @@
 import { prisma } from "@/lib/api/db";
 import { CollectionIncludingMembersAndLinkCount } from "@/types/global";
-import { existsSync, mkdirSync } from "fs";
+import createFolder from "@/lib/api/storage/createFolder";
 
 export default async function postCollection(
   collection: CollectionIncludingMembersAndLinkCount,
@@ -66,9 +66,7 @@ export default async function postCollection(
     },
   });
 
-  const collectionPath = `data/archives/${newCollection.id}`;
-  if (!existsSync(collectionPath))
-    mkdirSync(collectionPath, { recursive: true });
+  createFolder({ filePath: `archives/${newCollection.id}` });
 
   return { response: newCollection, status: 200 };
 }

lib/api/controllers/links/deleteLink.ts

@@ -1,8 +1,8 @@
 import { prisma } from "@/lib/api/db";
 import { LinkIncludingShortenedCollectionAndTags } from "@/types/global";
-import fs from "fs";
 import { Collection, Link, UsersAndCollections } from "@prisma/client";
 import getPermission from "@/lib/api/getPermission";
+import removeFile from "@/lib/api/storage/removeFile";
 
 export default async function deleteLink(
   link: LinkIncludingShortenedCollectionAndTags,
@@ -33,13 +33,8 @@ export default async function deleteLink(
     },
   });
 
-  fs.unlink(`data/archives/${link.collectionId}/${link.id}.pdf`, (err) => {
-    if (err) console.log(err);
-  });
-
-  fs.unlink(`data/archives/${link.collectionId}/${link.id}.png`, (err) => {
-    if (err) console.log(err);
-  });
+  removeFile({ filePath: `archives/${link.collectionId}/${link.id}.pdf` });
+  removeFile({ filePath: `archives/${link.collectionId}/${link.id}.png` });
 
   return { response: deleteLink, status: 200 };
 }

lib/api/controllers/links/postLink.ts

@@ -1,10 +1,9 @@
 import { prisma } from "@/lib/api/db";
 import { LinkIncludingShortenedCollectionAndTags } from "@/types/global";
-import getTitle from "../../getTitle";
-import archive from "../../archive";
+import getTitle from "@/lib/api/getTitle";
+import archive from "@/lib/api/archive";
 import { Collection, Link, UsersAndCollections } from "@prisma/client";
 import getPermission from "@/lib/api/getPermission";
-import { existsSync, mkdirSync } from "fs";
 
 export default async function postLink(
   link: LinkIncludingShortenedCollectionAndTags,
@@ -84,10 +83,6 @@ export default async function postLink(
     include: { tags: true, collection: true },
   });
 
-  const collectionPath = `data/archives/${newLink.collectionId}`;
-  if (!existsSync(collectionPath))
-    mkdirSync(collectionPath, { recursive: true });
-
   archive(newLink.url, newLink.collectionId, newLink.id);
 
   return { response: newLink, status: 200 };

lib/api/controllers/users/updateUser.ts

@@ -1,8 +1,8 @@
 import { prisma } from "@/lib/api/db";
 import { AccountSettings } from "@/types/global";
-import fs from "fs";
-import path from "path";
 import bcrypt from "bcrypt";
+import removeFile from "@/lib/api/storage/removeFile";
+import createFile from "@/lib/api/storage/createFile";
 
 export default async function updateUser(
   user: AccountSettings,
@@ -43,15 +43,12 @@ export default async function updateUser(
   if (profilePic.startsWith("data:image/jpeg;base64")) {
     if (user.profilePic.length < 1572864) {
       try {
-        const filePath = path.join(
-          process.cwd(),
-          `data/uploads/avatar/${userId}.jpg`
-        );
-
         const base64Data = profilePic.replace(/^data:image\/jpeg;base64,/, "");
 
-        fs.writeFile(filePath, base64Data, "base64", function (err) {
-          console.log(err);
+        await createFile({
+          filePath: `uploads/avatar/${userId}.jpg`,
+          data: base64Data,
+          isBase64: true,
         });
       } catch (err) {
         console.log("Error saving image:", err);
@@ -64,9 +61,7 @@ export default async function updateUser(
       };
     }
   } else if (profilePic == "") {
-    fs.unlink(`data/uploads/avatar/${userId}.jpg`, (err) => {
-      if (err) console.log(err);
-    });
+    removeFile({ filePath: `uploads/avatar/${userId}.jpg` });
   }
 
   // Other settings

lib/api/storage/createFile.ts

@@ -0,0 +1,40 @@
+import { PutObjectCommand, PutObjectCommandInput } from "@aws-sdk/client-s3";
+import fs from "fs";
+import path from "path";
+import s3Client from "./s3Client";
+
+export default async function createFile({
+  filePath,
+  data,
+  isBase64,
+}: {
+  filePath: string;
+  data: Buffer | string;
+  isBase64?: boolean;
+}) {
+  if (s3Client) {
+    const bucketParams: PutObjectCommandInput = {
+      Bucket: process.env.BUCKET_NAME,
+      Key: filePath,
+      Body: isBase64 ? Buffer.from(data as string, "base64") : data,
+    };
+
+    try {
+      await s3Client.send(new PutObjectCommand(bucketParams));
+
+      return true;
+    } catch (err) {
+      console.log("Error", err);
+      return false;
+    }
+  } else {
+    const storagePath = process.env.STORAGE_FOLDER;
+    const creationPath = path.join(process.cwd(), storagePath + "/" + filePath);
+
+    fs.writeFile(creationPath, data, isBase64 ? "base64" : {}, function (err) {
+      if (err) console.log(err);
+    });
+
+    return true;
+  }
+}

lib/api/storage/createFolder.ts

@@ -0,0 +1,14 @@
+import fs from "fs";
+import path from "path";
+import s3Client from "./s3Client";
+
+export default function createFolder({ filePath }: { filePath: string }) {
+  if (s3Client) {
+    // Do nothing, S3 builds files recursively
+  } else {
+    const storagePath = process.env.STORAGE_FOLDER;
+    const creationPath = path.join(process.cwd(), storagePath + "/" + filePath);
+
+    fs.mkdirSync(creationPath, { recursive: true });
+  }
+}

lib/api/storage/readFile.ts

@@ -0,0 +1,74 @@
+import { GetObjectCommand, GetObjectCommandInput } from "@aws-sdk/client-s3";
+import fs from "fs";
+import path from "path";
+import s3Client from "./s3Client";
+
+export default async function readFile({ filePath }: { filePath: string }) {
+  let contentType:
+    | "text/plain"
+    | "image/jpeg"
+    | "image/png"
+    | "application/pdf";
+
+  if (s3Client) {
+    const bucketParams: GetObjectCommandInput = {
+      Bucket: process.env.BUCKET_NAME,
+      Key: filePath,
+    };
+
+    try {
+      const response = await s3Client.send(new GetObjectCommand(bucketParams));
+      const data = await streamToBuffer(response.Body);
+
+      if (filePath.endsWith(".pdf")) {
+        contentType = "application/pdf";
+      } else if (filePath.endsWith(".png")) {
+        contentType = "image/png";
+      } else {
+        // if (filePath.endsWith(".jpg"))
+        contentType = "image/jpeg";
+      }
+
+      return { file: data, contentType };
+    } catch (err) {
+      console.log("Error", err);
+
+      contentType = "text/plain";
+
+      return {
+        file: "File not found, it's possible that the file you're looking for either doesn't exist or hasn't been created yet.",
+        contentType,
+      };
+    }
+  } else {
+    const storagePath = process.env.STORAGE_FOLDER;
+    const creationPath = path.join(process.cwd(), storagePath + "/" + filePath);
+
+    const file = fs.existsSync(creationPath)
+      ? fs.readFileSync(creationPath)
+      : "File not found, it's possible that the file you're looking for either doesn't exist or hasn't been created yet.";
+
+    if (file.toString().startsWith("File not found")) {
+      contentType = "text/plain";
+    } else if (filePath.endsWith(".pdf")) {
+      contentType = "application/pdf";
+    } else if (filePath.endsWith(".png")) {
+      contentType = "image/png";
+    } else {
+      // if (filePath.endsWith(".jpg"))
+      contentType = "image/jpeg";
+    }
+
+    return { file, contentType };
+  }
+}
+
+// Turn the file's body into buffer
+const streamToBuffer = (stream: any) => {
+  const chunks: any = [];
+  return new Promise((resolve, reject) => {
+    stream.on("data", (chunk: any) => chunks.push(Buffer.from(chunk)));
+    stream.on("error", (err: any) => reject(err));
+    stream.on("end", () => resolve(Buffer.concat(chunks)));
+  });
+};

lib/api/storage/removeFile.ts

@@ -0,0 +1,26 @@
+import fs from "fs";
+import path from "path";
+import s3Client from "./s3Client";
+import { PutObjectCommandInput, DeleteObjectCommand } from "@aws-sdk/client-s3";
+
+export default async function removeFile({ filePath }: { filePath: string }) {
+  if (s3Client) {
+    const bucketParams: PutObjectCommandInput = {
+      Bucket: process.env.BUCKET_NAME,
+      Key: filePath,
+    };
+
+    try {
+      await s3Client.send(new DeleteObjectCommand(bucketParams));
+    } catch (err) {
+      console.log("Error", err);
+    }
+  } else {
+    const storagePath = process.env.STORAGE_FOLDER;
+    const creationPath = path.join(process.cwd(), storagePath + "/" + filePath);
+
+    fs.unlink(creationPath, (err) => {
+      if (err) console.log(err);
+    });
+  }
+}

lib/api/storage/removeFolder.ts

@@ -0,0 +1,59 @@
+import fs from "fs";
+import path from "path";
+import s3Client from "./s3Client";
+import {
+  DeleteObjectsCommand,
+  DeleteObjectsCommandInput,
+  ListObjectsCommand,
+} from "@aws-sdk/client-s3";
+
+async function emptyS3Directory(bucket: string, dir: string) {
+  if (s3Client) {
+    const listParams = {
+      Bucket: bucket,
+      Prefix: dir,
+    };
+
+    const deleteParams: DeleteObjectsCommandInput = {
+      Bucket: bucket,
+      Delete: { Objects: [] },
+    };
+
+    const listedObjects = await s3Client.send(
+      new ListObjectsCommand(listParams)
+    );
+
+    if (listedObjects.Contents?.length === 0) return;
+
+    listedObjects.Contents?.forEach(({ Key }) => {
+      deleteParams.Delete?.Objects?.push({ Key });
+    });
+
+    console.log(listedObjects);
+
+    await s3Client.send(new DeleteObjectsCommand(deleteParams));
+
+    if (listedObjects.IsTruncated) await emptyS3Directory(bucket, dir);
+  }
+}
+
+export default async function removeFolder({ filePath }: { filePath: string }) {
+  if (s3Client) {
+    try {
+      await emptyS3Directory(process.env.BUCKET_NAME as string, filePath);
+    } catch (err) {
+      console.log("Error", err);
+    }
+  } else {
+    const storagePath = process.env.STORAGE_FOLDER;
+    const creationPath = path.join(process.cwd(), storagePath + "/" + filePath);
+
+    try {
+      fs.rmdirSync(creationPath, { recursive: true });
+    } catch (error) {
+      console.log(
+        "Collection's archive directory wasn't deleted most likely because it didn't exist..."
+      );
+    }
+  }
+}

lib/api/storage/s3Client.ts

@@ -0,0 +1,19 @@
+import { S3 } from "@aws-sdk/client-s3";
+
+const s3Client: S3 | undefined =
+  process.env.SPACES_ENDPOINT &&
+  process.env.SPACES_REGION &&
+  process.env.SPACES_KEY &&
+  process.env.SPACES_SECRET
+    ? new S3({
+        forcePathStyle: false,
+        endpoint: process.env.SPACES_ENDPOINT,
+        region: process.env.SPACES_REGION,
+        credentials: {
+          accessKeyId: process.env.SPACES_KEY,
+          secretAccessKey: process.env.SPACES_SECRET,
+        },
+      })
+    : undefined;
+
+export default s3Client;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "linkwarden",
-  "version": "3.0.0",
+  "version": "1.0.0",
   "main": "index.js",
   "repository": "https://github.com/Daniel31x13/link-warden.git",
   "author": "Daniel31X13 <daniel31x13@gmail.com>",
@@ -13,6 +13,7 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.363.0",
     "@fortawesome/fontawesome-svg-core": "^6.3.0",
     "@fortawesome/free-regular-svg-icons": "^6.3.0",
     "@fortawesome/free-solid-svg-icons": "^6.3.0",

pages/api/archives/[...params].ts

@@ -1,9 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from "next";
 import { getServerSession } from "next-auth/next";
 import { authOptions } from "pages/api/auth/[...nextauth]";
-import path from "path";
-import fs from "fs";
 import getPermission from "@/lib/api/getPermission";
+import readFile from "@/lib/api/storage/readFile";
 
 export default async function Index(req: NextApiRequest, res: NextApiResponse) {
   if (!req.query.params)
@@ -27,20 +26,10 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) {
       .status(401)
       .json({ response: "You don't have access to this collection." });
 
-  const requestedPath = `data/archives/${collectionId}/${linkId}`;
-
-  const filePath = path.join(process.cwd(), requestedPath);
-
-  const file = fs.existsSync(filePath)
-    ? fs.readFileSync(filePath)
-    : "File not found, it's possible that the file you're looking for either doesn't exist or hasn't been created yet.";
-
-  if (!fs.existsSync(filePath))
-    res.setHeader("Content-Type", "text/plain").status(404);
-  else if (filePath.endsWith(".pdf"))
-    res.setHeader("Content-Type", "application/pdf").status(200);
-  else if (filePath.endsWith(".png"))
-    res.setHeader("Content-Type", "image/png").status(200);
+  const { file, contentType } = await readFile({
+    filePath: `archives/${collectionId}/${linkId}`,
+  });
+  res.setHeader("Content-Type", contentType).status(200);
 
   return res.send(file);
 }

pages/api/avatar/[id].ts

@@ -2,8 +2,7 @@ import type { NextApiRequest, NextApiResponse } from "next";
 import { getServerSession } from "next-auth/next";
 import { authOptions } from "pages/api/auth/[...nextauth]";
 import { prisma } from "@/lib/api/db";
-import path from "path";
-import fs from "fs";
+import readFile from "@/lib/api/storage/readFile";
 
 export default async function Index(req: NextApiRequest, res: NextApiResponse) {
   const session = await getServerSession(req, res, authOptions);
@@ -41,17 +40,11 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) {
     }
   }
 
-  const filePath = path.join(
-    process.cwd(),
-    `data/uploads/avatar/${queryId}.jpg`
-  );
+  const { file, contentType } = await readFile({
+    filePath: `uploads/avatar/${queryId}.jpg`,
+  });
 
-  const file = fs.existsSync(filePath)
-    ? fs.readFileSync(filePath)
-    : "File not found.";
-
-  if (!fs.existsSync(filePath)) res.setHeader("Content-Type", "text/plain");
-  else res.setHeader("Content-Type", "image/jpeg");
+  res.setHeader("Content-Type", contentType);
 
   return res.send(file);
 }

types/enviornment.d.ts

@@ -0,0 +1,19 @@
+declare global {
+  namespace NodeJS {
+    interface ProcessEnv {
+      NEXTAUTH_SECRET: string;
+      DATABASE_URL: string;
+      NEXTAUTH_URL: string;
+      PAGINATION_TAKE_COUNT: string;
+      STORAGE_FOLDER?: string;
+      IS_CLOUD_INSTANCE?: true;
+      SPACES_KEY?: string;
+      SPACES_SECRET?: string;
+      SPACES_ENDPOINT?: string;
+      BUCKET_NAME?: string;
+      SPACES_REGION?: string;
+    }
+  }
+}
+
+export {};