Feat/import export (#136)

* added import/export functionality

.env.sample

@@ -6,6 +6,7 @@ NEXTAUTH_URL=http://localhost:3000
 
 PAGINATION_TAKE_COUNT=
 STORAGE_FOLDER=
+AUTOSCROLL_TIMEOUT=
 
 # AWS S3 Settings
 SPACES_KEY=

components/Modal/User/PrivacySettings.tsx

@@ -6,6 +6,9 @@ import { signOut, useSession } from "next-auth/react";
 import { faPenToSquare } from "@fortawesome/free-regular-svg-icons";
 import SubmitButton from "../../SubmitButton";
 import { toast } from "react-hot-toast";
+import Link from "next/link";
+import ClickAwayHandler from "@/components/ClickAwayHandler";
+import useInitialData from "@/hooks/useInitialData";
 
 type Props = {
   toggleSettingsModal: Function;
@@ -21,6 +24,7 @@ export default function PrivacySettings({
   const { update, data } = useSession();
   const { account, updateAccount } = useAccountStore();
 
+  const [importDropdown, setImportDropdown] = useState(false);
   const [submitLoader, setSubmitLoader] = useState(false);
 
   const [whitelistedUsersTextbox, setWhiteListedUsersTextbox] = useState(
@@ -46,6 +50,38 @@ export default function PrivacySettings({
     return wordsArray;
   };
 
+  const postJSONFile = async (e: any) => {
+    const file: File = e.target.files[0];
+
+    if (file) {
+      var reader = new FileReader();
+      reader.readAsText(file, "UTF-8");
+      reader.onload = async function (e) {
+        const load = toast.loading("Importing...");
+
+        const response = await fetch("/api/data", {
+          method: "POST",
+          body: e.target?.result,
+        });
+
+        const data = await response.json();
+
+        toast.dismiss(load);
+
+        toast.success("Imported the Bookmarks! Reloading the page...");
+
+        setImportDropdown(false);
+
+        setTimeout(() => {
+          location.reload();
+        }, 2000);
+      };
+      reader.onerror = function (e) {
+        console.log("Error:", e);
+      };
+    }
+  };
+
   const submit = async () => {
     setSubmitLoader(true);
 
@@ -115,6 +151,58 @@ export default function PrivacySettings({
         )}
       </div>
 
+      <div className="mt-5">
+        <p className="text-sm text-sky-700 mb-2">Import/Export Data</p>
+
+        <div className="flex gap-2">
+          <div
+            onClick={() => setImportDropdown(true)}
+            className="w-fit relative"
+            id="import-dropdown"
+          >
+            <div
+              id="import-dropdown"
+              className="border border-slate-200 rounded-md bg-white px-2 text-center select-none cursor-pointer text-sky-900 duration-100 hover:border-sky-700"
+            >
+              Import From
+            </div>
+            {importDropdown ? (
+              <ClickAwayHandler
+                onClickOutside={(e: Event) => {
+                  const target = e.target as HTMLInputElement;
+                  if (target.id !== "import-dropdown") setImportDropdown(false);
+                }}
+                className={`absolute top-7 left-0 w-36 py-1 shadow-md border border-sky-100 bg-gray-50 rounded-md flex flex-col z-20`}
+              >
+                <div className="cursor-pointer rounded-md">
+                  <label
+                    htmlFor="import-file"
+                    title="JSON"
+                    className="flex items-center gap-2 py-1 px-2 hover:bg-slate-200 duration-100 cursor-pointer"
+                  >
+                    Linkwarden
+                    <input
+                      type="file"
+                      name="photo"
+                      id="import-file"
+                      accept=".json"
+                      className="hidden"
+                      onChange={postJSONFile}
+                    />
+                  </label>
+                </div>
+              </ClickAwayHandler>
+            ) : null}
+          </div>
+
+          <Link className="w-fit" href="/api/data">
+            <div className="border border-slate-200 rounded-md bg-white px-2 text-center select-none cursor-pointer text-sky-900 duration-100 hover:border-sky-700">
+              Export Data
+            </div>
+          </Link>
+        </div>
+      </div>
+
       <SubmitButton
         onClick={submit}
         loading={submitLoader}

components/Sidebar.tsx

@@ -66,7 +66,9 @@ export default function Sidebar({ className }: { className?: string }) {
             icon={faChartSimple}
             className={`w-8 h-8 drop-shadow text-sky-500`}
           />
-          <p className="text-sky-700 text-xs font-semibold">Dashboard</p>
+          <p className="text-sky-700 text-xs xl:text-sm font-semibold">
+            Dashboard
+          </p>
         </Link>
 
         <Link
@@ -81,9 +83,7 @@ export default function Sidebar({ className }: { className?: string }) {
             icon={faLink}
             className={`w-8 h-8 drop-shadow text-sky-500`}
           />
-          <p className="text-sky-700 text-xs font-semibold">
+          <p className="text-sky-700 text-xs xl:text-sm font-semibold">Links</p>
-            <span className="hidden xl:inline-block">All</span> Links
-          </p>
         </Link>
 
         <Link
@@ -96,8 +96,8 @@ export default function Sidebar({ className }: { className?: string }) {
             icon={faFolder}
             className={`w-8 h-8 drop-shadow text-sky-500`}
           />
-          <p className="text-sky-700 text-xs font-semibold">
+          <p className="text-sky-700 text-xs xl:text-sm font-semibold">
-            <span className="hidden xl:inline-block">All</span> Collections
+            Collections
           </p>
         </Link>
       </div>

lib/api/archive.ts

@@ -10,7 +10,10 @@ export default async function archive(linkId: number, url: string) {
   try {
     await page.goto(url, { waitUntil: "domcontentloaded" });
 
-    await autoScroll(page);
+    await page.evaluate(
+      autoScroll,
+      Number(process.env.AUTOSCROLL_TIMEOUT) || 30
+    );
 
     const linkExists = await prisma.link.findUnique({
       where: {
@@ -47,29 +50,31 @@ export default async function archive(linkId: number, url: string) {
   }
 }
 
-const autoScroll = async (page: Page) => {
+const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => {
-  await page.evaluate(async () => {
+  const timeoutPromise = new Promise<void>((_, reject) => {
-    const timeoutPromise = new Promise<void>((_, reject) => {
+    setTimeout(() => {
-      setTimeout(() => {
+      reject(
-        reject(new Error("Auto scroll took too long (more than 20 seconds)."));
+        new Error(
-      }, 20000);
+          `Auto scroll took too long (more than ${AUTOSCROLL_TIMEOUT} seconds).`
-    });
+        )
-
+      );
-    const scrollingPromise = new Promise<void>((resolve) => {
+    }, AUTOSCROLL_TIMEOUT * 1000);
-      let totalHeight = 0;
-      let distance = 100;
-      let scrollDown = setInterval(() => {
-        let scrollHeight = document.body.scrollHeight;
-        window.scrollBy(0, distance);
-        totalHeight += distance;
-        if (totalHeight >= scrollHeight) {
-          clearInterval(scrollDown);
-          window.scroll(0, 0);
-          resolve();
-        }
-      }, 100);
-    });
-
-    await Promise.race([scrollingPromise, timeoutPromise]);
   });
+
+  const scrollingPromise = new Promise<void>((resolve) => {
+    let totalHeight = 0;
+    let distance = 100;
+    let scrollDown = setInterval(() => {
+      let scrollHeight = document.body.scrollHeight;
+      window.scrollBy(0, distance);
+      totalHeight += distance;
+      if (totalHeight >= scrollHeight) {
+        clearInterval(scrollDown);
+        window.scroll(0, 0);
+        resolve();
+      }
+    }, 100);
+  });
+
+  await Promise.race([scrollingPromise, timeoutPromise]);
 };

lib/api/controllers/collections/postCollection.ts

@@ -40,14 +40,6 @@ export default async function postCollection(
       name: collection.name.trim(),
       description: collection.description,
       color: collection.color,
-      members: {
-        create: collection.members.map((e) => ({
-          user: { connect: { id: e.user.id } },
-          canCreate: e.canCreate,
-          canUpdate: e.canUpdate,
-          canDelete: e.canDelete,
-        })),
-      },
     },
     include: {
       _count: {

lib/api/controllers/data/getData.ts

@@ -0,0 +1,24 @@
+import { prisma } from "@/lib/api/db";
+
+export default async function getData(userId: number) {
+  const user = await prisma.user.findUnique({
+    where: { id: userId },
+    include: {
+      collections: {
+        include: {
+          links: {
+            include: {
+              tags: true,
+            },
+          },
+        },
+      },
+    },
+  });
+
+  if (!user) return { response: "User not found.", status: 404 };
+
+  const { password, id, image, ...userData } = user;
+
+  return { response: userData, status: 200 };
+}

lib/api/controllers/data/postData.ts

@@ -0,0 +1,86 @@
+import { prisma } from "@/lib/api/db";
+import { Backup } from "@/types/global";
+import createFolder from "@/lib/api/storage/createFolder";
+
+export default async function getData(userId: number, data: Backup) {
+  // Import collections
+  try {
+    data.collections.forEach(async (e) => {
+      e.name = e.name.trim();
+
+      const findCollection = await prisma.user.findUnique({
+        where: {
+          id: userId,
+        },
+        select: {
+          collections: {
+            where: {
+              name: e.name,
+            },
+          },
+        },
+      });
+
+      const checkIfCollectionExists = findCollection?.collections[0];
+
+      let collectionId = findCollection?.collections[0]?.id;
+
+      if (!checkIfCollectionExists) {
+        const newCollection = await prisma.collection.create({
+          data: {
+            owner: {
+              connect: {
+                id: userId,
+              },
+            },
+            name: e.name,
+            description: e.description,
+            color: e.color,
+          },
+        });
+
+        createFolder({ filePath: `archives/${newCollection.id}` });
+
+        collectionId = newCollection.id;
+      }
+
+      // Import Links
+      e.links.forEach(async (e) => {
+        const newLink = await prisma.link.create({
+          data: {
+            url: e.url,
+            name: e.name,
+            description: e.description,
+            collection: {
+              connect: {
+                id: collectionId,
+              },
+            },
+            tags: {
+              connectOrCreate: e.tags.map((tag) => ({
+                where: {
+                  name_ownerId: {
+                    name: tag.name.trim(),
+                    ownerId: userId,
+                  },
+                },
+                create: {
+                  name: tag.name.trim(),
+                  owner: {
+                    connect: {
+                      id: userId,
+                    },
+                  },
+                },
+              })),
+            },
+          },
+        });
+      });
+    });
+  } catch (err) {
+    console.log(err);
+  }
+
+  return { response: "Success.", status: 200 };
+}

lib/api/controllers/links/postLink.ts

@@ -20,9 +20,6 @@ export default async function postLink(
     };
   }
 
-  // This has to move above we assign link.collection.name
-  // Because if the link is null (write then delete text on collection)
-  // It will try to do trim on empty string and will throw and error, this prevents it.
   if (!link.collection.name) {
     link.collection.name = "Unnamed Collection";
   }
@@ -54,7 +51,7 @@ export default async function postLink(
       ? link.description
       : await getTitle(link.url);
 
-  const newLink: Link = await prisma.link.create({
+  const newLink = await prisma.link.create({
     data: {
       url: link.url,
       name: link.name,

lib/api/storage/readFile.ts

@@ -9,12 +9,14 @@ import s3Client from "./s3Client";
 import util from "util";
 
 type ReturnContentTypes =
-  | "text/plain"
+  | "text/html"
   | "image/jpeg"
   | "image/png"
   | "application/pdf";
 
-export default async function readFile({ filePath }: { filePath: string }) {
+export default async function readFile(filePath: string) {
+  const isRequestingAvatar = filePath.startsWith("uploads/avatar");
+
   let contentType: ReturnContentTypes;
 
   if (s3Client) {
@@ -28,6 +30,7 @@ export default async function readFile({ filePath }: { filePath: string }) {
         | {
             file: Buffer | string;
             contentType: ReturnContentTypes;
+            status: number;
           }
         | undefined;
 
@@ -38,11 +41,12 @@ export default async function readFile({ filePath }: { filePath: string }) {
       try {
         await headObjectAsync(bucketParams);
       } catch (err) {
-        contentType = "text/plain";
+        contentType = "text/html";
 
         returnObject = {
-          file: "File not found, it's possible that the file you're looking for either doesn't exist or hasn't been created yet.",
+          file: isRequestingAvatar ? "File not found." : fileNotFoundTemplate,
           contentType,
+          status: isRequestingAvatar ? 200 : 400,
         };
       }
 
@@ -60,14 +64,14 @@ export default async function readFile({ filePath }: { filePath: string }) {
           // if (filePath.endsWith(".jpg"))
           contentType = "image/jpeg";
         }
-        returnObject = { file: data as Buffer, contentType };
+        returnObject = { file: data as Buffer, contentType, status: 200 };
       }
 
       return returnObject;
     } catch (err) {
       console.log("Error:", err);
 
-      contentType = "text/plain";
+      contentType = "text/html";
       return {
         file: "An internal occurred, please contact support.",
         contentType,
@@ -77,13 +81,7 @@ export default async function readFile({ filePath }: { filePath: string }) {
     const storagePath = process.env.STORAGE_FOLDER || "data";
     const creationPath = path.join(process.cwd(), storagePath + "/" + filePath);
 
-    const file = fs.existsSync(creationPath)
+    if (filePath.endsWith(".pdf")) {
-      ? fs.readFileSync(creationPath)
-      : "File not found, it's possible that the file you're looking for either doesn't exist or hasn't been created yet.";
-
-    if (file.toString().startsWith("File not found")) {
-      contentType = "text/plain";
-    } else if (filePath.endsWith(".pdf")) {
       contentType = "application/pdf";
     } else if (filePath.endsWith(".png")) {
       contentType = "image/png";
@@ -92,7 +90,16 @@ export default async function readFile({ filePath }: { filePath: string }) {
       contentType = "image/jpeg";
     }
 
-    return { file, contentType };
+    if (!fs.existsSync(creationPath))
+      return {
+        file: isRequestingAvatar ? "File not found." : fileNotFoundTemplate,
+        contentType: "text/html",
+        status: isRequestingAvatar ? 200 : 400,
+      };
+    else {
+      const file = fs.readFileSync(creationPath);
+      return { file, contentType, status: 200 };
+    }
   }
 }
 
@@ -105,3 +112,21 @@ const streamToBuffer = (stream: any) => {
     stream.on("end", () => resolve(Buffer.concat(chunks)));
   });
 };
+
+const fileNotFoundTemplate = `<!DOCTYPE html>
+                              <html lang="en">
+                              <head>
+                                <meta charset="UTF-8">
+                                <meta name="viewport" content="width=device-width, initial-scale=1.0">
+                                <title>File not found</title>
+                              </head>
+                              <body style="margin-left: auto; margin-right: auto; max-width: 500px; padding: 1rem; font-family: sans-serif; background-color: rgb(251, 251, 251);">
+                                <h1>File not found</h1>
+                                <h2>It is possible that the file you're looking for either doesn't exist or hasn't been created yet.</h2>
+                                <h3>Some possible reasons are:</h3>
+                                <ul>
+                                  <li>You are trying to access a file too early, before it has been fully archived.</li>
+                                  <li>The file doesn't exist either because it encountered an error while being archived, or it simply doesn't exist.</li>
+                                </ul>
+                              </body>
+                              </html>`;

lib/client/avatarExists.ts

@@ -1,4 +1,4 @@
 export default async function avatarExists(fileUrl: string): Promise<boolean> {
   const response = await fetch(fileUrl, { method: "HEAD" });
-  return !(response.headers.get("content-type") === "text/plain");
+  return !(response.headers.get("content-type") === "text/html");
 }

pages/api/archives/[...params].ts

@@ -31,10 +31,10 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) {
       .status(401)
       .json({ response: "You don't have access to this collection." });
 
-  const { file, contentType } = await readFile({
+  const { file, contentType, status } = await readFile(
-    filePath: `archives/${collectionId}/${linkId}`,
+    `archives/${collectionId}/${linkId}`
-  });
+  );
-  res.setHeader("Content-Type", contentType).status(200);
+  res.setHeader("Content-Type", contentType).status(status as number);
 
   return res.send(file);
 }

pages/api/avatar/[id].ts

@@ -13,7 +13,7 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) {
 
   if (!userId || !username)
     return res
-      .setHeader("Content-Type", "text/plain")
+      .setHeader("Content-Type", "text/html")
       .status(401)
       .send("You must be logged in.");
   else if (session?.user?.isSubscriber === false)
@@ -24,7 +24,7 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) {
 
   if (!queryId)
     return res
-      .setHeader("Content-Type", "text/plain")
+      .setHeader("Content-Type", "text/html")
       .status(401)
       .send("Invalid parameters.");
 
@@ -34,27 +34,27 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) {
         id: queryId,
       },
       include: {
-        whitelistedUsers: true
+        whitelistedUsers: true,
-      }
+      },
     });
 
-    const whitelistedUsernames = targetUser?.whitelistedUsers.map(whitelistedUsername => whitelistedUsername.username);
+    const whitelistedUsernames = targetUser?.whitelistedUsers.map(
+      (whitelistedUsername) => whitelistedUsername.username
+    );
 
-    if (
+    if (targetUser?.isPrivate && !whitelistedUsernames?.includes(username)) {
-      targetUser?.isPrivate &&
-      !whitelistedUsernames?.includes(username)
-    ) {
       return res
-        .setHeader("Content-Type", "text/plain")
+        .setHeader("Content-Type", "text/html")
         .send("This profile is private.");
     }
   }
 
-  const { file, contentType } = await readFile({
+  const { file, contentType, status } = await readFile(
-    filePath: `uploads/avatar/${queryId}.jpg`,
+    `uploads/avatar/${queryId}.jpg`
-  });
+  );
 
-  res.setHeader("Content-Type", contentType);
+  return res
-
+    .setHeader("Content-Type", contentType)
-  return res.send(file);
+    .status(status as number)
+    .send(file);
 }

pages/api/data/index.ts

@@ -0,0 +1,31 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+import { getServerSession } from "next-auth/next";
+import { authOptions } from "@/pages/api/auth/[...nextauth]";
+import getData from "@/lib/api/controllers/data/getData";
+import postData from "@/lib/api/controllers/data/postData";
+
+export default async function users(req: NextApiRequest, res: NextApiResponse) {
+  const session = await getServerSession(req, res, authOptions);
+
+  if (!session?.user.id) {
+    return res.status(401).json({ response: "You must be logged in." });
+  } else if (session?.user?.isSubscriber === false)
+    res.status(401).json({
+      response:
+        "You are not a subscriber, feel free to reach out to us at support@linkwarden.app in case of any issues.",
+    });
+
+  if (req.method === "GET") {
+    const data = await getData(session.user.id);
+    if (data.status === 200)
+      return res
+        .setHeader("Content-Type", "application/json")
+        .setHeader("Content-Disposition", "attachment; filename=backup.json")
+        .status(data.status)
+        .json(data.response);
+  } else if (req.method === "POST") {
+    console.log(JSON.parse(req.body));
+    const data = await postData(session.user.id, JSON.parse(req.body));
+    return res.status(data.status).json({ response: data.response });
+  }
+}

prisma/schema.prisma

@@ -93,31 +93,32 @@ model Collection {
 }
 
 model UsersAndCollections {
-  user   User @relation(fields: [userId], references: [id])
+  user         User       @relation(fields: [userId], references: [id])
-  userId Int
+  userId       Int
 
   collection   Collection @relation(fields: [collectionId], references: [id])
   collectionId Int
 
-  canCreate Boolean
+  canCreate    Boolean
-  canUpdate Boolean
+  canUpdate    Boolean
-  canDelete Boolean
+  canDelete    Boolean
 
   @@id([userId, collectionId])
 }
 
 model Link {
-  id          Int    @id @default(autoincrement())
+  id             Int        @id @default(autoincrement())
-  name        String
+  name           String
-  url         String
+  url            String
-  description String @default("")
+  description    String     @default("")
 
-  pinnedBy User[]
+  pinnedBy       User[]
 
-  collection   Collection @relation(fields: [collectionId], references: [id])
+  collection     Collection @relation(fields: [collectionId], references: [id])
-  collectionId Int
+  collectionId   Int
-  tags         Tag[]
+  tags           Tag[]
-  createdAt    DateTime   @default(now())
+
+  createdAt      DateTime   @default(now())
 }
 
 model Tag {

types/enviornment.d.ts

@@ -6,6 +6,7 @@ declare global {
       NEXTAUTH_URL: string;
       PAGINATION_TAKE_COUNT?: string;
       STORAGE_FOLDER?: string;
+      AUTOSCROLL_TIMEOUT?: string;
 
       SPACES_KEY?: string;
       SPACES_SECRET?: string;

types/global.ts

@@ -36,7 +36,7 @@ export interface CollectionIncludingMembersAndLinkCount
 export interface AccountSettings extends User {
   profilePic: string;
   newPassword?: string;
-  whitelistedUsers: string[]
+  whitelistedUsers: string[];
 }
 
 interface LinksIncludingTags extends Link {
@@ -77,3 +77,11 @@ export type PublicLinkRequestQuery = {
   cursor?: number;
   collectionId: number;
 };
+
+interface CollectionIncludingLinks extends Collection {
+  links: LinksIncludingTags[];
+}
+
+export interface Backup extends Omit<User, "password" | "id" | "image"> {
+  collections: CollectionIncludingLinks[];
+}