refactored import and add support for subcollections

lib/api/controllers/migration/importFromHTMLFile.ts

@@ -1,6 +1,7 @@
 import { prisma } from "@/lib/api/db";
 import createFolder from "@/lib/api/storage/createFolder";
 import { JSDOM } from "jsdom";
+import { parse, Node, Element, TextNode } from "himalaya";
 
 const MAX_LINKS_PER_USER = Number(process.env.MAX_LINKS_PER_USER) || 30000;
 
@@ -11,6 +12,11 @@ export default async function importFromHTMLFile(
   const dom = new JSDOM(rawData);
   const document = dom.window.document;
 
+  // remove bad tags
+  document.querySelectorAll("meta").forEach((e) => (e.outerHTML = e.innerHTML));
+  document.querySelectorAll("META").forEach((e) => (e.outerHTML = e.innerHTML));
+  document.querySelectorAll("P").forEach((e) => (e.outerHTML = e.innerHTML));
+
   const bookmarks = document.querySelectorAll("A");
   const totalImports = bookmarks.length;
 
@@ -28,153 +34,161 @@ export default async function importFromHTMLFile(
       status: 400,
     };
 
-  const folders = document.querySelectorAll("H3");
+  const jsonData = parse(document.documentElement.outerHTML);
-  let unorganizedCollectionId: number | null = null;
 
-  if (folders.length === 0) {
+  // console.log(jsonData);
-    const unorganizedCollection = await prisma.collection.findFirst({
-      where: {
-        name: "Imported",
-        ownerId: userId,
-      },
-    });
 
-    if (!unorganizedCollection) {
+  for (const item of jsonData) {
-      const newUnorganizedCollection = await prisma.collection.create({
+    console.log(item);
-        data: {
+    await processBookmarks(userId, item as Element);
-          name: "Imported",
-          description:
-            "Automatically created collection for imported bookmarks.",
-          ownerId: userId,
-        },
-      });
-      unorganizedCollectionId = newUnorganizedCollection.id;
-    } else {
-      unorganizedCollectionId = unorganizedCollection.id;
-    }
-
-    createFolder({ filePath: `archives/${unorganizedCollectionId}` });
   }
 
-  await prisma
-    .$transaction(
-      async () => {
-        if (unorganizedCollectionId) {
-          // @ts-ignore
-          for (const bookmark of bookmarks) {
-            createBookmark(userId, bookmark, unorganizedCollectionId);
-          }
-        } else {
-          // @ts-ignore
-          for (const folder of folders) {
-            await createCollectionAndBookmarks(
-              userId,
-              folder,
-              folder.nextElementSibling,
-              null
-            );
-          }
-        }
-      },
-      { timeout: 30000 }
-    )
-    .catch((err) => console.log(err));
-
   return { response: "Success.", status: 200 };
 }
 
-const createCollectionAndBookmarks = async (
+async function processBookmarks(
   userId: number,
-  folder: any,
+  data: Node,
-  folderContent: any,
+  parentCollectionId?: number
-  parentId: number | null
+) {
+  if (data.type === "element") {
+    for (const item of data.children) {
+      if (item.type === "element" && item.tagName === "dt") {
+        let collectionId;
+        const collectionName = item.children.find(
+          (e) => e.type === "element" && e.tagName === "h3"
+        ) as Element;
+
+        console.log("collection:", item);
+        console.log("collectionName:", collectionName);
+
+        // This is a collection or sub-collection
+        if (collectionName) {
+          collectionId = await createCollection(
+            userId,
+            (collectionName.children[0] as TextNode).content,
+            parentCollectionId
+          );
+        }
+        await processBookmarks(
+          userId,
+          item,
+          collectionId || parentCollectionId
+        );
+      } else if (item.type === "element" && item.tagName === "a") {
+        // This is a link
+
+        // get link href
+        const linkUrl = item?.attributes.find((e) => e.key === "href")?.value;
+
+        // get link name
+        const linkName = (
+          item?.children.find((e) => e.type === "text") as TextNode
+        )?.content;
+
+        // get link tags
+        const linkTags = item?.attributes
+          .find((e) => e.key === "tags")
+          ?.value.split(",");
+
+        console.log("link:", item);
+
+        if (linkUrl && parentCollectionId) {
+          await createLink(
+            userId,
+            linkUrl,
+            parentCollectionId,
+            linkName,
+            "",
+            linkTags
+          );
+        } else if (linkUrl) {
+          // create a collection named "Imported Bookmarks" and add the link to it
+          const collectionId = await createCollection(userId, "Imports");
+
+          await createLink(
+            userId,
+            linkUrl,
+            collectionId,
+            linkName,
+            "",
+            linkTags
+          );
+        }
+
+        await processBookmarks(userId, item, parentCollectionId);
+      } else {
+        // This could be anything else
+        await processBookmarks(userId, item, parentCollectionId);
+      }
+
+      // Add more conditions as necessary based on your JSON structure
+    }
+  }
+}
+
+const createCollection = async (
+  userId: number,
+  collectionName: string,
+  parentId?: number
 ) => {
   const findCollection = await prisma.collection.findFirst({
     where: {
-      name: folder.textContent.trim(),
+      parentId,
+      name: collectionName,
       ownerId: userId,
     },
   });
 
-  const checkIfCollectionExists = findCollection;
+  if (findCollection) {
-  let collectionId = findCollection?.id;
+    return findCollection.id;
+  }
 
-  if (!checkIfCollectionExists || !collectionId) {
+  const collectionId = await prisma.collection.create({
-    const newCollection = await prisma.collection.create({
+    data: {
-      data: {
+      name: collectionName,
-        name: folder.textContent.trim(),
+      parent: parentId
-        description: "",
+        ? {
-        color: "#0ea5e9",
+            connect: {
-        isPublic: false,
+              id: parentId,
-        ownerId: userId,
+            },
-        parentId
+          }
+        : undefined,
+      owner: {
+        connect: {
+          id: userId,
+        },
       },
-    });
-
-    createFolder({ filePath: `archives/${newCollection.id}` });
-
-    collectionId = newCollection.id;
-  }
-
-  createFolder({ filePath: `archives/${collectionId}` });
-
-  const bookmarks = folderContent.querySelectorAll("A");
-  for (const bookmark of bookmarks) {
-    createBookmark(userId, bookmark, collectionId);
-  }
-
-  const subfolders = folderContent.querySelectorAll("H3");
-  for (const subfolder of subfolders) {
-    await createCollectionAndBookmarks(userId, subfolder, subfolder.nextElementSibling, collectionId);
-  }
-};
-
-const createBookmark = async (
-  userId: number,
-  bookmark: any,
-  collectionId: number
-) => {
-  // Move up to the parent node (<DT>) and then find the next sibling
-  let parentDT = bookmark.parentNode;
-  let nextSibling = parentDT ? parentDT.nextSibling : null;
-  let description = "";
-
-  // Loop through siblings to skip any potential text nodes or whitespace
-  while (nextSibling && nextSibling.nodeType !== 1) {
-    nextSibling = nextSibling.nextSibling;
-  }
-
-  // Check if the next sibling element is a <DD> tag and use its content as the description
-  if (nextSibling && nextSibling.tagName === "DD") {
-    description = nextSibling.textContent.trim();
-  }
-
-  const linkName = bookmark.textContent.trim();
-  const linkURL = bookmark.getAttribute("HREF");
-
-  const existingLink = await prisma.link.findFirst({
-    where: {
-      url: linkURL,
-      collectionId
     },
   });
 
-  // Create the link only if it doesn't already exist
+  createFolder({ filePath: `archives/${collectionId.id}` });
-  if (!existingLink) {
+
-    await prisma.link.create({
+  return collectionId.id;
-      data: {
+};
-        name: linkName,
+
-        url: linkURL,
+const createLink = async (
-        tags: bookmark.getAttribute("TAGS")
+  userId: number,
+  url: string,
+  collectionId: number,
+  name?: string,
+  description?: string,
+  tags?: string[]
+) => {
+  await prisma.link.create({
+    data: {
+      name: name || "",
+      url,
+      description,
+      collectionId,
+      tags:
+        tags && tags[0]
           ? {
-            connectOrCreate: bookmark
+              connectOrCreate: tags.map((tag: string) => {
-              .getAttribute("TAGS")
+                return (
-              .split(",")
+                  {
-              .map((tag: string) =>
-                tag
-                  ? {
                     where: {
-                      data: {
+                      name_ownerId: {
                         name: tag.trim(),
                         ownerId: userId,
                       },
@@ -187,14 +201,11 @@ const createBookmark = async (
                         },
                       },
                     },
-                  }
+                  } || undefined
-                  : undefined
+                );
-              ),
+              }),
-          }
+            }
           : undefined,
-        description,
+    },
-        collectionId,
+  });
-      },
+};
-    });
-  }
-};

package.json

@@ -44,6 +44,7 @@
     "eslint-config-next": "13.4.9",
     "formidable": "^3.5.1",
     "framer-motion": "^10.16.4",
+    "himalaya": "^1.1.0",
     "jimp": "^0.22.10",
     "jsdom": "^22.1.0",
     "lottie-web": "^5.12.2",

prisma/schema.prisma

@@ -165,4 +165,3 @@ model AccessToken {
   createdAt  DateTime  @default(now())
   updatedAt  DateTime  @default(now()) @updatedAt
 }
-

types/himalaya.d.ts

@@ -0,0 +1,22 @@
+declare module "himalaya" {
+  export interface Attribute {
+    key: string;
+    value: string;
+  }
+
+  export interface TextNode {
+    type: "text";
+    content: string;
+  }
+
+  export type Node = TextNode | Element;
+
+  export interface Element {
+    type: "element";
+    tagName: string;
+    attributes: Attribute[];
+    children: Node[];
+  }
+
+  export function parse(html: string): Node[];
+}

yarn.lock

@@ -3713,6 +3713,11 @@ hexoid@^1.0.0:
   resolved "https://registry.yarnpkg.com/hexoid/-/hexoid-1.0.0.tgz#ad10c6573fb907de23d9ec63a711267d9dc9bc18"
   integrity sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==
 
+himalaya@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/himalaya/-/himalaya-1.1.0.tgz#31724ae9d35714cd7c6f4be94888953f3604606a"
+  integrity sha512-LLase1dHCRMel68/HZTFft0N0wti0epHr3nNY7ynpLbyZpmrKMQ8YIpiOV77TM97cNpC8Wb2n6f66IRggwdWPw==
+
 hoist-non-react-statics@^3.3.1:
   version "3.3.2"
   resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45"