improved archive handler

components/Announcement.tsx

@@ -10,8 +10,8 @@ export default function Announcement({ toggleAnnouncementBar }: Props) {
   const announcementId = localStorage.getItem("announcementId");
 
   return (
-    <div className="fixed left-0 right-0 bottom-20 sm:bottom-10 w-full p-5 z-30">
-      <div className="mx-auto w-full p-2 flex justify-between gap-2 items-center border border-primary shadow-xl rounded-xl bg-base-300 backdrop-blur-sm bg-opacity-80 max-w-md">
+    <div className="fixed mx-auto bottom-20 sm:bottom-10 w-full pointer-events-none p-5 z-30">
+      <div className="mx-auto pointer-events-auto p-2 flex justify-between gap-2 items-center border border-primary shadow-xl rounded-xl bg-base-300 backdrop-blur-sm bg-opacity-80 max-w-md">
         <i className="bi-stars text-2xl text-yellow-600 dark:text-yellow-500"></i>
         <p className="w-4/5 text-center text-sm sm:text-base">
           <Trans

lib/api/archiveHandler.ts

@@ -2,7 +2,7 @@ import { LaunchOptions, chromium, devices } from "playwright";
 import { prisma } from "./db";
 import sendToWayback from "./preservationScheme/sendToWayback";
 import { Collection, Link, User } from "@prisma/client";
-import validateUrlSize from "./validateUrlSize";
+import fetchHeaders from "./fetchHeaders";
 import createFolder from "./storage/createFolder";
 import { removeFiles } from "./manageLinkFiles";
 import handleMonolith from "./preservationScheme/handleMonolith";
@@ -65,17 +65,9 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
       (async () => {
         const user = link.collection?.owner;
 
-        const validatedUrl = link.url
-          ? await validateUrlSize(link.url)
-          : undefined;
+        const header = link.url ? await fetchHeaders(link.url) : undefined;
 
-        if (
-          validatedUrl === null &&
-          process.env.IGNORE_URL_SIZE_LIMIT !== "true"
-        )
-          throw "Something went wrong while retrieving the file size.";
-
-        const contentType = validatedUrl?.get("content-type");
+        const contentType = header?.get("content-type");
         let linkType = "url";
         let imageExtension = "png";
 

lib/api/controllers/links/postLink.ts

@@ -1,8 +1,7 @@
 import { prisma } from "@/lib/api/db";
 import { LinkIncludingShortenedCollectionAndTags } from "@/types/global";
-import getTitle from "@/lib/shared/getTitle";
+import fetchTitleAndHeaders from "@/lib/shared/fetchTitleAndHeaders";
 import createFolder from "@/lib/api/storage/createFolder";
-import validateUrlSize from "../../validateUrlSize";
 import setLinkCollection from "../../setLinkCollection";
 
 const MAX_LINKS_PER_USER = Number(process.env.MAX_LINKS_PER_USER) || 30000;
@@ -70,17 +69,12 @@ export default async function postLink(
       status: 400,
     };
 
-  const title =
-    !(link.name && link.name !== "") && link.url
-      ? await getTitle(link.url)
-      : "";
+  const { title, headers } = await fetchTitleAndHeaders(link.url || "");
 
   const name =
     link.name && link.name !== "" ? link.name : link.url ? title : "";
 
-  const validatedUrl = link.url ? await validateUrlSize(link.url) : undefined;
-
-  const contentType = validatedUrl?.get("content-type");
+  const contentType = headers?.get("content-type");
   let linkType = "url";
   let imageExtension = "png";
 

lib/api/fetchHeaders.ts

@@ -2,7 +2,7 @@ import fetch from "node-fetch";
 import https from "https";
 import { SocksProxyAgent } from "socks-proxy-agent";
 
-export default async function validateUrlSize(url: string) {
+export default async function fetchHeaders(url: string) {
   if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null;
 
   try {
@@ -29,13 +29,17 @@ export default async function validateUrlSize(url: string) {
       };
     }
 
-    const response = await fetch(url, fetchOpts);
+    const responsePromise = fetch(url, fetchOpts);
 
-    const totalSizeMB =
-      Number(response.headers.get("content-length")) / Math.pow(1024, 2);
-    if (totalSizeMB > Number(process.env.NEXT_PUBLIC_MAX_FILE_BUFFER || 10))
-      return null;
-    else return response.headers;
+    const timeoutPromise = new Promise((_, reject) => {
+      setTimeout(() => {
+        reject(new Error("Fetch header timeout"));
+      }, 10 * 1000); // Stop after 10 seconds
+    });
+
+    const response = await Promise.race([responsePromise, timeoutPromise]);
+
+    return (response as Response)?.headers || null;
   } catch (err) {
     console.log(err);
     return null;

lib/api/preservationScheme/imageHandler.ts

@@ -1,6 +1,7 @@
 import { Link } from "@prisma/client";
 import { prisma } from "../db";
 import createFile from "../storage/createFile";
+import generatePreview from "../generatePreview";
 
 const imageHandler = async ({ url, id }: Link, extension: string) => {
   const image = await fetch(url as string).then((res) => res.blob());
@@ -18,6 +19,8 @@ const imageHandler = async ({ url, id }: Link, extension: string) => {
   });
 
   if (linkExists) {
+    await generatePreview(buffer, linkExists.collectionId, id);
+
     await createFile({
       data: buffer,
       filePath: `archives/${linkExists.collectionId}/${id}.${extension}`,

lib/shared/fetchTitleAndHeaders.ts

@@ -2,7 +2,7 @@ import fetch from "node-fetch";
 import https from "https";
 import { SocksProxyAgent } from "socks-proxy-agent";
 
-export default async function getTitle(url: string) {
+export default async function fetchTitleAndHeaders(url: string) {
   try {
     const httpsAgent = new https.Agent({
       rejectUnauthorized:
@@ -41,12 +41,16 @@ export default async function getTitle(url: string) {
 
       // regular expression to find the <title> tag
       let match = text.match(/<title.*>([^<]*)<\/title>/);
-      if (match) return match[1];
-      else return "";
+
+      const title = match[1] || "";
+      const headers = (response as Response)?.headers || null;
+
+      return { title, headers };
     } else {
-      return "";
+      return { title: "", headers: null };
     }
   } catch (err) {
     console.log(err);
+    return { title: "", headers: null };
   }
 }

package.json

@@ -60,7 +60,7 @@
     "next-i18next": "^15.3.0",
     "node-fetch": "^2.7.0",
     "nodemailer": "^6.9.3",
-    "playwright": "^1.43.1",
+    "playwright": "^1.45.0",
     "react": "18.2.0",
     "react-colorful": "^5.6.1",
     "react-dom": "18.2.0",
@@ -77,7 +77,7 @@
     "zustand": "^4.3.8"
   },
   "devDependencies": {
-    "@playwright/test": "^1.43.1",
+    "@playwright/test": "^1.45.0",
     "@types/bcrypt": "^5.0.0",
     "@types/dompurify": "^3.0.4",
     "@types/jsdom": "^21.1.3",

yarn.lock

@@ -1293,12 +1293,12 @@
     tiny-glob "^0.2.9"
     tslib "^2.4.0"
 
-"@playwright/test@^1.43.1":
-  version "1.43.1"
-  resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.43.1.tgz#16728a59eb8ce0f60472f98d8886d6cab0fa3e42"
-  integrity sha512-HgtQzFgNEEo4TE22K/X7sYTYNqEMMTZmFS8kTq6m8hXj+m1D8TgwgIbumHddJa9h4yl4GkKb8/bgAl2+g7eDgA==
+"@playwright/test@^1.45.0":
+  version "1.45.0"
+  resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.45.0.tgz#790a66165a46466c0d7099dd260881802f5aba7e"
+  integrity sha512-TVYsfMlGAaxeUllNkywbwek67Ncf8FRGn8ZlRdO291OL3NjG9oMbfVhyP82HQF0CZLMrYsvesqoUekxdWuF9Qw==
   dependencies:
-    playwright "1.43.1"
+    playwright "1.45.0"
 
 "@prisma/client@^4.16.2":
   version "4.16.2"
@@ -5005,17 +5005,17 @@ pixelmatch@^4.0.2:
   dependencies:
     pngjs "^3.0.0"
 
-playwright-core@1.43.1:
-  version "1.43.1"
-  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.43.1.tgz#0eafef9994c69c02a1a3825a4343e56c99c03b02"
-  integrity sha512-EI36Mto2Vrx6VF7rm708qSnesVQKbxEWvPrfA1IPY6HgczBplDx7ENtx+K2n4kJ41sLLkuGfmb0ZLSSXlDhqPg==
+playwright-core@1.45.0:
+  version "1.45.0"
+  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.45.0.tgz#5741a670b7c9060ce06852c0051d84736fb94edc"
+  integrity sha512-lZmHlFQ0VYSpAs43dRq1/nJ9G/6SiTI7VPqidld9TDefL9tX87bTKExWZZUF5PeRyqtXqd8fQi2qmfIedkwsNQ==
 
-playwright@1.43.1, playwright@^1.43.1:
-  version "1.43.1"
-  resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.43.1.tgz#8ad08984ac66c9ef3d0db035be54dd7ec9f1c7d9"
-  integrity sha512-V7SoH0ai2kNt1Md9E3Gwas5B9m8KR2GVvwZnAI6Pg0m3sh7UvgiYhRrhsziCmqMJNouPckiOhk8T+9bSAK0VIA==
+playwright@1.45.0, playwright@^1.45.0:
+  version "1.45.0"
+  resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.45.0.tgz#400c709c64438690f13705cb9c88ef93089c5c27"
+  integrity sha512-4z3ac3plDfYzGB6r0Q3LF8POPR20Z8D0aXcxbJvmfMgSSq1hkcgvFRXJk9rUq5H/MJ0Ktal869hhOdI/zUTeLA==
   dependencies:
-    playwright-core "1.43.1"
+    playwright-core "1.45.0"
   optionalDependencies:
     fsevents "2.3.2"