Merge pull request #562 from linkwarden/hotfix/title-fetching

added a new env var + bug fixed

.env.sample

@@ -21,6 +21,7 @@ ARCHIVE_TAKE_COUNT=
 BROWSER_TIMEOUT=
 IGNORE_UNAUTHORIZED_CA=
 IGNORE_HTTPS_ERRORS=
+IGNORE_URL_SIZE_LIMIT=
 
 # AWS S3 Settings
 SPACES_KEY=

lib/api/validateUrlSize.ts

@@ -1,17 +1,35 @@
 import fetch from "node-fetch";
 import https from "https";
+import { SocksProxyAgent } from "socks-proxy-agent";
 
 export default async function validateUrlSize(url: string) {
+  if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null;
+
   try {
     const httpsAgent = new https.Agent({
       rejectUnauthorized:
         process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true,
     });
 
-    const response = await fetch(url, {
+    let fetchOpts = {
       method: "HEAD",
       agent: httpsAgent,
-    });
+    };
+
+    if (process.env.PROXY) {
+      let proxy = new URL(process.env.PROXY);
+      if (process.env.PROXY_USERNAME) {
+        proxy.username = process.env.PROXY_USERNAME;
+        proxy.password = process.env.PROXY_PASSWORD || "";
+      }
+
+      fetchOpts = {
+        method: "HEAD",
+        agent: new SocksProxyAgent(proxy.toString()),
+      };
+    }
+
+    const response = await fetch(url, fetchOpts);
 
     const totalSizeMB =
       Number(response.headers.get("content-length")) / Math.pow(1024, 2);

lib/shared/getTitle.ts

@@ -27,14 +27,25 @@ export default async function getTitle(url: string) {
       fetchOpts = { agent: new SocksProxyAgent(proxy.toString()) }; //TODO: add support for http/https proxies
     }
 
-    const response = await fetch(url, fetchOpts);
+    const responsePromise = fetch(url, fetchOpts);
+    const timeoutPromise = new Promise((_, reject) => {
+      setTimeout(() => {
+        reject(new Error("Fetch title timeout"));
+      }, 10 * 1000); // Stop after 10 seconds
+    });
 
-    const text = await response.text();
+    const response = await Promise.race([responsePromise, timeoutPromise]);
 
-    // regular expression to find the <title> tag
+    if ((response as any)?.status) {
-    let match = text.match(/<title.*>([^<]*)<\/title>/);
+      const text = await (response as any).text();
-    if (match) return match[1];
+
-    else return "";
+      // regular expression to find the <title> tag
+      let match = text.match(/<title.*>([^<]*)<\/title>/);
+      if (match) return match[1];
+      else return "";
+    } else {
+      return "";
+    }
   } catch (err) {
     console.log(err);
   }

types/enviornment.d.ts

@@ -13,6 +13,7 @@ declare global {
       MAX_LINKS_PER_USER?: string;
       ARCHIVE_TAKE_COUNT?: string;
       IGNORE_UNAUTHORIZED_CA?: string;
+      IGNORE_URL_SIZE_LIMIT?: string;
 
       SPACES_KEY?: string;
       SPACES_SECRET?: string;