Merge pull request #567 from linkwarden/hotfix/title-fetching

Hotfix/title fetching

.env.sample

@@ -21,6 +21,7 @@ ARCHIVE_TAKE_COUNT=
 BROWSER_TIMEOUT=
 IGNORE_UNAUTHORIZED_CA=
 IGNORE_HTTPS_ERRORS=
+IGNORE_URL_SIZE_LIMIT=
 
 # AWS S3 Settings
 SPACES_KEY=

components/SettingsSidebar.tsx

@@ -4,7 +4,7 @@ import { useRouter } from "next/router";
 import React, { useEffect, useState } from "react";
 
 export default function SettingsSidebar({ className }: { className?: string }) {
-  const LINKWARDEN_VERSION = "v2.5.1";
+  const LINKWARDEN_VERSION = "v2.5.2";
 
   const { collections } = useCollectionStore();
 

lib/api/validateUrlSize.ts

@@ -1,17 +1,35 @@
 import fetch from "node-fetch";
 import https from "https";
+import { SocksProxyAgent } from "socks-proxy-agent";
 
 export default async function validateUrlSize(url: string) {
+  if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null;
+
   try {
     const httpsAgent = new https.Agent({
       rejectUnauthorized:
         process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true,
     });
 
-    const response = await fetch(url, {
+    let fetchOpts = {
       method: "HEAD",
       agent: httpsAgent,
-    });
+    };
+
+    if (process.env.PROXY) {
+      let proxy = new URL(process.env.PROXY);
+      if (process.env.PROXY_USERNAME) {
+        proxy.username = process.env.PROXY_USERNAME;
+        proxy.password = process.env.PROXY_PASSWORD || "";
+      }
+
+      fetchOpts = {
+        method: "HEAD",
+        agent: new SocksProxyAgent(proxy.toString()),
+      };
+    }
+
+    const response = await fetch(url, fetchOpts);
 
     const totalSizeMB =
       Number(response.headers.get("content-length")) / Math.pow(1024, 2);

lib/shared/getTitle.ts

@@ -27,14 +27,25 @@ export default async function getTitle(url: string) {
       fetchOpts = { agent: new SocksProxyAgent(proxy.toString()) }; //TODO: add support for http/https proxies
     }
 
-    const response = await fetch(url, fetchOpts);
+    const responsePromise = fetch(url, fetchOpts);
+    const timeoutPromise = new Promise((_, reject) => {
+      setTimeout(() => {
+        reject(new Error("Fetch title timeout"));
+      }, 10 * 1000); // Stop after 10 seconds
+    });
 
-    const text = await response.text();
+    const response = await Promise.race([responsePromise, timeoutPromise]);
+
+    if ((response as any)?.status) {
+      const text = await (response as any).text();
 
       // regular expression to find the <title> tag
       let match = text.match(/<title.*>([^<]*)<\/title>/);
       if (match) return match[1];
       else return "";
+    } else {
+      return "";
+    }
   } catch (err) {
     console.log(err);
   }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "linkwarden",
-  "version": "2.5.1",
+  "version": "0.0.0",
   "main": "index.js",
   "repository": "https://github.com/linkwarden/linkwarden.git",
   "author": "Daniel31X13 <daniel31x13@gmail.com>",

types/enviornment.d.ts

@@ -13,6 +13,7 @@ declare global {
       MAX_LINKS_PER_USER?: string;
       ARCHIVE_TAKE_COUNT?: string;
       IGNORE_UNAUTHORIZED_CA?: string;
+      IGNORE_URL_SIZE_LIMIT?: string;
 
       SPACES_KEY?: string;
       SPACES_SECRET?: string;