From 8cf621bc62eefeb8e069f6c5b5664ba2c1b7fb1c Mon Sep 17 00:00:00 2001 From: daniel31x13 Date: Wed, 17 Apr 2024 18:02:54 -0400 Subject: [PATCH 1/2] added a new env var + bug fixed --- .env.sample | 1 + lib/api/validateUrlSize.ts | 22 ++++++++++++++++++++-- lib/shared/getTitle.ts | 23 +++++++++++++++++------ types/enviornment.d.ts | 1 + 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/.env.sample b/.env.sample index 14f74df..6cee384 100644 --- a/.env.sample +++ b/.env.sample @@ -21,6 +21,7 @@ ARCHIVE_TAKE_COUNT= BROWSER_TIMEOUT= IGNORE_UNAUTHORIZED_CA= IGNORE_HTTPS_ERRORS= +IGNORE_URL_SIZE_LIMIT= # AWS S3 Settings SPACES_KEY= diff --git a/lib/api/validateUrlSize.ts b/lib/api/validateUrlSize.ts index eae0b43..d5826b8 100644 --- a/lib/api/validateUrlSize.ts +++ b/lib/api/validateUrlSize.ts @@ -1,17 +1,35 @@ import fetch from "node-fetch"; import https from "https"; +import { SocksProxyAgent } from "socks-proxy-agent"; export default async function validateUrlSize(url: string) { + if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null; + try { const httpsAgent = new https.Agent({ rejectUnauthorized: process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true, }); - const response = await fetch(url, { + let fetchOpts = { method: "HEAD", agent: httpsAgent, - }); + }; + + if (process.env.PROXY) { + let proxy = new URL(process.env.PROXY); + if (process.env.PROXY_USERNAME) { + proxy.username = process.env.PROXY_USERNAME; + proxy.password = process.env.PROXY_PASSWORD || ""; + } + + fetchOpts = { + method: "HEAD", + agent: new SocksProxyAgent(proxy.toString()), + }; + } + + const response = await fetch(url, fetchOpts); const totalSizeMB = Number(response.headers.get("content-length")) / Math.pow(1024, 2); diff --git a/lib/shared/getTitle.ts b/lib/shared/getTitle.ts index 82fee37..01488fd 100644 --- a/lib/shared/getTitle.ts +++ b/lib/shared/getTitle.ts @@ -27,14 +27,25 @@ export default async function getTitle(url: string) { fetchOpts = { agent: new SocksProxyAgent(proxy.toString()) }; //TODO: add support for http/https proxies } - const response = await fetch(url, fetchOpts); + const responsePromise = fetch(url, fetchOpts); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject(new Error("Fetch title timeout")); + }, 10 * 1000); // Stop after 10 seconds + }); - const text = await response.text(); + const response = await Promise.race([responsePromise, timeoutPromise]); - // regular expression to find the tag - let match = text.match(/<title.*>([^<]*)<\/title>/); - if (match) return match[1]; - else return ""; + if ((response as any)?.status) { + const text = await (response as any).text(); + + // regular expression to find the <title> tag + let match = text.match(/<title.*>([^<]*)<\/title>/); + if (match) return match[1]; + else return ""; + } else { + return ""; + } } catch (err) { console.log(err); } diff --git a/types/enviornment.d.ts b/types/enviornment.d.ts index a0295c9..b737915 100644 --- a/types/enviornment.d.ts +++ b/types/enviornment.d.ts @@ -13,6 +13,7 @@ declare global { MAX_LINKS_PER_USER?: string; ARCHIVE_TAKE_COUNT?: string; IGNORE_UNAUTHORIZED_CA?: string; + IGNORE_URL_SIZE_LIMIT?: string; SPACES_KEY?: string; SPACES_SECRET?: string; From e8edd1c9a06acc28a9a976ffe7e4753d266d272c Mon Sep 17 00:00:00 2001 From: daniel31x13 <daniel31x13@gmail.com> Date: Wed, 17 Apr 2024 18:06:04 -0400 Subject: [PATCH 2/2] update version number --- components/SettingsSidebar.tsx | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/SettingsSidebar.tsx b/components/SettingsSidebar.tsx index 9e169df..ca6f6cf 100644 --- a/components/SettingsSidebar.tsx +++ b/components/SettingsSidebar.tsx @@ -4,7 +4,7 @@ import { useRouter } from "next/router"; import React, { useEffect, useState } from "react"; export default function SettingsSidebar({ className }: { className?: string }) { - const LINKWARDEN_VERSION = "v2.5.1"; + const LINKWARDEN_VERSION = "v2.5.2"; const { collections } = useCollectionStore(); diff --git a/package.json b/package.json index 4568588..c8f48ec 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "linkwarden", - "version": "2.5.1", + "version": "0.0.0", "main": "index.js", "repository": "https://github.com/linkwarden/linkwarden.git", "author": "Daniel31X13 <daniel31x13@gmail.com>",