diff --git a/.env.sample b/.env.sample index 14f74df..6cee384 100644 --- a/.env.sample +++ b/.env.sample @@ -21,6 +21,7 @@ ARCHIVE_TAKE_COUNT= BROWSER_TIMEOUT= IGNORE_UNAUTHORIZED_CA= IGNORE_HTTPS_ERRORS= +IGNORE_URL_SIZE_LIMIT= # AWS S3 Settings SPACES_KEY= diff --git a/lib/api/validateUrlSize.ts b/lib/api/validateUrlSize.ts index eae0b43..d5826b8 100644 --- a/lib/api/validateUrlSize.ts +++ b/lib/api/validateUrlSize.ts @@ -1,17 +1,35 @@ import fetch from "node-fetch"; import https from "https"; +import { SocksProxyAgent } from "socks-proxy-agent"; export default async function validateUrlSize(url: string) { + if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null; + try { const httpsAgent = new https.Agent({ rejectUnauthorized: process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true, }); - const response = await fetch(url, { + let fetchOpts = { method: "HEAD", agent: httpsAgent, - }); + }; + + if (process.env.PROXY) { + let proxy = new URL(process.env.PROXY); + if (process.env.PROXY_USERNAME) { + proxy.username = process.env.PROXY_USERNAME; + proxy.password = process.env.PROXY_PASSWORD || ""; + } + + fetchOpts = { + method: "HEAD", + agent: new SocksProxyAgent(proxy.toString()), + }; + } + + const response = await fetch(url, fetchOpts); const totalSizeMB = Number(response.headers.get("content-length")) / Math.pow(1024, 2); diff --git a/lib/shared/getTitle.ts b/lib/shared/getTitle.ts index 82fee37..01488fd 100644 --- a/lib/shared/getTitle.ts +++ b/lib/shared/getTitle.ts @@ -27,14 +27,25 @@ export default async function getTitle(url: string) { fetchOpts = { agent: new SocksProxyAgent(proxy.toString()) }; //TODO: add support for http/https proxies } - const response = await fetch(url, fetchOpts); + const responsePromise = fetch(url, fetchOpts); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject(new Error("Fetch title timeout")); + }, 10 * 1000); // Stop after 10 seconds + }); - const text = await response.text(); + const response = await Promise.race([responsePromise, timeoutPromise]); - // regular expression to find the tag - let match = text.match(/<title.*>([^<]*)<\/title>/); - if (match) return match[1]; - else return ""; + if ((response as any)?.status) { + const text = await (response as any).text(); + + // regular expression to find the <title> tag + let match = text.match(/<title.*>([^<]*)<\/title>/); + if (match) return match[1]; + else return ""; + } else { + return ""; + } } catch (err) { console.log(err); } diff --git a/types/enviornment.d.ts b/types/enviornment.d.ts index a0295c9..b737915 100644 --- a/types/enviornment.d.ts +++ b/types/enviornment.d.ts @@ -13,6 +13,7 @@ declare global { MAX_LINKS_PER_USER?: string; ARCHIVE_TAKE_COUNT?: string; IGNORE_UNAUTHORIZED_CA?: string; + IGNORE_URL_SIZE_LIMIT?: string; SPACES_KEY?: string; SPACES_SECRET?: string;