diff --git a/.env.sample b/.env.sample index 14f74df..6cee384 100644 --- a/.env.sample +++ b/.env.sample @@ -21,6 +21,7 @@ ARCHIVE_TAKE_COUNT= BROWSER_TIMEOUT= IGNORE_UNAUTHORIZED_CA= IGNORE_HTTPS_ERRORS= +IGNORE_URL_SIZE_LIMIT= # AWS S3 Settings SPACES_KEY= diff --git a/lib/api/validateUrlSize.ts b/lib/api/validateUrlSize.ts index eae0b43..d5826b8 100644 --- a/lib/api/validateUrlSize.ts +++ b/lib/api/validateUrlSize.ts @@ -1,17 +1,35 @@ import fetch from "node-fetch"; import https from "https"; +import { SocksProxyAgent } from "socks-proxy-agent"; export default async function validateUrlSize(url: string) { + if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null; + try { const httpsAgent = new https.Agent({ rejectUnauthorized: process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true, }); - const response = await fetch(url, { + let fetchOpts = { method: "HEAD", agent: httpsAgent, - }); + }; + + if (process.env.PROXY) { + let proxy = new URL(process.env.PROXY); + if (process.env.PROXY_USERNAME) { + proxy.username = process.env.PROXY_USERNAME; + proxy.password = process.env.PROXY_PASSWORD || ""; + } + + fetchOpts = { + method: "HEAD", + agent: new SocksProxyAgent(proxy.toString()), + }; + } + + const response = await fetch(url, fetchOpts); const totalSizeMB = Number(response.headers.get("content-length")) / Math.pow(1024, 2); diff --git a/lib/shared/getTitle.ts b/lib/shared/getTitle.ts index 82fee37..01488fd 100644 --- a/lib/shared/getTitle.ts +++ b/lib/shared/getTitle.ts @@ -27,14 +27,25 @@ export default async function getTitle(url: string) { fetchOpts = { agent: new SocksProxyAgent(proxy.toString()) }; //TODO: add support for http/https proxies } - const response = await fetch(url, fetchOpts); + const responsePromise = fetch(url, fetchOpts); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject(new Error("Fetch title timeout")); + }, 10 * 1000); // Stop after 10 seconds + }); - const text = await response.text(); + const response = await Promise.race([responsePromise, timeoutPromise]); - // regular expression to find the