diff --git a/.env.sample b/.env.sample index ad56075..14f74df 100644 --- a/.env.sample +++ b/.env.sample @@ -35,6 +35,15 @@ NEXT_PUBLIC_EMAIL_PROVIDER= EMAIL_FROM= EMAIL_SERVER= +# Proxy settings +PROXY= +PROXY_USERNAME= +PROXY_PASSWORD= +PROXY_BYPASS= + +# PDF archive settings +PDF_MARGIN_TOP= +PDF_MARGIN_BOTTOM= # # SSO Providers diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.ts index c43b6db..96a77b2 100644 --- a/lib/api/archiveHandler.ts +++ b/lib/api/archiveHandler.ts @@ -1,4 +1,4 @@ -import { chromium, devices } from "playwright"; +import { LaunchOptions, chromium, devices } from "playwright"; import { prisma } from "./db"; import createFile from "./storage/createFile"; import sendToWayback from "./sendToWayback"; @@ -20,11 +20,23 @@ type LinksAndCollectionAndOwner = Link & { const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5; export default async function archiveHandler(link: LinksAndCollectionAndOwner) { - const browser = await chromium.launch(); + // allow user to configure a proxy + let browserOptions: LaunchOptions = {}; + if (process.env.PROXY) { + browserOptions.proxy = { + server: process.env.PROXY, + bypass: process.env.PROXY_BYPASS, + username: process.env.PROXY_USERNAME, + password: process.env.PROXY_PASSWORD, + }; + } + + const browser = await chromium.launch(browserOptions); const context = await browser.newContext({ ...devices["Desktop Chrome"], ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true", }); + const page = await context.newPage(); const timeoutPromise = new Promise((_, reject) => { @@ -241,6 +253,13 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) { }) ); } + + // apply administrator's defined pdf margins or default to 15px + const margins = { + top: process.env.PDF_MARGIN_TOP || "15px", + bottom: process.env.PDF_MARGIN_BOTTOM || "15px", + }; + if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) { processingPromises.push( page @@ -248,7 +267,7 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) { width: "1366px", height: "1931px", printBackground: true, - margin: { top: "15px", bottom: "15px" }, + margin: margins, }) .then((pdf) => { return createFile({ diff --git a/lib/shared/getTitle.ts b/lib/shared/getTitle.ts index 4c2a5d0..82fee37 100644 --- a/lib/shared/getTitle.ts +++ b/lib/shared/getTitle.ts @@ -1,5 +1,7 @@ import fetch from "node-fetch"; import https from "https"; +import { SocksProxyAgent } from "socks-proxy-agent"; + export default async function getTitle(url: string) { try { const httpsAgent = new https.Agent({ @@ -7,9 +9,26 @@ export default async function getTitle(url: string) { process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true, }); - const response = await fetch(url, { + // fetchOpts allows a proxy to be defined + let fetchOpts = { agent: httpsAgent, - }); + }; + + if (process.env.PROXY) { + // parse proxy url + let proxy = new URL(process.env.PROXY); + // if authentication set, apply to proxy URL + if (process.env.PROXY_USERNAME) { + proxy.username = process.env.PROXY_USERNAME; + proxy.password = process.env.PROXY_PASSWORD || ""; + } + + // add socks5 proxy to fetchOpts + fetchOpts = { agent: new SocksProxyAgent(proxy.toString()) }; //TODO: add support for http/https proxies + } + + const response = await fetch(url, fetchOpts); + const text = await response.text(); // regular expression to find the tag diff --git a/package.json b/package.json index 7199f79..8e5b900 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "micro": "^10.0.1", "next": "13.4.12", "next-auth": "^4.22.1", + "node-fetch": "^2.7.0", "nodemailer": "^6.9.3", "playwright": "^1.35.1", "react": "18.2.0", @@ -59,6 +60,7 @@ "react-hot-toast": "^2.4.1", "react-image-file-resizer": "^0.4.8", "react-select": "^5.7.4", + "socks-proxy-agent": "^8.0.2", "stripe": "^12.13.0", "vaul": "^0.8.8", "zustand": "^4.3.8" diff --git a/types/enviornment.d.ts b/types/enviornment.d.ts index 58d74c5..a0295c9 100644 --- a/types/enviornment.d.ts +++ b/types/enviornment.d.ts @@ -36,6 +36,16 @@ declare global { NEXT_PUBLIC_TRIAL_PERIOD_DAYS?: string; BASE_URL?: string; + // Proxy settings + PROXY?: string; + PROXY_USERNAME?: string; + PROXY_PASSWORD?: string; + PROXY_BYPASS?: string; + + // PDF archive settings + PDF_MARGIN_TOP?: string; + PDF_MARGIN_BOTTOM?: string; + // // SSO Providers // diff --git a/yarn.lock b/yarn.lock index 7f3b8b8..1b4d92a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2113,6 +2113,13 @@ agent-base@6: dependencies: debug "4" +agent-base@^7.0.2: + version "7.1.0" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.0.tgz#536802b76bc0b34aa50195eb2442276d613e3434" + integrity sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg== + dependencies: + debug "^4.3.4" + ajv@^6.12.3, ajv@^6.12.4: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" @@ -3852,6 +3859,14 @@ iota-array@^1.0.0: resolved "https://registry.yarnpkg.com/iota-array/-/iota-array-1.0.0.tgz#81ef57fe5d05814cd58c2483632a99c30a0e8087" integrity sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA== +ip-address@^9.0.5: + version "9.0.5" + resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-9.0.5.tgz#117a960819b08780c3bd1f14ef3c1cc1d3f3ea5a" + integrity sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g== + dependencies: + jsbn "1.1.0" + sprintf-js "^1.1.3" + is-arguments@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.1.1.tgz#15b3f88fda01f2a97fec84ca761a560f123efa9b" @@ -4116,6 +4131,11 @@ js-yaml@^4.1.0: dependencies: argparse "^2.0.1" +jsbn@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040" + integrity sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A== + jsbn@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" @@ -4474,7 +4494,7 @@ node-bitmap@0.0.1: resolved "https://registry.yarnpkg.com/node-bitmap/-/node-bitmap-0.0.1.tgz#180eac7003e0c707618ef31368f62f84b2a69091" integrity sha512-Jx5lPaaLdIaOsj2mVLWMWulXF6GQVdyLvNSxmiYCvZ8Ma2hfKX0POoR2kgKOqz+oFsRreq0yYZjQ2wjE9VNzCA== -node-fetch@^2.6.1: +node-fetch@^2.6.1, node-fetch@^2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== @@ -5362,6 +5382,28 @@ slash@^4.0.0: resolved "https://registry.yarnpkg.com/slash/-/slash-4.0.0.tgz#2422372176c4c6c5addb5e2ada885af984b396a7" integrity sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew== +smart-buffer@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" + integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== + +socks-proxy-agent@^8.0.2: + version "8.0.2" + resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz#5acbd7be7baf18c46a3f293a840109a430a640ad" + integrity sha512-8zuqoLv1aP/66PHF5TqwJ7Czm3Yv32urJQHrVyhD7mmA6d61Zv8cIXQYPTWwmg6qlupnPvs/QKDmfa4P/qct2g== + dependencies: + agent-base "^7.0.2" + debug "^4.3.4" + socks "^2.7.1" + +socks@^2.7.1: + version "2.7.3" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.3.tgz#7d8a75d7ce845c0a96f710917174dba0d543a785" + integrity sha512-vfuYK48HXCTFD03G/1/zkIls3Ebr2YNa4qU9gHDZdblHLiqhJrJGkY3+0Nx0JpN9qBhJbVObc1CNciT1bIZJxw== + dependencies: + ip-address "^9.0.5" + smart-buffer "^4.2.0" + source-map-js@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" @@ -5377,6 +5419,11 @@ spawn-command@0.0.2: resolved "https://registry.yarnpkg.com/spawn-command/-/spawn-command-0.0.2.tgz#9544e1a43ca045f8531aac1a48cb29bdae62338e" integrity sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ== +sprintf-js@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.3.tgz#4914b903a2f8b685d17fdf78a70e917e872e444a" + integrity sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA== + sshpk@^1.7.0: version "1.17.0" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5"