diff --git a/.env.sample b/.env.sample
index ad56075..14f74df 100644
--- a/.env.sample
+++ b/.env.sample
@@ -35,6 +35,15 @@ NEXT_PUBLIC_EMAIL_PROVIDER=
EMAIL_FROM=
EMAIL_SERVER=
+# Proxy settings
+PROXY=
+PROXY_USERNAME=
+PROXY_PASSWORD=
+PROXY_BYPASS=
+
+# PDF archive settings
+PDF_MARGIN_TOP=
+PDF_MARGIN_BOTTOM=
#
# SSO Providers
diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.ts
index c43b6db..96a77b2 100644
--- a/lib/api/archiveHandler.ts
+++ b/lib/api/archiveHandler.ts
@@ -1,4 +1,4 @@
-import { chromium, devices } from "playwright";
+import { LaunchOptions, chromium, devices } from "playwright";
import { prisma } from "./db";
import createFile from "./storage/createFile";
import sendToWayback from "./sendToWayback";
@@ -20,11 +20,23 @@ type LinksAndCollectionAndOwner = Link & {
const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;
export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
- const browser = await chromium.launch();
+ // allow user to configure a proxy
+ let browserOptions: LaunchOptions = {};
+ if (process.env.PROXY) {
+ browserOptions.proxy = {
+ server: process.env.PROXY,
+ bypass: process.env.PROXY_BYPASS,
+ username: process.env.PROXY_USERNAME,
+ password: process.env.PROXY_PASSWORD,
+ };
+ }
+
+ const browser = await chromium.launch(browserOptions);
const context = await browser.newContext({
...devices["Desktop Chrome"],
ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",
});
+
const page = await context.newPage();
const timeoutPromise = new Promise((_, reject) => {
@@ -241,6 +253,13 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
})
);
}
+
+ // apply administrator's defined pdf margins or default to 15px
+ const margins = {
+ top: process.env.PDF_MARGIN_TOP || "15px",
+ bottom: process.env.PDF_MARGIN_BOTTOM || "15px",
+ };
+
if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {
processingPromises.push(
page
@@ -248,7 +267,7 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
width: "1366px",
height: "1931px",
printBackground: true,
- margin: { top: "15px", bottom: "15px" },
+ margin: margins,
})
.then((pdf) => {
return createFile({
diff --git a/lib/shared/getTitle.ts b/lib/shared/getTitle.ts
index 4c2a5d0..82fee37 100644
--- a/lib/shared/getTitle.ts
+++ b/lib/shared/getTitle.ts
@@ -1,5 +1,7 @@
import fetch from "node-fetch";
import https from "https";
+import { SocksProxyAgent } from "socks-proxy-agent";
+
export default async function getTitle(url: string) {
try {
const httpsAgent = new https.Agent({
@@ -7,9 +9,26 @@ export default async function getTitle(url: string) {
process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true,
});
- const response = await fetch(url, {
+ // fetchOpts allows a proxy to be defined
+ let fetchOpts = {
agent: httpsAgent,
- });
+ };
+
+ if (process.env.PROXY) {
+ // parse proxy url
+ let proxy = new URL(process.env.PROXY);
+ // if authentication set, apply to proxy URL
+ if (process.env.PROXY_USERNAME) {
+ proxy.username = process.env.PROXY_USERNAME;
+ proxy.password = process.env.PROXY_PASSWORD || "";
+ }
+
+ // add socks5 proxy to fetchOpts
+ fetchOpts = { agent: new SocksProxyAgent(proxy.toString()) }; //TODO: add support for http/https proxies
+ }
+
+ const response = await fetch(url, fetchOpts);
+
const text = await response.text();
// regular expression to find the
tag
diff --git a/package.json b/package.json
index 7199f79..8e5b900 100644
--- a/package.json
+++ b/package.json
@@ -51,6 +51,7 @@
"micro": "^10.0.1",
"next": "13.4.12",
"next-auth": "^4.22.1",
+ "node-fetch": "^2.7.0",
"nodemailer": "^6.9.3",
"playwright": "^1.35.1",
"react": "18.2.0",
@@ -59,6 +60,7 @@
"react-hot-toast": "^2.4.1",
"react-image-file-resizer": "^0.4.8",
"react-select": "^5.7.4",
+ "socks-proxy-agent": "^8.0.2",
"stripe": "^12.13.0",
"vaul": "^0.8.8",
"zustand": "^4.3.8"
diff --git a/types/enviornment.d.ts b/types/enviornment.d.ts
index 58d74c5..a0295c9 100644
--- a/types/enviornment.d.ts
+++ b/types/enviornment.d.ts
@@ -36,6 +36,16 @@ declare global {
NEXT_PUBLIC_TRIAL_PERIOD_DAYS?: string;
BASE_URL?: string;
+ // Proxy settings
+ PROXY?: string;
+ PROXY_USERNAME?: string;
+ PROXY_PASSWORD?: string;
+ PROXY_BYPASS?: string;
+
+ // PDF archive settings
+ PDF_MARGIN_TOP?: string;
+ PDF_MARGIN_BOTTOM?: string;
+
//
// SSO Providers
//
diff --git a/yarn.lock b/yarn.lock
index 7f3b8b8..1b4d92a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2113,6 +2113,13 @@ agent-base@6:
dependencies:
debug "4"
+agent-base@^7.0.2:
+ version "7.1.0"
+ resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.0.tgz#536802b76bc0b34aa50195eb2442276d613e3434"
+ integrity sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==
+ dependencies:
+ debug "^4.3.4"
+
ajv@^6.12.3, ajv@^6.12.4:
version "6.12.6"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
@@ -3852,6 +3859,14 @@ iota-array@^1.0.0:
resolved "https://registry.yarnpkg.com/iota-array/-/iota-array-1.0.0.tgz#81ef57fe5d05814cd58c2483632a99c30a0e8087"
integrity sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA==
+ip-address@^9.0.5:
+ version "9.0.5"
+ resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-9.0.5.tgz#117a960819b08780c3bd1f14ef3c1cc1d3f3ea5a"
+ integrity sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==
+ dependencies:
+ jsbn "1.1.0"
+ sprintf-js "^1.1.3"
+
is-arguments@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.1.1.tgz#15b3f88fda01f2a97fec84ca761a560f123efa9b"
@@ -4116,6 +4131,11 @@ js-yaml@^4.1.0:
dependencies:
argparse "^2.0.1"
+jsbn@1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040"
+ integrity sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==
+
jsbn@~0.1.0:
version "0.1.1"
resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513"
@@ -4474,7 +4494,7 @@ node-bitmap@0.0.1:
resolved "https://registry.yarnpkg.com/node-bitmap/-/node-bitmap-0.0.1.tgz#180eac7003e0c707618ef31368f62f84b2a69091"
integrity sha512-Jx5lPaaLdIaOsj2mVLWMWulXF6GQVdyLvNSxmiYCvZ8Ma2hfKX0POoR2kgKOqz+oFsRreq0yYZjQ2wjE9VNzCA==
-node-fetch@^2.6.1:
+node-fetch@^2.6.1, node-fetch@^2.7.0:
version "2.7.0"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
@@ -5362,6 +5382,28 @@ slash@^4.0.0:
resolved "https://registry.yarnpkg.com/slash/-/slash-4.0.0.tgz#2422372176c4c6c5addb5e2ada885af984b396a7"
integrity sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==
+smart-buffer@^4.2.0:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae"
+ integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==
+
+socks-proxy-agent@^8.0.2:
+ version "8.0.2"
+ resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz#5acbd7be7baf18c46a3f293a840109a430a640ad"
+ integrity sha512-8zuqoLv1aP/66PHF5TqwJ7Czm3Yv32urJQHrVyhD7mmA6d61Zv8cIXQYPTWwmg6qlupnPvs/QKDmfa4P/qct2g==
+ dependencies:
+ agent-base "^7.0.2"
+ debug "^4.3.4"
+ socks "^2.7.1"
+
+socks@^2.7.1:
+ version "2.7.3"
+ resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.3.tgz#7d8a75d7ce845c0a96f710917174dba0d543a785"
+ integrity sha512-vfuYK48HXCTFD03G/1/zkIls3Ebr2YNa4qU9gHDZdblHLiqhJrJGkY3+0Nx0JpN9qBhJbVObc1CNciT1bIZJxw==
+ dependencies:
+ ip-address "^9.0.5"
+ smart-buffer "^4.2.0"
+
source-map-js@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c"
@@ -5377,6 +5419,11 @@ spawn-command@0.0.2:
resolved "https://registry.yarnpkg.com/spawn-command/-/spawn-command-0.0.2.tgz#9544e1a43ca045f8531aac1a48cb29bdae62338e"
integrity sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==
+sprintf-js@^1.1.3:
+ version "1.1.3"
+ resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.3.tgz#4914b903a2f8b685d17fdf78a70e917e872e444a"
+ integrity sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==
+
sshpk@^1.7.0:
version "1.17.0"
resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5"