Merge pull request #478 from go-compile/main

feat: Socks5 proxy support

.env.sample

@@ -35,6 +35,15 @@ NEXT_PUBLIC_EMAIL_PROVIDER=
 EMAIL_FROM=
 EMAIL_SERVER=
 
+# Proxy settings
+PROXY=
+PROXY_USERNAME=
+PROXY_PASSWORD=
+PROXY_BYPASS=
+
+# PDF archive settings
+PDF_MARGIN_TOP=
+PDF_MARGIN_BOTTOM=
 
 #
 # SSO Providers

lib/api/archiveHandler.ts

@@ -1,4 +1,4 @@
-import { chromium, devices } from "playwright";
+import { LaunchOptions, chromium, devices } from "playwright";
 import { prisma } from "./db";
 import createFile from "./storage/createFile";
 import sendToWayback from "./sendToWayback";
@@ -20,11 +20,23 @@ type LinksAndCollectionAndOwner = Link & {
 const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;
 
 export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
-  const browser = await chromium.launch();
+  // allow user to configure a proxy
+  let browserOptions: LaunchOptions = {};
+  if (process.env.PROXY) {
+    browserOptions.proxy = {
+      server: process.env.PROXY,
+      bypass: process.env.PROXY_BYPASS,
+      username: process.env.PROXY_USERNAME,
+      password: process.env.PROXY_PASSWORD,
+    };
+  }
+
+  const browser = await chromium.launch(browserOptions);
   const context = await browser.newContext({
     ...devices["Desktop Chrome"],
     ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",
   });
+
   const page = await context.newPage();
 
   const timeoutPromise = new Promise((_, reject) => {
@@ -241,6 +253,13 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
                 })
               );
             }
+
+            // apply administrator's defined pdf margins or default to 15px
+            const margins = {
+              top: process.env.PDF_MARGIN_TOP || "15px",
+              bottom: process.env.PDF_MARGIN_BOTTOM || "15px",
+            };
+
             if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {
               processingPromises.push(
                 page
@@ -248,7 +267,7 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
                     width: "1366px",
                     height: "1931px",
                     printBackground: true,
-                    margin: { top: "15px", bottom: "15px" },
+                    margin: margins,
                   })
                   .then((pdf) => {
                     return createFile({

lib/shared/getTitle.ts

@@ -1,5 +1,7 @@
 import fetch from "node-fetch";
 import https from "https";
+import { SocksProxyAgent } from "socks-proxy-agent";
+
 export default async function getTitle(url: string) {
   try {
     const httpsAgent = new https.Agent({
@@ -7,9 +9,26 @@ export default async function getTitle(url: string) {
         process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true,
     });
 
-    const response = await fetch(url, {
+    // fetchOpts allows a proxy to be defined
+    let fetchOpts = {
       agent: httpsAgent,
-    });
+    };
+
+    if (process.env.PROXY) {
+      // parse proxy url
+      let proxy = new URL(process.env.PROXY);
+      // if authentication set, apply to proxy URL
+      if (process.env.PROXY_USERNAME) {
+        proxy.username = process.env.PROXY_USERNAME;
+        proxy.password = process.env.PROXY_PASSWORD || "";
+      }
+
+      // add socks5 proxy to fetchOpts
+      fetchOpts = { agent: new SocksProxyAgent(proxy.toString()) }; //TODO: add support for http/https proxies
+    }
+
+    const response = await fetch(url, fetchOpts);
+
     const text = await response.text();
 
     // regular expression to find the <title> tag

package.json

@@ -51,6 +51,7 @@
     "micro": "^10.0.1",
     "next": "13.4.12",
     "next-auth": "^4.22.1",
+    "node-fetch": "^2.7.0",
     "nodemailer": "^6.9.3",
     "playwright": "^1.35.1",
     "react": "18.2.0",
@@ -59,6 +60,7 @@
     "react-hot-toast": "^2.4.1",
     "react-image-file-resizer": "^0.4.8",
     "react-select": "^5.7.4",
+    "socks-proxy-agent": "^8.0.2",
     "stripe": "^12.13.0",
     "vaul": "^0.8.8",
     "zustand": "^4.3.8"

types/enviornment.d.ts

@@ -36,6 +36,16 @@ declare global {
       NEXT_PUBLIC_TRIAL_PERIOD_DAYS?: string;
       BASE_URL?: string;
 
+      // Proxy settings
+      PROXY?: string;
+      PROXY_USERNAME?: string;
+      PROXY_PASSWORD?: string;
+      PROXY_BYPASS?: string;
+
+      // PDF archive settings
+      PDF_MARGIN_TOP?: string;
+      PDF_MARGIN_BOTTOM?: string;
+
       //
       // SSO Providers
       //

yarn.lock

@@ -2113,6 +2113,13 @@ agent-base@6:
   dependencies:
     debug "4"
 
+agent-base@^7.0.2:
+  version "7.1.0"
+  resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.0.tgz#536802b76bc0b34aa50195eb2442276d613e3434"
+  integrity sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==
+  dependencies:
+    debug "^4.3.4"
+
 ajv@^6.12.3, ajv@^6.12.4:
   version "6.12.6"
   resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
@@ -3852,6 +3859,14 @@ iota-array@^1.0.0:
   resolved "https://registry.yarnpkg.com/iota-array/-/iota-array-1.0.0.tgz#81ef57fe5d05814cd58c2483632a99c30a0e8087"
   integrity sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA==
 
+ip-address@^9.0.5:
+  version "9.0.5"
+  resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-9.0.5.tgz#117a960819b08780c3bd1f14ef3c1cc1d3f3ea5a"
+  integrity sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==
+  dependencies:
+    jsbn "1.1.0"
+    sprintf-js "^1.1.3"
+
 is-arguments@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.1.1.tgz#15b3f88fda01f2a97fec84ca761a560f123efa9b"
@@ -4116,6 +4131,11 @@ js-yaml@^4.1.0:
   dependencies:
     argparse "^2.0.1"
 
+jsbn@1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040"
+  integrity sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==
+
 jsbn@~0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513"
@@ -4474,7 +4494,7 @@ node-bitmap@0.0.1:
   resolved "https://registry.yarnpkg.com/node-bitmap/-/node-bitmap-0.0.1.tgz#180eac7003e0c707618ef31368f62f84b2a69091"
   integrity sha512-Jx5lPaaLdIaOsj2mVLWMWulXF6GQVdyLvNSxmiYCvZ8Ma2hfKX0POoR2kgKOqz+oFsRreq0yYZjQ2wjE9VNzCA==
 
-node-fetch@^2.6.1:
+node-fetch@^2.6.1, node-fetch@^2.7.0:
   version "2.7.0"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
   integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
@@ -5362,6 +5382,28 @@ slash@^4.0.0:
   resolved "https://registry.yarnpkg.com/slash/-/slash-4.0.0.tgz#2422372176c4c6c5addb5e2ada885af984b396a7"
   integrity sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==
 
+smart-buffer@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae"
+  integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==
+
+socks-proxy-agent@^8.0.2:
+  version "8.0.2"
+  resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz#5acbd7be7baf18c46a3f293a840109a430a640ad"
+  integrity sha512-8zuqoLv1aP/66PHF5TqwJ7Czm3Yv32urJQHrVyhD7mmA6d61Zv8cIXQYPTWwmg6qlupnPvs/QKDmfa4P/qct2g==
+  dependencies:
+    agent-base "^7.0.2"
+    debug "^4.3.4"
+    socks "^2.7.1"
+
+socks@^2.7.1:
+  version "2.7.3"
+  resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.3.tgz#7d8a75d7ce845c0a96f710917174dba0d543a785"
+  integrity sha512-vfuYK48HXCTFD03G/1/zkIls3Ebr2YNa4qU9gHDZdblHLiqhJrJGkY3+0Nx0JpN9qBhJbVObc1CNciT1bIZJxw==
+  dependencies:
+    ip-address "^9.0.5"
+    smart-buffer "^4.2.0"
+
 source-map-js@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c"
@@ -5377,6 +5419,11 @@ spawn-command@0.0.2:
   resolved "https://registry.yarnpkg.com/spawn-command/-/spawn-command-0.0.2.tgz#9544e1a43ca045f8531aac1a48cb29bdae62338e"
   integrity sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==
 
+sprintf-js@^1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.3.tgz#4914b903a2f8b685d17fdf78a70e917e872e444a"
+  integrity sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==
+
 sshpk@^1.7.0:
   version "1.17.0"
   resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5"