From 1c9d626d087dd9dbd9dedd992f11186a212a8d45 Mon Sep 17 00:00:00 2001
From: Daniel <daniel31x13@gmail.com>
Date: Sun, 3 Jul 2022 21:19:46 +0430
Subject: [PATCH] Improved archiving.

---
 api/modules/getData.js | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/api/modules/getData.js b/api/modules/getData.js
index dcb6567..2cf047e 100644
--- a/api/modules/getData.js
+++ b/api/modules/getData.js
@@ -5,7 +5,7 @@ const { screenshotDirectory, pdfDirectory } = require("../config.js");
 
 module.exports = async (link, id) => {
   const browser = await puppeteer.launch({
-    args: ['--no-sandbox'],
+    args: ["--no-sandbox"],
     timeout: 10000,
   });
   const page = await browser.newPage();
@@ -14,7 +14,14 @@ module.exports = async (link, id) => {
     blocker.enableBlockingInPage(page);
   });
 
-  await page.goto(link, { waitUntil: "load", timeout: 0 });
+  await page.goto(link, { waitUntil: "load", timeout: 300000 });
+
+  await page.setViewport({
+    width: 1200,
+    height: 800,
+  });
+
+  await autoScroll(page);
 
   await page.screenshot({
     path: screenshotDirectory + "/" + id + ".png",
@@ -24,3 +31,24 @@ module.exports = async (link, id) => {
 
   await browser.close();
 };
+
+async function autoScroll(page) {
+  await page.evaluate(async () => {
+    await new Promise((resolve, reject) => {
+      let totalHeight = 0;
+      let distance = 100;
+      let timer = setInterval(() => {
+        let scrollHeight = document.body.scrollHeight;
+        window.scrollBy(0, distance);
+        totalHeight += distance;
+
+        if (totalHeight >= scrollHeight - window.innerHeight) {
+          clearInterval(timer);
+          resolve();
+        }
+      }, 100);
+    });
+
+    window.scrollTo(0,0);
+  });
+}