Improved archiving.

api/modules/getData.js

@@ -5,7 +5,7 @@ const { screenshotDirectory, pdfDirectory } = require("../config.js");
 
 module.exports = async (link, id) => {
   const browser = await puppeteer.launch({
-    args: ['--no-sandbox'],
+    args: ["--no-sandbox"],
     timeout: 10000,
   });
   const page = await browser.newPage();
@@ -14,7 +14,14 @@ module.exports = async (link, id) => {
     blocker.enableBlockingInPage(page);
   });
 
-  await page.goto(link, { waitUntil: "load", timeout: 0 });
+  await page.goto(link, { waitUntil: "load", timeout: 300000 });
+
+  await page.setViewport({
+    width: 1200,
+    height: 800,
+  });
+
+  await autoScroll(page);
 
   await page.screenshot({
     path: screenshotDirectory + "/" + id + ".png",
@@ -24,3 +31,24 @@ module.exports = async (link, id) => {
 
   await browser.close();
 };
+
+async function autoScroll(page) {
+  await page.evaluate(async () => {
+    await new Promise((resolve, reject) => {
+      let totalHeight = 0;
+      let distance = 100;
+      let timer = setInterval(() => {
+        let scrollHeight = document.body.scrollHeight;
+        window.scrollBy(0, distance);
+        totalHeight += distance;
+
+        if (totalHeight >= scrollHeight - window.innerHeight) {
+          clearInterval(timer);
+          resolve();
+        }
+      }, 100);
+    });
+
+    window.scrollTo(0,0);
+  });
+}