code refactoring

This commit is contained in:
daniel31x13 2024-06-26 13:54:03 -04:00
parent c68f9d68ad
commit 3618ba907d
7 changed files with 229 additions and 160 deletions

View File

@ -124,7 +124,7 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {
clearInterval(interval);
}
};
}, [link, getLink, link?.singlefile]);
}, [link?.singlefile]);
const updateArchive = async () => {
const load = toast.loading(t("sending_request"));
@ -210,10 +210,7 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {
className="mx-auto mb-3"
size={20}
/>
<p className="text-center">
There are more preserved formats in the queue
</p>
<p className="text-center">{t("there_are_more_formats")}</p>
<p className="text-center text-sm">{t("check_back_later")}</p>
</div>
) : undefined}

View File

@ -1,4 +1,4 @@
import { LaunchOptions, chromium, devices } from "playwright";
import { LaunchOptions, Page, chromium, devices } from "playwright";
import { prisma } from "./db";
import createFile from "./storage/createFile";
import sendToWayback from "./preservationScheme/sendToWayback";
@ -9,6 +9,7 @@ import generatePreview from "./generatePreview";
import { removeFiles } from "./manageLinkFiles";
import archiveAsSinglefile from "./preservationScheme/archiveAsSinglefile";
import archiveAsReadability from "./preservationScheme/archiveAsReadablility";
import shell from "shelljs";
type LinksAndCollectionAndOwner = Link & {
collection: Collection & {
@ -50,6 +51,26 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
const page = await context.newPage();
// await page.goto("https://github.com", {
// waitUntil: "domcontentloaded",
// });
// console.log("Opening page:", link.url);
// await page.evaluate(autoScroll, Number(process.env.AUTOSCROLL_TIMEOUT) || 30);
// const dom = await page.content();
// console.log("The content", dom);
// shell
// .echo(dom)
// .exec(
// "monolith - -I -b https://marketplace.visualstudio.com/items?itemName=42Crunch.vscode-openapi -j -F -o monolith.html"
// );
// console.log("Monolith created!");
createFolder({
filePath: `archives/preview/${link.collectionId}`,
});
@ -111,13 +132,13 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
});
// SingleFile
if (
!link.singlefile?.startsWith("archive") &&
!link.singlefile?.startsWith("unavailable") &&
user.archiveAsSinglefile &&
link.url
)
await archiveAsSinglefile(link);
// if (
// !link.singlefile?.startsWith("archive") &&
// !link.singlefile?.startsWith("unavailable") &&
// user.archiveAsSinglefile &&
// link.url
// )
// await archiveAsSinglefile(link);
// send to archive.org
if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url);
@ -131,13 +152,6 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
} else if (link.url) {
// archive url
const context = await browser.newContext({
...devices["Desktop Chrome"],
ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",
});
const page = await context.newPage();
await page.goto(link.url, { waitUntil: "domcontentloaded" });
const content = await page.content();
@ -150,115 +164,20 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
await archiveAsReadability(content, link);
// Preview
if (
!link.preview?.startsWith("archives") &&
!link.preview?.startsWith("unavailable")
)
await getArchivePreview(link, page);
const ogImageUrl = await page.evaluate(() => {
const metaTag = document.querySelector('meta[property="og:image"]');
return metaTag ? (metaTag as any).content : null;
});
if (ogImageUrl) {
console.log("Found og:image URL:", ogImageUrl);
// Download the image
const imageResponse = await page.goto(ogImageUrl);
// Check if imageResponse is not null
if (imageResponse && !link.preview?.startsWith("archive")) {
const buffer = await imageResponse.body();
await generatePreview(buffer, link.collectionId, link.id);
}
await page.goBack();
} else if (!link.preview?.startsWith("archive")) {
console.log("No og:image found");
await page
.screenshot({ type: "jpeg", quality: 20 })
.then((screenshot) => {
return createFile({
data: screenshot,
filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,
});
})
.then(() => {
return prisma.link.update({
where: { id: link.id },
data: {
preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,
},
});
});
}
}
if (
(!link.image?.startsWith("archives") &&
!link.image?.startsWith("unavailable")) ||
(!link.pdf?.startsWith("archives") &&
!link.pdf?.startsWith("unavailable"))
) {
// Screenshot/PDF
await page.evaluate(
autoScroll,
Number(process.env.AUTOSCROLL_TIMEOUT) || 30
);
// Check if the user hasn't deleted the link by the time we're done scrolling
const linkExists = await prisma.link.findUnique({
where: { id: link.id },
});
if (linkExists) {
const processingPromises = [];
if (
user.archiveAsScreenshot &&
!link.image?.startsWith("archive")
) {
processingPromises.push(
page.screenshot({ fullPage: true }).then((screenshot) => {
return createFile({
data: screenshot,
filePath: `archives/${linkExists.collectionId}/${link.id}.png`,
});
})
);
}
// apply administrator's defined pdf margins or default to 15px
const margins = {
top: process.env.PDF_MARGIN_TOP || "15px",
bottom: process.env.PDF_MARGIN_BOTTOM || "15px",
};
if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {
processingPromises.push(
page
.pdf({
width: "1366px",
height: "1931px",
printBackground: true,
margin: margins,
})
.then((pdf) => {
return createFile({
data: pdf,
filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`,
});
})
);
}
await Promise.allSettled(processingPromises);
await prisma.link.update({
where: { id: link.id },
data: {
image: user.archiveAsScreenshot
? `archives/${linkExists.collectionId}/${link.id}.png`
: undefined,
pdf: user.archiveAsPDF
? `archives/${linkExists.collectionId}/${link.id}.pdf`
: undefined,
},
});
}
if (
(!link.image?.startsWith("archives") &&
!link.image?.startsWith("unavailable")) ||
(!link.pdf?.startsWith("archives") &&
!link.pdf?.startsWith("unavailable"))
)
await captureScreenshotAndPdf(link, page, user);
}
})(),
timeoutPromise,
@ -302,31 +221,6 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
}
}
const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => {
const timeoutPromise = new Promise<void>((_, reject) => {
setTimeout(() => {
reject(new Error(`Webpage was too long to be archived.`));
}, AUTOSCROLL_TIMEOUT * 1000);
});
const scrollingPromise = new Promise<void>((resolve) => {
let totalHeight = 0;
let distance = 100;
let scrollDown = setInterval(() => {
let scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if (totalHeight >= scrollHeight) {
clearInterval(scrollDown);
window.scroll(0, 0);
resolve();
}
}, 100);
});
await Promise.race([scrollingPromise, timeoutPromise]);
};
const imageHandler = async ({ url, id }: Link, extension: string) => {
const image = await fetch(url as string).then((res) => res.blob());
@ -374,3 +268,133 @@ const pdfHandler = async ({ url, id }: Link) => {
});
}
};
const getArchivePreview = async (
link: LinksAndCollectionAndOwner,
page: Page
) => {
const ogImageUrl = await page.evaluate(() => {
const metaTag = document.querySelector('meta[property="og:image"]');
return metaTag ? (metaTag as any).content : null;
});
if (ogImageUrl) {
console.log("Found og:image URL:", ogImageUrl);
// Download the image
const imageResponse = await page.goto(ogImageUrl);
// Check if imageResponse is not null
if (imageResponse && !link.preview?.startsWith("archive")) {
const buffer = await imageResponse.body();
generatePreview(buffer, link.collectionId, link.id);
}
await page.goBack();
} else if (!link.preview?.startsWith("archive")) {
console.log("No og:image found");
await page
.screenshot({ type: "jpeg", quality: 20 })
.then((screenshot) => {
return createFile({
data: screenshot,
filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,
});
})
.then(() => {
return prisma.link.update({
where: { id: link.id },
data: {
preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,
},
});
});
}
};
const captureScreenshotAndPdf = async (
link: LinksAndCollectionAndOwner,
page: Page,
user: User
) => {
await page.evaluate(autoScroll, Number(process.env.AUTOSCROLL_TIMEOUT) || 30);
// Check if the user hasn't deleted the link by the time we're done scrolling
const linkExists = await prisma.link.findUnique({
where: { id: link.id },
});
if (linkExists) {
const processingPromises = [];
if (user.archiveAsScreenshot && !link.image?.startsWith("archive")) {
processingPromises.push(
page.screenshot({ fullPage: true, type: "png" }).then((screenshot) => {
return createFile({
data: screenshot,
filePath: `archives/${linkExists.collectionId}/${link.id}.png`,
});
})
);
}
const margins = {
top: process.env.PDF_MARGIN_TOP || "15px",
bottom: process.env.PDF_MARGIN_BOTTOM || "15px",
};
if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {
processingPromises.push(
page
.pdf({
width: "1366px",
height: "1931px",
printBackground: true,
margin: margins,
})
.then((pdf) => {
return createFile({
data: pdf,
filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`,
});
})
);
}
await Promise.allSettled(processingPromises);
await prisma.link.update({
where: { id: link.id },
data: {
image: user.archiveAsScreenshot
? `archives/${linkExists.collectionId}/${link.id}.png`
: undefined,
pdf: user.archiveAsPDF
? `archives/${linkExists.collectionId}/${link.id}.pdf`
: undefined,
},
});
}
};
const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => {
const timeoutPromise = new Promise<void>((resolve) => {
setTimeout(() => {
resolve();
}, AUTOSCROLL_TIMEOUT * 1000);
});
const scrollingPromise = new Promise<void>((resolve) => {
let totalHeight = 0;
let distance = 100;
let scrollDown = setInterval(() => {
let scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if (totalHeight >= scrollHeight) {
clearInterval(scrollDown);
window.scroll(0, 0);
resolve();
}
}, 100);
});
await Promise.race([scrollingPromise, timeoutPromise]);
};

View File

@ -1,7 +1,6 @@
import Jimp from "jimp";
import { prisma } from "./db";
import createFile from "./storage/createFile";
import createFolder from "./storage/createFolder";
const generatePreview = async (
buffer: Buffer,

View File

@ -14,11 +14,9 @@ const archiveAsReadablility = async (content: string, link: Link) => {
const articleText = article?.textContent
.replace(/ +(?= )/g, "") // strip out multiple spaces
.replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaks
if (
articleText &&
articleText !== "" &&
!link.readable?.startsWith("archive")
) {
console.log(articleText);
if (articleText && articleText !== "") {
const collectionId = (
await prisma.link.findUnique({
where: { id: link.id },

View File

@ -70,6 +70,7 @@
"react-masonry-css": "^1.0.16",
"react-select": "^5.7.4",
"react-spinners": "^0.13.8",
"shelljs": "^0.8.5",
"socks-proxy-agent": "^8.0.2",
"stripe": "^12.13.0",
"tailwind-merge": "^2.3.0",

View File

@ -220,8 +220,9 @@
"github": "GitHub",
"twitter": "Twitter",
"mastodon": "Mastodon",
"link_preservation_in_queue": "LThe Link preservation is currently in the queue",
"link_preservation_in_queue": "The Link preservation is currently in the queue",
"check_back_later": "Please check back later to see the result",
"there_are_more_formats": "There are more preserved formats in the queue",
"settings": "Settings",
"switch_to": "Switch to {{theme}}",
"logout": "Logout",

View File

@ -3514,6 +3514,11 @@ function-bind@^1.1.1:
resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==
function-bind@^1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c"
integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==
function.prototype.name@^1.1.5:
version "1.1.5"
resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.5.tgz#cce0505fe1ffb80503e6f9e46cc64e46a12a9621"
@ -3651,7 +3656,7 @@ glob@7.1.7:
once "^1.3.0"
path-is-absolute "^1.0.0"
glob@^7.1.3:
glob@^7.0.0, glob@^7.1.3:
version "7.2.3"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b"
integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==
@ -3821,6 +3826,13 @@ has@^1.0.3:
dependencies:
function-bind "^1.1.1"
hasown@^2.0.0:
version "2.0.2"
resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003"
integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==
dependencies:
function-bind "^1.1.2"
hexoid@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/hexoid/-/hexoid-1.0.0.tgz#ad10c6573fb907de23d9ec63a711267d9dc9bc18"
@ -3972,6 +3984,11 @@ internal-slot@^1.0.3, internal-slot@^1.0.4:
has "^1.0.3"
side-channel "^1.0.4"
interpret@^1.0.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e"
integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==
invariant@^2.2.4:
version "2.2.4"
resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6"
@ -4053,6 +4070,13 @@ is-core-module@^2.10.0, is-core-module@^2.11.0, is-core-module@^2.9.0:
dependencies:
has "^1.0.3"
is-core-module@^2.13.0:
version "2.13.1"
resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.1.tgz#ad0d7532c6fea9da1ebdc82742d74525c6273384"
integrity sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==
dependencies:
hasown "^2.0.0"
is-date-object@^1.0.1, is-date-object@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.5.tgz#0841d5536e724c25597bf6ea62e1bd38298df31f"
@ -5372,6 +5396,13 @@ readdirp@~3.6.0:
dependencies:
picomatch "^2.2.1"
rechoir@^0.6.2:
version "0.6.2"
resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384"
integrity sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==
dependencies:
resolve "^1.1.6"
redux@^4.0.0, redux@^4.0.1:
version "4.2.1"
resolved "https://registry.yarnpkg.com/redux/-/redux-4.2.1.tgz#c08f4306826c49b5e9dc901dee0452ea8fce6197"
@ -5439,6 +5470,15 @@ resolve-from@^4.0.0:
resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"
integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==
resolve@^1.1.6:
version "1.22.8"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d"
integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==
dependencies:
is-core-module "^2.13.0"
path-parse "^1.0.7"
supports-preserve-symlinks-flag "^1.0.0"
resolve@^1.1.7, resolve@^1.19.0, resolve@^1.22.1, resolve@^1.22.2:
version "1.22.2"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.2.tgz#0ed0943d4e301867955766c9f3e1ae6d01c6845f"
@ -5565,6 +5605,15 @@ shell-quote@^1.8.1:
resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.8.1.tgz#6dbf4db75515ad5bac63b4f1894c3a154c766680"
integrity sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==
shelljs@^0.8.5:
version "0.8.5"
resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.5.tgz#de055408d8361bed66c669d2f000538ced8ee20c"
integrity sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==
dependencies:
glob "^7.0.0"
interpret "^1.0.0"
rechoir "^0.6.2"
side-channel@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf"