xwx.moe/el.xwx.moe
xwx.moe/ el.xwx.moe
1
0
Disbranĉigi 0
Fontkodo Issues Tirpetoj Actions Packages Projects Eldonoj Wiki Activity

throw error if a link was blocking the queue for too long

This commit is contained in:
daniel31x13 2023-12-29 23:59:00 -05:00
parent ee6dcdcc5b
commit f1dadf1546
3 changed files with 232 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.env.sample
View File

@ -18,6 +18,7 @@ RE_ARCHIVE_LIMIT=
NEXT_PUBLIC_MAX_FILE_SIZE= NEXT_PUBLIC_MAX_FILE_SIZE=
MAX_LINKS_PER_USER= MAX_LINKS_PER_USER=
ARCHIVE_TAKE_COUNT= ARCHIVE_TAKE_COUNT=
BROWSER_TIMEOUT=
# AWS S3 Settings # AWS S3 Settings
SPACES_KEY= SPACES_KEY=

38
lib/api/archiveHandler.ts
View File

@ -17,13 +17,31 @@ type LinksAndCollectionAndOwner = Link & {
}; };
}; };
const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;
export default async function archiveHandler(link: LinksAndCollectionAndOwner) { export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
const browser = await chromium.launch(); const browser = await chromium.launch({ headless: false });
const context = await browser.newContext(devices["Desktop Chrome"]); const context = await browser.newContext(devices["Desktop Chrome"]);
const page = await context.newPage(); const page = await context.newPage();
const timeoutPromise = new Promise((_, reject) => {
setTimeout(
() =>
reject(
new Error(
`Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`
)
),
BROWSER_TIMEOUT * 60000
);
});
try { try {
const validatedUrl = link.url ? await validateUrlSize(link.url) : undefined; await Promise.race([
(async () => {
const validatedUrl = link.url
? await validateUrlSize(link.url)
: undefined;
if (validatedUrl === null) throw "File is too large to be stored."; if (validatedUrl === null) throw "File is too large to be stored.";
@ -56,8 +74,12 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
user.archiveAsPDF && !link.pdf?.startsWith("archive") user.archiveAsPDF && !link.pdf?.startsWith("archive")
? "pending" ? "pending"
: undefined, : undefined,
readable: !link.readable?.startsWith("archive") ? "pending" : undefined, readable: !link.readable?.startsWith("archive")
preview: !link.readable?.startsWith("archive") ? "pending" : undefined, ? "pending"
: undefined,
preview: !link.readable?.startsWith("archive")
? "pending"
: undefined,
lastPreserved: new Date().toISOString(), lastPreserved: new Date().toISOString(),
}, },
}); });
@ -202,7 +224,10 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
if (linkExists) { if (linkExists) {
const processingPromises = []; const processingPromises = [];
if (user.archiveAsScreenshot && !link.image?.startsWith("archive")) { if (
user.archiveAsScreenshot &&
!link.image?.startsWith("archive")
) {
processingPromises.push( processingPromises.push(
page.screenshot({ fullPage: true }).then((screenshot) => { page.screenshot({ fullPage: true }).then((screenshot) => {
return createFile({ return createFile({
@ -243,6 +268,9 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
}); });
} }
} }
})(),
timeoutPromise,
]);
} catch (err) { } catch (err) {
console.log(err); console.log(err);
console.log("Failed Link details:", link); console.log("Failed Link details:", link);

1
types/enviornment.d.ts vendored
View File

@ -12,6 +12,7 @@ declare global {
NEXT_PUBLIC_MAX_FILE_SIZE?: string; NEXT_PUBLIC_MAX_FILE_SIZE?: string;
MAX_LINKS_PER_USER?: string; MAX_LINKS_PER_USER?: string;
ARCHIVE_TAKE_COUNT?: string; ARCHIVE_TAKE_COUNT?: string;
NEXT_PUBLIC_MAX_FILE_SIZE?: string;
SPACES_KEY?: string; SPACES_KEY?: string;
SPACES_SECRET?: string; SPACES_SECRET?: string;