throw error if a link was blocking the queue for too long

This commit is contained in:
daniel31x13 2023-12-29 23:59:00 -05:00
parent ee6dcdcc5b
commit f1dadf1546
3 changed files with 232 additions and 202 deletions

View File

@ -18,6 +18,7 @@ RE_ARCHIVE_LIMIT=
NEXT_PUBLIC_MAX_FILE_SIZE=
MAX_LINKS_PER_USER=
ARCHIVE_TAKE_COUNT=
BROWSER_TIMEOUT=
# AWS S3 Settings
SPACES_KEY=

View File

@ -17,13 +17,31 @@ type LinksAndCollectionAndOwner = Link & {
};
};
const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;
export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
const browser = await chromium.launch();
const browser = await chromium.launch({ headless: false });
const context = await browser.newContext(devices["Desktop Chrome"]);
const page = await context.newPage();
const timeoutPromise = new Promise((_, reject) => {
setTimeout(
() =>
reject(
new Error(
`Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`
)
),
BROWSER_TIMEOUT * 60000
);
});
try {
const validatedUrl = link.url ? await validateUrlSize(link.url) : undefined;
await Promise.race([
(async () => {
const validatedUrl = link.url
? await validateUrlSize(link.url)
: undefined;
if (validatedUrl === null) throw "File is too large to be stored.";
@ -56,8 +74,12 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
user.archiveAsPDF && !link.pdf?.startsWith("archive")
? "pending"
: undefined,
readable: !link.readable?.startsWith("archive") ? "pending" : undefined,
preview: !link.readable?.startsWith("archive") ? "pending" : undefined,
readable: !link.readable?.startsWith("archive")
? "pending"
: undefined,
preview: !link.readable?.startsWith("archive")
? "pending"
: undefined,
lastPreserved: new Date().toISOString(),
},
});
@ -202,7 +224,10 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
if (linkExists) {
const processingPromises = [];
if (user.archiveAsScreenshot && !link.image?.startsWith("archive")) {
if (
user.archiveAsScreenshot &&
!link.image?.startsWith("archive")
) {
processingPromises.push(
page.screenshot({ fullPage: true }).then((screenshot) => {
return createFile({
@ -243,6 +268,9 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
});
}
}
})(),
timeoutPromise,
]);
} catch (err) {
console.log(err);
console.log("Failed Link details:", link);

View File

@ -12,6 +12,7 @@ declare global {
NEXT_PUBLIC_MAX_FILE_SIZE?: string;
MAX_LINKS_PER_USER?: string;
ARCHIVE_TAKE_COUNT?: string;
NEXT_PUBLIC_MAX_FILE_SIZE?: string;
SPACES_KEY?: string;
SPACES_SECRET?: string;