throw error if a link was blocking the queue for too long
This commit is contained in:
parent
ee6dcdcc5b
commit
f1dadf1546
|
@ -18,6 +18,7 @@ RE_ARCHIVE_LIMIT=
|
|||
NEXT_PUBLIC_MAX_FILE_SIZE=
|
||||
MAX_LINKS_PER_USER=
|
||||
ARCHIVE_TAKE_COUNT=
|
||||
BROWSER_TIMEOUT=
|
||||
|
||||
# AWS S3 Settings
|
||||
SPACES_KEY=
|
||||
|
|
|
@ -17,13 +17,31 @@ type LinksAndCollectionAndOwner = Link & {
|
|||
};
|
||||
};
|
||||
|
||||
const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;
|
||||
|
||||
export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
|
||||
const browser = await chromium.launch();
|
||||
const browser = await chromium.launch({ headless: false });
|
||||
const context = await browser.newContext(devices["Desktop Chrome"]);
|
||||
const page = await context.newPage();
|
||||
|
||||
const timeoutPromise = new Promise((_, reject) => {
|
||||
setTimeout(
|
||||
() =>
|
||||
reject(
|
||||
new Error(
|
||||
`Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`
|
||||
)
|
||||
),
|
||||
BROWSER_TIMEOUT * 60000
|
||||
);
|
||||
});
|
||||
|
||||
try {
|
||||
const validatedUrl = link.url ? await validateUrlSize(link.url) : undefined;
|
||||
await Promise.race([
|
||||
(async () => {
|
||||
const validatedUrl = link.url
|
||||
? await validateUrlSize(link.url)
|
||||
: undefined;
|
||||
|
||||
if (validatedUrl === null) throw "File is too large to be stored.";
|
||||
|
||||
|
@ -56,8 +74,12 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
|
|||
user.archiveAsPDF && !link.pdf?.startsWith("archive")
|
||||
? "pending"
|
||||
: undefined,
|
||||
readable: !link.readable?.startsWith("archive") ? "pending" : undefined,
|
||||
preview: !link.readable?.startsWith("archive") ? "pending" : undefined,
|
||||
readable: !link.readable?.startsWith("archive")
|
||||
? "pending"
|
||||
: undefined,
|
||||
preview: !link.readable?.startsWith("archive")
|
||||
? "pending"
|
||||
: undefined,
|
||||
lastPreserved: new Date().toISOString(),
|
||||
},
|
||||
});
|
||||
|
@ -202,7 +224,10 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
|
|||
if (linkExists) {
|
||||
const processingPromises = [];
|
||||
|
||||
if (user.archiveAsScreenshot && !link.image?.startsWith("archive")) {
|
||||
if (
|
||||
user.archiveAsScreenshot &&
|
||||
!link.image?.startsWith("archive")
|
||||
) {
|
||||
processingPromises.push(
|
||||
page.screenshot({ fullPage: true }).then((screenshot) => {
|
||||
return createFile({
|
||||
|
@ -243,6 +268,9 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
|
|||
});
|
||||
}
|
||||
}
|
||||
})(),
|
||||
timeoutPromise,
|
||||
]);
|
||||
} catch (err) {
|
||||
console.log(err);
|
||||
console.log("Failed Link details:", link);
|
||||
|
|
|
@ -12,6 +12,7 @@ declare global {
|
|||
NEXT_PUBLIC_MAX_FILE_SIZE?: string;
|
||||
MAX_LINKS_PER_USER?: string;
|
||||
ARCHIVE_TAKE_COUNT?: string;
|
||||
NEXT_PUBLIC_MAX_FILE_SIZE?: string;
|
||||
|
||||
SPACES_KEY?: string;
|
||||
SPACES_SECRET?: string;
|
||||
|
|
Ŝarĝante…
Reference in New Issue