diff --git a/.env.sample b/.env.sample index 30bc044..48d62e9 100644 --- a/.env.sample +++ b/.env.sample @@ -15,7 +15,6 @@ NEXT_PUBLIC_DISABLE_REGISTRATION= NEXT_PUBLIC_CREDENTIALS_ENABLED= DISABLE_NEW_SSO_USERS= RE_ARCHIVE_LIMIT= -NEXT_PUBLIC_MAX_FILE_SIZE= MAX_LINKS_PER_USER= ARCHIVE_TAKE_COUNT= BROWSER_TIMEOUT= @@ -23,6 +22,13 @@ IGNORE_UNAUTHORIZED_CA= IGNORE_HTTPS_ERRORS= IGNORE_URL_SIZE_LIMIT= ADMINISTRATOR= +NEXT_PUBLIC_MAX_FILE_BUFFER= +MONOLITH_MAX_BUFFER= +MONOLITH_CUSTOM_OPTIONS= +PDF_MAX_BUFFER= +SCREENSHOT_MAX_BUFFER= +READABILITY_MAX_BUFFER= +PREVIEW_MAX_BUFFER= # AWS S3 Settings SPACES_KEY= @@ -48,9 +54,9 @@ PROXY_BYPASS= PDF_MARGIN_TOP= PDF_MARGIN_BOTTOM= -# -# SSO Providers -# +################# +# SSO Providers # +################# # 42 School NEXT_PUBLIC_FORTYTWO_ENABLED= diff --git a/Dockerfile b/Dockerfile index 9de1541..d51b65d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,30 @@ WORKDIR /data COPY ./package.json ./yarn.lock ./playwright.config.ts ./ -# Increase timeout to pass github actions arm64 build RUN --mount=type=cache,sharing=locked,target=/usr/local/share/.cache/yarn yarn install --network-timeout 10000000 +RUN apt-get update + +RUN apt-get install -y \ + build-essential \ + curl \ + libssl-dev \ + pkg-config + +RUN apt-get update + +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y + +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN cargo install monolith + RUN npx playwright install-deps && \ apt-get clean && \ yarn cache clean +RUN yarn playwright install + COPY . . RUN yarn prisma generate && \ diff --git a/README.md b/README.md index cd3725a..c3f0bc2 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ We've forked the old version from the current repository into [this repo](https: ## Features -- 📸 Auto capture a screenshot, PDF, and readable view of each webpage. +- 📸 Auto capture a screenshot, PDF, single html file, and readable view of each webpage. - 🏛️ Send your webpage to Wayback Machine ([archive.org](https://archive.org)) for a snapshot. (Optional) - 📂 Organize links by collection, sub-collection, name, description and multiple tags. - 👥 Collaborate on gathering links in a collection. diff --git a/components/Announcement.tsx b/components/Announcement.tsx index 2f97297..5486f8b 100644 --- a/components/Announcement.tsx +++ b/components/Announcement.tsx @@ -10,8 +10,8 @@ export default function Announcement({ toggleAnnouncementBar }: Props) { const announcementId = localStorage.getItem("announcementId"); return ( -
-
+
+

- ) : undefined} + ) : // : link.type === "monolith" ? ( + // + // ) + undefined} ); } diff --git a/components/ModalContent/EditCollectionSharingModal.tsx b/components/ModalContent/EditCollectionSharingModal.tsx index 3e349ed..9a9f5fa 100644 --- a/components/ModalContent/EditCollectionSharingModal.tsx +++ b/components/ModalContent/EditCollectionSharingModal.tsx @@ -68,6 +68,7 @@ export default function EditCollectionSharingModal({ username: "", image: "", archiveAsScreenshot: undefined as unknown as boolean, + archiveAsMonolith: undefined as unknown as boolean, archiveAsPDF: undefined as unknown as boolean, }); diff --git a/components/ModalContent/NewLinkModal.tsx b/components/ModalContent/NewLinkModal.tsx index e2fc181..0284687 100644 --- a/components/ModalContent/NewLinkModal.tsx +++ b/components/ModalContent/NewLinkModal.tsx @@ -30,6 +30,7 @@ export default function NewLinkModal({ onClose }: Props) { image: "", pdf: "", readable: "", + monolith: "", textContent: "", collection: { name: "", diff --git a/components/ModalContent/PreservedFormatsModal.tsx b/components/ModalContent/PreservedFormatsModal.tsx index 66ae53c..87e84a7 100644 --- a/components/ModalContent/PreservedFormatsModal.tsx +++ b/components/ModalContent/PreservedFormatsModal.tsx @@ -12,12 +12,14 @@ import { useSession } from "next-auth/react"; import { pdfAvailable, readabilityAvailable, + monolithAvailable, screenshotAvailable, } from "@/lib/shared/getArchiveValidity"; import PreservedFormatRow from "@/components/PreserverdFormatRow"; import useAccountStore from "@/store/account"; import getPublicUserData from "@/lib/client/getPublicUserData"; import { useTranslation } from "next-i18next"; +import { BeatLoader } from "react-spinners"; type Props = { onClose: Function; @@ -41,6 +43,7 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) { username: "", image: "", archiveAsScreenshot: undefined as unknown as boolean, + archiveAsMonolith: undefined as unknown as boolean, archiveAsPDF: undefined as unknown as boolean, }); @@ -58,6 +61,7 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) { username: account.username as string, image: account.image as string, archiveAsScreenshot: account.archiveAsScreenshot as boolean, + archiveAsMonolith: account.archiveAsScreenshot as boolean, archiveAsPDF: account.archiveAsPDF as boolean, }); } @@ -72,6 +76,9 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) { (collectionOwner.archiveAsScreenshot === true ? link.pdf && link.pdf !== "pending" : true) && + (collectionOwner.archiveAsMonolith === true + ? link.monolith && link.monolith !== "pending" + : true) && (collectionOwner.archiveAsPDF === true ? link.pdf && link.pdf !== "pending" : true) && @@ -80,6 +87,15 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) { ); }; + const atLeastOneFormatAvailable = () => { + return ( + screenshotAvailable(link) || + pdfAvailable(link) || + readabilityAvailable(link) || + monolithAvailable(link) + ); + }; + useEffect(() => { (async () => { const data = await getLink(link.id as number, isPublic); @@ -108,7 +124,7 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) { clearInterval(interval); } }; - }, [link, getLink]); + }, [link?.monolith]); const updateArchive = async () => { const load = toast.loading(t("sending_request")); @@ -133,56 +149,81 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {

{t("preserved_formats")}

- {isReady() && - (screenshotAvailable(link) || - pdfAvailable(link) || - readabilityAvailable(link)) ? ( + {screenshotAvailable(link) || + pdfAvailable(link) || + readabilityAvailable(link) || + monolithAvailable(link) ? (

{t("available_formats")}

) : ( "" )} -
- {isReady() ? ( - <> - {screenshotAvailable(link) ? ( - - ) : undefined} - {pdfAvailable(link) ? ( - - ) : undefined} - {readabilityAvailable(link) ? ( - - ) : undefined} - - ) : ( -
- +
+ {monolithAvailable(link) ? ( + + ) : undefined} + + {screenshotAvailable(link) ? ( + + ) : undefined} + + {pdfAvailable(link) ? ( + + ) : undefined} + + {readabilityAvailable(link) ? ( + + ) : undefined} + + {!isReady() && !atLeastOneFormatAvailable() ? ( +
+ +

{t("preservation_in_queue")}

{t("check_back_later")}

- )} + ) : !isReady() && atLeastOneFormatAvailable() ? ( +
+ +

{t("there_are_more_formats")}

+

{t("check_back_later")}

+
+ ) : undefined}
{ if (!submitLoader && file) { + let fileType: ArchivedFormat | null = null; + let linkType: "url" | "image" | "monolith" | "pdf" | null = null; + + if (file?.type === "image/jpg" || file.type === "image/jpeg") { + fileType = ArchivedFormat.jpeg; + linkType = "image"; + } else if (file.type === "image/png") { + fileType = ArchivedFormat.png; + linkType = "image"; + } else if (file.type === "application/pdf") { + fileType = ArchivedFormat.pdf; + linkType = "pdf"; + } + // else if (file.type === "text/html") { + // fileType = ArchivedFormat.monolith; + // linkType = "monolith"; + // } + setSubmitLoader(true); const load = toast.loading(t("creating")); @@ -122,14 +144,14 @@ export default function UploadFileModal({ onClose }: Props) {

{t("file_types", { - size: process.env.NEXT_PUBLIC_MAX_FILE_SIZE || 30, + size: process.env.NEXT_PUBLIC_MAX_FILE_BUFFER || 10, })}

diff --git a/components/PreserverdFormatRow.tsx b/components/PreserverdFormatRow.tsx index e7100cd..ce0f21d 100644 --- a/components/PreserverdFormatRow.tsx +++ b/components/PreserverdFormatRow.tsx @@ -4,6 +4,7 @@ import { ArchivedFormat, LinkIncludingShortenedCollectionAndTags, } from "@/types/global"; +import toast from "react-hot-toast"; import Link from "next/link"; import { useRouter } from "next/router"; import { useSession } from "next-auth/react"; @@ -60,7 +61,7 @@ export default function PreservedFormatRow({ clearInterval(interval); } }; - }, [link?.image, link?.pdf, link?.readable]); + }, [link?.image, link?.pdf, link?.readable, link?.monolith]); const handleDownload = () => { const path = `/api/v1/archives/${link?.id}?format=${format}`; @@ -68,10 +69,15 @@ export default function PreservedFormatRow({ .then((response) => { if (response.ok) { // Create a temporary link and click it to trigger the download - const link = document.createElement("a"); - link.href = path; - link.download = format === ArchivedFormat.pdf ? "PDF" : "Screenshot"; - link.click(); + const anchorElement = document.createElement("a"); + anchorElement.href = path; + anchorElement.download = + format === ArchivedFormat.monolith + ? "Webpage" + : format === ArchivedFormat.pdf + ? "PDF" + : "Screenshot"; + anchorElement.click(); } else { console.error("Failed to download file"); } diff --git a/components/ReadableView.tsx b/components/ReadableView.tsx index d4c5eb5..b600236 100644 --- a/components/ReadableView.tsx +++ b/components/ReadableView.tsx @@ -81,9 +81,11 @@ export default function ReadableView({ link }: Props) { (link?.image === "pending" || link?.pdf === "pending" || link?.readable === "pending" || + link?.monolith === "pending" || !link?.image || !link?.pdf || - !link?.readable) + !link?.readable || + !link?.monolith) ) { interval = setInterval(() => getLink(link.id as number), 5000); } else { @@ -97,7 +99,7 @@ export default function ReadableView({ link }: Props) { clearInterval(interval); } }; - }, [link?.image, link?.pdf, link?.readable]); + }, [link?.image, link?.pdf, link?.readable, link?.monolith]); const rgbToHex = (r: number, g: number, b: number): string => "#" + diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.ts index 0cb514d..c5e5f09 100644 --- a/lib/api/archiveHandler.ts +++ b/lib/api/archiveHandler.ts @@ -1,15 +1,16 @@ import { LaunchOptions, chromium, devices } from "playwright"; import { prisma } from "./db"; -import createFile from "./storage/createFile"; -import sendToWayback from "./sendToWayback"; -import { Readability } from "@mozilla/readability"; -import { JSDOM } from "jsdom"; -import DOMPurify from "dompurify"; +import sendToWayback from "./preservationScheme/sendToWayback"; import { Collection, Link, User } from "@prisma/client"; -import validateUrlSize from "./validateUrlSize"; +import fetchHeaders from "./fetchHeaders"; import createFolder from "./storage/createFolder"; -import generatePreview from "./generatePreview"; import { removeFiles } from "./manageLinkFiles"; +import handleMonolith from "./preservationScheme/handleMonolith"; +import handleReadablility from "./preservationScheme/handleReadablility"; +import handleArchivePreview from "./preservationScheme/handleArchivePreview"; +import handleScreenshotAndPdf from "./preservationScheme/handleScreenshotAndPdf"; +import imageHandler from "./preservationScheme/imageHandler"; +import pdfHandler from "./preservationScheme/pdfHandler"; type LinksAndCollectionAndOwner = Link & { collection: Collection & { @@ -20,6 +21,18 @@ type LinksAndCollectionAndOwner = Link & { const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5; export default async function archiveHandler(link: LinksAndCollectionAndOwner) { + const timeoutPromise = new Promise((_, reject) => { + setTimeout( + () => + reject( + new Error( + `Browser has been open for more than ${BROWSER_TIMEOUT} minutes.` + ) + ), + BROWSER_TIMEOUT * 60000 + ); + }); + // allow user to configure a proxy let browserOptions: LaunchOptions = {}; if (process.env.PROXY) { @@ -39,18 +52,6 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) { const page = await context.newPage(); - const timeoutPromise = new Promise((_, reject) => { - setTimeout( - () => - reject( - new Error( - `Browser has been open for more than ${BROWSER_TIMEOUT} minutes.` - ) - ), - BROWSER_TIMEOUT * 60000 - ); - }); - createFolder({ filePath: `archives/preview/${link.collectionId}`, }); @@ -62,17 +63,11 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) { try { await Promise.race([ (async () => { - const validatedUrl = link.url - ? await validateUrlSize(link.url) - : undefined; + const user = link.collection?.owner; - if ( - validatedUrl === null && - process.env.IGNORE_URL_SIZE_LIMIT !== "true" - ) - throw "Something went wrong while retrieving the file size."; + const header = link.url ? await fetchHeaders(link.url) : undefined; - const contentType = validatedUrl?.get("content-type"); + const contentType = header?.get("content-type"); let linkType = "url"; let imageExtension = "png"; @@ -84,12 +79,7 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) { else if (contentType.includes("image/png")) imageExtension = "png"; } - const user = link.collection?.owner; - - // send to archive.org - if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url); - - const targetLink = await prisma.link.update({ + await prisma.link.update({ where: { id: link.id }, data: { type: linkType, @@ -104,6 +94,9 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) { readable: !link.readable?.startsWith("archive") ? "pending" : undefined, + monolith: !link.monolith?.startsWith("archive") + ? "pending" + : undefined, preview: !link.readable?.startsWith("archive") ? "pending" : undefined, @@ -111,6 +104,9 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) { }, }); + // send to archive.org + if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url); + if (linkType === "image" && !link.image?.startsWith("archive")) { await imageHandler(link, imageExtension); // archive image (jpeg/png) return; @@ -124,151 +120,37 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) { const content = await page.content(); - // TODO single file - // const session = await page.context().newCDPSession(page); - // const doc = await session.send("Page.captureSnapshot", { - // format: "mhtml", - // }); - // const saveDocLocally = (doc: any) => { - // console.log(doc); - // return createFile({ - // data: doc, - // filePath: `archives/${targetLink.collectionId}/${link.id}.mhtml`, - // }); - // }; - // saveDocLocally(doc.data); + // Preview + if ( + !link.preview?.startsWith("archives") && + !link.preview?.startsWith("unavailable") + ) + await handleArchivePreview(link, page); // Readability - const window = new JSDOM("").window; - const purify = DOMPurify(window); - const cleanedUpContent = purify.sanitize(content); - const dom = new JSDOM(cleanedUpContent, { url: link.url || "" }); - const article = new Readability(dom.window.document).parse(); - const articleText = article?.textContent - .replace(/ +(?= )/g, "") // strip out multiple spaces - .replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaks if ( - articleText && - articleText !== "" && - !link.readable?.startsWith("archive") - ) { - await createFile({ - data: JSON.stringify(article), - filePath: `archives/${targetLink.collectionId}/${link.id}_readability.json`, - }); - - await prisma.link.update({ - where: { id: link.id }, - data: { - readable: `archives/${targetLink.collectionId}/${link.id}_readability.json`, - textContent: articleText, - }, - }); - } - - // Preview - - const ogImageUrl = await page.evaluate(() => { - const metaTag = document.querySelector('meta[property="og:image"]'); - return metaTag ? (metaTag as any).content : null; - }); - - if (ogImageUrl) { - console.log("Found og:image URL:", ogImageUrl); - - // Download the image - const imageResponse = await page.goto(ogImageUrl); - - // Check if imageResponse is not null - if (imageResponse && !link.preview?.startsWith("archive")) { - const buffer = await imageResponse.body(); - await generatePreview(buffer, link.collectionId, link.id); - } - - await page.goBack(); - } else if (!link.preview?.startsWith("archive")) { - console.log("No og:image found"); - await page - .screenshot({ type: "jpeg", quality: 20 }) - .then((screenshot) => { - return createFile({ - data: screenshot, - filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`, - }); - }) - .then(() => { - return prisma.link.update({ - where: { id: link.id }, - data: { - preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`, - }, - }); - }); - } + !link.readable?.startsWith("archives") && + !link.readable?.startsWith("unavailable") + ) + await handleReadablility(content, link); // Screenshot/PDF - await page.evaluate( - autoScroll, - Number(process.env.AUTOSCROLL_TIMEOUT) || 30 - ); + if ( + (!link.image?.startsWith("archives") && + !link.image?.startsWith("unavailable")) || + (!link.pdf?.startsWith("archives") && + !link.pdf?.startsWith("unavailable")) + ) + await handleScreenshotAndPdf(link, page, user); - // Check if the user hasn't deleted the link by the time we're done scrolling - const linkExists = await prisma.link.findUnique({ - where: { id: link.id }, - }); - if (linkExists) { - const processingPromises = []; - - if ( - user.archiveAsScreenshot && - !link.image?.startsWith("archive") - ) { - processingPromises.push( - page.screenshot({ fullPage: true }).then((screenshot) => { - return createFile({ - data: screenshot, - filePath: `archives/${linkExists.collectionId}/${link.id}.png`, - }); - }) - ); - } - - // apply administrator's defined pdf margins or default to 15px - const margins = { - top: process.env.PDF_MARGIN_TOP || "15px", - bottom: process.env.PDF_MARGIN_BOTTOM || "15px", - }; - - if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) { - processingPromises.push( - page - .pdf({ - width: "1366px", - height: "1931px", - printBackground: true, - margin: margins, - }) - .then((pdf) => { - return createFile({ - data: pdf, - filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`, - }); - }) - ); - } - await Promise.allSettled(processingPromises); - await prisma.link.update({ - where: { id: link.id }, - data: { - image: user.archiveAsScreenshot - ? `archives/${linkExists.collectionId}/${link.id}.png` - : undefined, - pdf: user.archiveAsPDF - ? `archives/${linkExists.collectionId}/${link.id}.pdf` - : undefined, - }, - }); - } + // Monolith + if ( + !link.monolith?.startsWith("archive") && + !link.monolith?.startsWith("unavailable") && + user.archiveAsMonolith && + link.url + ) + await handleMonolith(link, content); } })(), timeoutPromise, @@ -293,6 +175,9 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) { image: !finalLink.image?.startsWith("archives") ? "unavailable" : undefined, + monolith: !finalLink.monolith?.startsWith("archives") + ? "unavailable" + : undefined, pdf: !finalLink.pdf?.startsWith("archives") ? "unavailable" : undefined, @@ -308,76 +193,3 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) { await browser.close(); } } - -const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => { - const timeoutPromise = new Promise((_, reject) => { - setTimeout(() => { - reject(new Error(`Webpage was too long to be archived.`)); - }, AUTOSCROLL_TIMEOUT * 1000); - }); - - const scrollingPromise = new Promise((resolve) => { - let totalHeight = 0; - let distance = 100; - let scrollDown = setInterval(() => { - let scrollHeight = document.body.scrollHeight; - window.scrollBy(0, distance); - totalHeight += distance; - if (totalHeight >= scrollHeight) { - clearInterval(scrollDown); - window.scroll(0, 0); - resolve(); - } - }, 100); - }); - - await Promise.race([scrollingPromise, timeoutPromise]); -}; - -const imageHandler = async ({ url, id }: Link, extension: string) => { - const image = await fetch(url as string).then((res) => res.blob()); - - const buffer = Buffer.from(await image.arrayBuffer()); - - const linkExists = await prisma.link.findUnique({ - where: { id }, - }); - - if (linkExists) { - await createFile({ - data: buffer, - filePath: `archives/${linkExists.collectionId}/${id}.${extension}`, - }); - - await prisma.link.update({ - where: { id }, - data: { - image: `archives/${linkExists.collectionId}/${id}.${extension}`, - }, - }); - } -}; - -const pdfHandler = async ({ url, id }: Link) => { - const pdf = await fetch(url as string).then((res) => res.blob()); - - const buffer = Buffer.from(await pdf.arrayBuffer()); - - const linkExists = await prisma.link.findUnique({ - where: { id }, - }); - - if (linkExists) { - await createFile({ - data: buffer, - filePath: `archives/${linkExists.collectionId}/${id}.pdf`, - }); - - await prisma.link.update({ - where: { id }, - data: { - pdf: `archives/${linkExists.collectionId}/${id}.pdf`, - }, - }); - } -}; diff --git a/lib/api/controllers/collections/collectionId/deleteCollectionById.ts b/lib/api/controllers/collections/collectionId/deleteCollectionById.ts index e8fe51a..1e21809 100644 --- a/lib/api/controllers/collections/collectionId/deleteCollectionById.ts +++ b/lib/api/controllers/collections/collectionId/deleteCollectionById.ts @@ -1,6 +1,6 @@ import { prisma } from "@/lib/api/db"; import getPermission from "@/lib/api/getPermission"; -import { Collection, UsersAndCollections } from "@prisma/client"; +import { UsersAndCollections } from "@prisma/client"; import removeFolder from "@/lib/api/storage/removeFolder"; export default async function deleteCollection( @@ -58,6 +58,7 @@ export default async function deleteCollection( }); await removeFolder({ filePath: `archives/${collectionId}` }); + await removeFolder({ filePath: `archives/preview/${collectionId}` }); await removeFromOrders(userId, collectionId); @@ -100,6 +101,7 @@ async function deleteSubCollections(collectionId: number) { }); await removeFolder({ filePath: `archives/${subCollection.id}` }); + await removeFolder({ filePath: `archives/preview/${subCollection.id}` }); } } diff --git a/lib/api/controllers/links/getLinks.ts b/lib/api/controllers/links/getLinks.ts index 0086e93..976f1f1 100644 --- a/lib/api/controllers/links/getLinks.ts +++ b/lib/api/controllers/links/getLinks.ts @@ -2,7 +2,8 @@ import { prisma } from "@/lib/api/db"; import { LinkRequestQuery, Sort } from "@/types/global"; export default async function getLink(userId: number, query: LinkRequestQuery) { - const POSTGRES_IS_ENABLED = process.env.DATABASE_URL.startsWith("postgresql"); + const POSTGRES_IS_ENABLED = + process.env.DATABASE_URL?.startsWith("postgresql"); let order: any; if (query.sort === Sort.DateNewestFirst) order = { id: "desc" }; diff --git a/lib/api/controllers/links/linkId/deleteLinkById.ts b/lib/api/controllers/links/linkId/deleteLinkById.ts index dba90cb..abff6f0 100644 --- a/lib/api/controllers/links/linkId/deleteLinkById.ts +++ b/lib/api/controllers/links/linkId/deleteLinkById.ts @@ -1,7 +1,6 @@ import { prisma } from "@/lib/api/db"; import { Link, UsersAndCollections } from "@prisma/client"; import getPermission from "@/lib/api/getPermission"; -import removeFile from "@/lib/api/storage/removeFile"; import { removeFiles } from "@/lib/api/manageLinkFiles"; export default async function deleteLink(userId: number, linkId: number) { diff --git a/lib/api/controllers/links/postLink.ts b/lib/api/controllers/links/postLink.ts index 7f77aeb..8056c11 100644 --- a/lib/api/controllers/links/postLink.ts +++ b/lib/api/controllers/links/postLink.ts @@ -1,10 +1,8 @@ import { prisma } from "@/lib/api/db"; import { LinkIncludingShortenedCollectionAndTags } from "@/types/global"; -import getTitle from "@/lib/shared/getTitle"; -import { UsersAndCollections } from "@prisma/client"; -import getPermission from "@/lib/api/getPermission"; +import fetchTitleAndHeaders from "@/lib/shared/fetchTitleAndHeaders"; import createFolder from "@/lib/api/storage/createFolder"; -import validateUrlSize from "../../validateUrlSize"; +import setLinkCollection from "../../setLinkCollection"; const MAX_LINKS_PER_USER = Number(process.env.MAX_LINKS_PER_USER) || 30000; @@ -24,93 +22,10 @@ export default async function postLink( } } - if (!link.collection.id && link.collection.name) { - link.collection.name = link.collection.name.trim(); + const linkCollection = await setLinkCollection(link, userId); - // find the collection with the name and the user's id - const findCollection = await prisma.collection.findFirst({ - where: { - name: link.collection.name, - ownerId: userId, - parentId: link.collection.parentId, - }, - }); - - if (findCollection) { - const collectionIsAccessible = await getPermission({ - userId, - collectionId: findCollection.id, - }); - - const memberHasAccess = collectionIsAccessible?.members.some( - (e: UsersAndCollections) => e.userId === userId && e.canCreate - ); - - if (!(collectionIsAccessible?.ownerId === userId || memberHasAccess)) - return { response: "Collection is not accessible.", status: 401 }; - - link.collection.id = findCollection.id; - link.collection.ownerId = findCollection.ownerId; - } else { - const collection = await prisma.collection.create({ - data: { - name: link.collection.name, - ownerId: userId, - }, - }); - - link.collection.id = collection.id; - - await prisma.user.update({ - where: { - id: userId, - }, - data: { - collectionOrder: { - push: link.collection.id, - }, - }, - }); - } - } else if (link.collection.id) { - const collectionIsAccessible = await getPermission({ - userId, - collectionId: link.collection.id, - }); - - const memberHasAccess = collectionIsAccessible?.members.some( - (e: UsersAndCollections) => e.userId === userId && e.canCreate - ); - - if (!(collectionIsAccessible?.ownerId === userId || memberHasAccess)) - return { response: "Collection is not accessible.", status: 401 }; - } else if (!link.collection.id) { - link.collection.name = "Unorganized"; - link.collection.parentId = null; - - // find the collection with the name "Unorganized" and the user's id - const unorganizedCollection = await prisma.collection.findFirst({ - where: { - name: "Unorganized", - ownerId: userId, - }, - }); - - link.collection.id = unorganizedCollection?.id; - - await prisma.user.update({ - where: { - id: userId, - }, - data: { - collectionOrder: { - push: link.collection.id, - }, - }, - }); - } else { - return { response: "Uncaught error.", status: 500 }; - } + if (!linkCollection) + return { response: "Collection is not accessible.", status: 400 }; const user = await prisma.user.findUnique({ where: { @@ -124,8 +39,6 @@ export default async function postLink( const urlWithoutWww = hasWwwPrefix ? url?.replace(`://www.`, "://") : url; const urlWithWww = hasWwwPrefix ? url : url?.replace("://", `://www.`); - console.log(url, urlWithoutWww, urlWithWww); - const existingLink = await prisma.link.findFirst({ where: { OR: [{ url: urlWithWww }, { url: urlWithoutWww }], @@ -135,8 +48,6 @@ export default async function postLink( }, }); - console.log(url, urlWithoutWww, urlWithWww, "DONE!"); - if (existingLink) return { response: "Link already exists", @@ -147,30 +58,23 @@ export default async function postLink( const numberOfLinksTheUserHas = await prisma.link.count({ where: { collection: { - ownerId: userId, + ownerId: linkCollection.ownerId, }, }, }); - if (numberOfLinksTheUserHas + 1 > MAX_LINKS_PER_USER) + if (numberOfLinksTheUserHas > MAX_LINKS_PER_USER) return { - response: `Error: Each user can only have a maximum of ${MAX_LINKS_PER_USER} Links.`, + response: `Each collection owner can only have a maximum of ${MAX_LINKS_PER_USER} Links.`, status: 400, }; - link.collection.name = link.collection.name.trim(); - - const title = - !(link.name && link.name !== "") && link.url - ? await getTitle(link.url) - : ""; + const { title, headers } = await fetchTitleAndHeaders(link.url || ""); const name = link.name && link.name !== "" ? link.name : link.url ? title : ""; - const validatedUrl = link.url ? await validateUrlSize(link.url) : undefined; - - const contentType = validatedUrl?.get("content-type"); + const contentType = headers?.get("content-type"); let linkType = "url"; let imageExtension = "png"; @@ -190,7 +94,7 @@ export default async function postLink( type: linkType, collection: { connect: { - id: link.collection.id, + id: linkCollection.id, }, }, tags: { @@ -198,14 +102,14 @@ export default async function postLink( where: { name_ownerId: { name: tag.name.trim(), - ownerId: link.collection.ownerId, + ownerId: linkCollection.ownerId, }, }, create: { name: tag.name.trim(), owner: { connect: { - id: link.collection.ownerId, + id: linkCollection.ownerId, }, }, }, diff --git a/lib/api/controllers/migration/importFromHTMLFile.ts b/lib/api/controllers/migration/importFromHTMLFile.ts index 21fb3c4..6e6cf71 100644 --- a/lib/api/controllers/migration/importFromHTMLFile.ts +++ b/lib/api/controllers/migration/importFromHTMLFile.ts @@ -31,7 +31,7 @@ export default async function importFromHTMLFile( if (totalImports + numberOfLinksTheUserHas > MAX_LINKS_PER_USER) return { - response: `Error: Each user can only have a maximum of ${MAX_LINKS_PER_USER} Links.`, + response: `Each collection owner can only have a maximum of ${MAX_LINKS_PER_USER} Links.`, status: 400, }; diff --git a/lib/api/controllers/migration/importFromLinkwarden.ts b/lib/api/controllers/migration/importFromLinkwarden.ts index fb486cb..5e872e8 100644 --- a/lib/api/controllers/migration/importFromLinkwarden.ts +++ b/lib/api/controllers/migration/importFromLinkwarden.ts @@ -26,7 +26,7 @@ export default async function importFromLinkwarden( if (totalImports + numberOfLinksTheUserHas > MAX_LINKS_PER_USER) return { - response: `Error: Each user can only have a maximum of ${MAX_LINKS_PER_USER} Links.`, + response: `Each collection owner can only have a maximum of ${MAX_LINKS_PER_USER} Links.`, status: 400, }; diff --git a/lib/api/controllers/migration/importFromWallabag.ts b/lib/api/controllers/migration/importFromWallabag.ts index 6f9f404..568952a 100644 --- a/lib/api/controllers/migration/importFromWallabag.ts +++ b/lib/api/controllers/migration/importFromWallabag.ts @@ -47,7 +47,7 @@ export default async function importFromWallabag( if (totalImports + numberOfLinksTheUserHas > MAX_LINKS_PER_USER) return { - response: `Error: Each user can only have a maximum of ${MAX_LINKS_PER_USER} Links.`, + response: `Each collection owner can only have a maximum of ${MAX_LINKS_PER_USER} Links.`, status: 400, }; diff --git a/lib/api/controllers/public/links/getPublicLinksUnderCollection.ts b/lib/api/controllers/public/links/getPublicLinksUnderCollection.ts index e94a3e1..1b48984 100644 --- a/lib/api/controllers/public/links/getPublicLinksUnderCollection.ts +++ b/lib/api/controllers/public/links/getPublicLinksUnderCollection.ts @@ -4,7 +4,8 @@ import { LinkRequestQuery, Sort } from "@/types/global"; export default async function getLink( query: Omit ) { - const POSTGRES_IS_ENABLED = process.env.DATABASE_URL.startsWith("postgresql"); + const POSTGRES_IS_ENABLED = + process.env.DATABASE_URL?.startsWith("postgresql"); let order: any; if (query.sort === Sort.DateNewestFirst) order = { id: "desc" }; diff --git a/lib/api/controllers/public/users/getPublicUser.ts b/lib/api/controllers/public/users/getPublicUser.ts index 04c7994..68b6fab 100644 --- a/lib/api/controllers/public/users/getPublicUser.ts +++ b/lib/api/controllers/public/users/getPublicUser.ts @@ -75,6 +75,7 @@ export default async function getPublicUser( username: lessSensitiveInfo.username, image: lessSensitiveInfo.image, archiveAsScreenshot: lessSensitiveInfo.archiveAsScreenshot, + archiveAsMonolith: lessSensitiveInfo.archiveAsMonolith, archiveAsPDF: lessSensitiveInfo.archiveAsPDF, }; diff --git a/lib/api/controllers/session/createSession.ts b/lib/api/controllers/session/createSession.ts index c2ff37e..c1a6aa7 100644 --- a/lib/api/controllers/session/createSession.ts +++ b/lib/api/controllers/session/createSession.ts @@ -21,12 +21,12 @@ export default async function createSession( jti: crypto.randomUUID(), }, maxAge: expiryDateSecond || 604800, - secret: process.env.NEXTAUTH_SECRET, + secret: process.env.NEXTAUTH_SECRET as string, }); const tokenBody = await decode({ token, - secret: process.env.NEXTAUTH_SECRET, + secret: process.env.NEXTAUTH_SECRET as string, }); const createToken = await prisma.accessToken.create({ diff --git a/lib/api/controllers/tokens/postToken.ts b/lib/api/controllers/tokens/postToken.ts index f88030d..faedd8a 100644 --- a/lib/api/controllers/tokens/postToken.ts +++ b/lib/api/controllers/tokens/postToken.ts @@ -65,12 +65,12 @@ export default async function postToken( jti: crypto.randomUUID(), }, maxAge: expiryDateSecond || 604800, - secret: process.env.NEXTAUTH_SECRET, + secret: process.env.NEXTAUTH_SECRET as string, }); const tokenBody = await decode({ token, - secret: process.env.NEXTAUTH_SECRET, + secret: process.env.NEXTAUTH_SECRET as string, }); const createToken = await prisma.accessToken.create({ diff --git a/lib/api/controllers/users/userId/deleteUserById.ts b/lib/api/controllers/users/userId/deleteUserById.ts index 9b65304..a6e6106 100644 --- a/lib/api/controllers/users/userId/deleteUserById.ts +++ b/lib/api/controllers/users/userId/deleteUserById.ts @@ -80,7 +80,7 @@ export default async function deleteUserById( }); // Delete archive folders - removeFolder({ filePath: `archives/${collection.id}` }); + await removeFolder({ filePath: `archives/${collection.id}` }); await removeFolder({ filePath: `archives/preview/${collection.id}`, diff --git a/lib/api/controllers/users/userId/updateUserById.ts b/lib/api/controllers/users/userId/updateUserById.ts index b6e8bea..ebae737 100644 --- a/lib/api/controllers/users/userId/updateUserById.ts +++ b/lib/api/controllers/users/userId/updateUserById.ts @@ -207,6 +207,7 @@ export default async function updateUserById( ), locale: i18n.locales.includes(data.locale) ? data.locale : "en", archiveAsScreenshot: data.archiveAsScreenshot, + archiveAsMonolith: data.archiveAsMonolith, archiveAsPDF: data.archiveAsPDF, archiveAsWaybackMachine: data.archiveAsWaybackMachine, linksRouteTo: data.linksRouteTo, diff --git a/lib/api/validateUrlSize.ts b/lib/api/fetchHeaders.ts similarity index 65% rename from lib/api/validateUrlSize.ts rename to lib/api/fetchHeaders.ts index d5826b8..f423661 100644 --- a/lib/api/validateUrlSize.ts +++ b/lib/api/fetchHeaders.ts @@ -2,7 +2,7 @@ import fetch from "node-fetch"; import https from "https"; import { SocksProxyAgent } from "socks-proxy-agent"; -export default async function validateUrlSize(url: string) { +export default async function fetchHeaders(url: string) { if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null; try { @@ -29,13 +29,17 @@ export default async function validateUrlSize(url: string) { }; } - const response = await fetch(url, fetchOpts); + const responsePromise = fetch(url, fetchOpts); - const totalSizeMB = - Number(response.headers.get("content-length")) / Math.pow(1024, 2); - if (totalSizeMB > (Number(process.env.NEXT_PUBLIC_MAX_FILE_SIZE) || 30)) - return null; - else return response.headers; + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject(new Error("Fetch header timeout")); + }, 10 * 1000); // Stop after 10 seconds + }); + + const response = await Promise.race([responsePromise, timeoutPromise]); + + return (response as Response)?.headers || null; } catch (err) { console.log(err); return null; diff --git a/lib/api/generatePreview.ts b/lib/api/generatePreview.ts index 6e81630..2693be2 100644 --- a/lib/api/generatePreview.ts +++ b/lib/api/generatePreview.ts @@ -1,7 +1,6 @@ import Jimp from "jimp"; import { prisma } from "./db"; import createFile from "./storage/createFile"; -import createFolder from "./storage/createFolder"; const generatePreview = async ( buffer: Buffer, @@ -9,27 +8,39 @@ const generatePreview = async ( linkId: number ) => { if (buffer && collectionId && linkId) { - // Load the image using Jimp - await Jimp.read(buffer, async (err, image) => { - if (image && !err) { - image?.resize(1280, Jimp.AUTO).quality(20); - const processedBuffer = await image?.getBufferAsync(Jimp.MIME_JPEG); + try { + const image = await Jimp.read(buffer); - createFile({ - data: processedBuffer, - filePath: `archives/preview/${collectionId}/${linkId}.jpeg`, - }).then(() => { - return prisma.link.update({ - where: { id: linkId }, - data: { - preview: `archives/preview/${collectionId}/${linkId}.jpeg`, - }, - }); - }); + if (!image) { + console.log("Error generating preview: Image not found"); + return; } - }).catch((err) => { + + image.resize(1280, Jimp.AUTO).quality(20); + const processedBuffer = await image.getBufferAsync(Jimp.MIME_JPEG); + + if ( + Buffer.byteLength(processedBuffer) > + 1024 * 1024 * Number(process.env.PREVIEW_MAX_BUFFER || 0.1) + ) { + console.log("Error generating preview: Buffer size exceeded"); + return; + } + + await createFile({ + data: processedBuffer, + filePath: `archives/preview/${collectionId}/${linkId}.jpeg`, + }); + + await prisma.link.update({ + where: { id: linkId }, + data: { + preview: `archives/preview/${collectionId}/${linkId}.jpeg`, + }, + }); + } catch (err) { console.error("Error processing the image:", err); - }); + } } }; diff --git a/lib/api/getPermission.ts b/lib/api/getPermission.ts index 93dd04c..61dc5c5 100644 --- a/lib/api/getPermission.ts +++ b/lib/api/getPermission.ts @@ -3,14 +3,12 @@ import { prisma } from "@/lib/api/db"; type Props = { userId: number; collectionId?: number; - collectionName?: string; linkId?: number; }; export default async function getPermission({ userId, collectionId, - collectionName, linkId, }: Props) { if (linkId) { @@ -26,11 +24,10 @@ export default async function getPermission({ }); return check; - } else if (collectionId || collectionName) { + } else if (collectionId) { const check = await prisma.collection.findFirst({ where: { - id: collectionId || undefined, - name: collectionName || undefined, + id: collectionId, OR: [{ ownerId: userId }, { members: { some: { userId } } }], }, include: { members: true }, diff --git a/lib/api/manageLinkFiles.ts b/lib/api/manageLinkFiles.ts index 7bacdab..7498772 100644 --- a/lib/api/manageLinkFiles.ts +++ b/lib/api/manageLinkFiles.ts @@ -16,6 +16,10 @@ const removeFiles = async (linkId: number, collectionId: number) => { await removeFile({ filePath: `archives/${collectionId}/${linkId}.jpg`, }); + // HTML + await removeFile({ + filePath: `archives/${collectionId}/${linkId}.html`, + }); // Preview await removeFile({ filePath: `archives/preview/${collectionId}/${linkId}.jpeg`, @@ -47,6 +51,11 @@ const moveFiles = async (linkId: number, from: number, to: number) => { `archives/${to}/${linkId}.jpg` ); + await moveFile( + `archives/${from}/${linkId}.html`, + `archives/${to}/${linkId}.html` + ); + await moveFile( `archives/preview/${from}/${linkId}.jpeg`, `archives/preview/${to}/${linkId}.jpeg` diff --git a/lib/api/preservationScheme/handleArchivePreview.ts b/lib/api/preservationScheme/handleArchivePreview.ts new file mode 100644 index 0000000..9714985 --- /dev/null +++ b/lib/api/preservationScheme/handleArchivePreview.ts @@ -0,0 +1,61 @@ +import { Collection, Link, User } from "@prisma/client"; +import { Page } from "playwright"; +import generatePreview from "../generatePreview"; +import createFile from "../storage/createFile"; +import { prisma } from "../db"; + +type LinksAndCollectionAndOwner = Link & { + collection: Collection & { + owner: User; + }; +}; + +const handleArchivePreview = async ( + link: LinksAndCollectionAndOwner, + page: Page +) => { + const ogImageUrl = await page.evaluate(() => { + const metaTag = document.querySelector('meta[property="og:image"]'); + return metaTag ? (metaTag as any).content : null; + }); + + if (ogImageUrl) { + console.log("Found og:image URL:", ogImageUrl); + + // Download the image + const imageResponse = await page.goto(ogImageUrl); + + // Check if imageResponse is not null + if (imageResponse && !link.preview?.startsWith("archive")) { + const buffer = await imageResponse.body(); + generatePreview(buffer, link.collectionId, link.id); + } + + await page.goBack(); + } else if (!link.preview?.startsWith("archive")) { + console.log("No og:image found"); + await page + .screenshot({ type: "jpeg", quality: 20 }) + .then(async (screenshot) => { + if ( + Buffer.byteLength(screenshot) > + 1024 * 1024 * Number(process.env.PREVIEW_MAX_BUFFER || 0.1) + ) + return console.log("Error generating preview: Buffer size exceeded"); + + await createFile({ + data: screenshot, + filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`, + }); + + await prisma.link.update({ + where: { id: link.id }, + data: { + preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`, + }, + }); + }); + } +}; + +export default handleArchivePreview; diff --git a/lib/api/preservationScheme/handleMonolith.ts b/lib/api/preservationScheme/handleMonolith.ts new file mode 100644 index 0000000..f7c08f5 --- /dev/null +++ b/lib/api/preservationScheme/handleMonolith.ts @@ -0,0 +1,46 @@ +import { execSync } from "child_process"; +import createFile from "../storage/createFile"; +import { prisma } from "../db"; +import { Link } from "@prisma/client"; + +const handleMonolith = async (link: Link, content: string) => { + if (!link.url) return; + + try { + let html = execSync( + `monolith - -I -b ${link.url} ${ + process.env.MONOLITH_CUSTOM_OPTIONS || "-j -F -s" + } -o -`, + { + timeout: 120000, + maxBuffer: 1024 * 1024 * Number(process.env.MONOLITH_MAX_BUFFER || 5), + input: content, + } + ); + + if (!html?.length) + return console.error("Error archiving as Monolith: Empty buffer"); + + if ( + Buffer.byteLength(html) > + 1024 * 1024 * Number(process.env.MONOLITH_MAX_BUFFER || 6) + ) + return console.error("Error archiving as Monolith: Buffer size exceeded"); + + await createFile({ + data: html, + filePath: `archives/${link.collectionId}/${link.id}.html`, + }).then(async () => { + await prisma.link.update({ + where: { id: link.id }, + data: { + monolith: `archives/${link.collectionId}/${link.id}.html`, + }, + }); + }); + } catch (err) { + console.log("Error running MONOLITH:", err); + } +}; + +export default handleMonolith; diff --git a/lib/api/preservationScheme/handleReadablility.ts b/lib/api/preservationScheme/handleReadablility.ts new file mode 100644 index 0000000..e274fc2 --- /dev/null +++ b/lib/api/preservationScheme/handleReadablility.ts @@ -0,0 +1,51 @@ +import { Readability } from "@mozilla/readability"; +import { JSDOM } from "jsdom"; +import DOMPurify from "dompurify"; +import { prisma } from "../db"; +import createFile from "../storage/createFile"; +import { Link } from "@prisma/client"; + +const handleReadablility = async (content: string, link: Link) => { + const window = new JSDOM("").window; + const purify = DOMPurify(window); + const cleanedUpContent = purify.sanitize(content); + const dom = new JSDOM(cleanedUpContent, { url: link.url || "" }); + const article = new Readability(dom.window.document).parse(); + const articleText = article?.textContent + .replace(/ +(?= )/g, "") // strip out multiple spaces + .replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaks + + if (articleText && articleText !== "") { + const collectionId = ( + await prisma.link.findUnique({ + where: { id: link.id }, + select: { collectionId: true }, + }) + )?.collectionId; + + const data = JSON.stringify(article); + + if ( + Buffer.byteLength(data, "utf8") > + 1024 * 1024 * Number(process.env.READABILITY_MAX_BUFFER || 1) + ) + return console.error( + "Error archiving as Readability: Buffer size exceeded" + ); + + await createFile({ + data, + filePath: `archives/${collectionId}/${link.id}_readability.json`, + }); + + await prisma.link.update({ + where: { id: link.id }, + data: { + readable: `archives/${collectionId}/${link.id}_readability.json`, + textContent: articleText, + }, + }); + } +}; + +export default handleReadablility; diff --git a/lib/api/preservationScheme/handleScreenshotAndPdf.ts b/lib/api/preservationScheme/handleScreenshotAndPdf.ts new file mode 100644 index 0000000..fee5126 --- /dev/null +++ b/lib/api/preservationScheme/handleScreenshotAndPdf.ts @@ -0,0 +1,122 @@ +import { Collection, Link, User } from "@prisma/client"; +import { Page } from "playwright"; +import createFile from "../storage/createFile"; +import { prisma } from "../db"; + +type LinksAndCollectionAndOwner = Link & { + collection: Collection & { + owner: User; + }; +}; +const handleScreenshotAndPdf = async ( + link: LinksAndCollectionAndOwner, + page: Page, + user: User +) => { + await page.evaluate(autoScroll, Number(process.env.AUTOSCROLL_TIMEOUT) || 30); + + // Check if the user hasn't deleted the link by the time we're done scrolling + const linkExists = await prisma.link.findUnique({ + where: { id: link.id }, + }); + if (linkExists) { + const processingPromises = []; + + if (user.archiveAsScreenshot && !link.image?.startsWith("archive")) { + processingPromises.push( + page + .screenshot({ fullPage: true, type: "jpeg" }) + .then(async (screenshot) => { + if ( + Buffer.byteLength(screenshot) > + 1024 * 1024 * Number(process.env.SCREENSHOT_MAX_BUFFER || 2) + ) + return console.log( + "Error archiving as Screenshot: Buffer size exceeded" + ); + + await createFile({ + data: screenshot, + filePath: `archives/${linkExists.collectionId}/${link.id}.jpeg`, + }); + await prisma.link.update({ + where: { id: link.id }, + data: { + image: user.archiveAsScreenshot + ? `archives/${linkExists.collectionId}/${link.id}.jpeg` + : undefined, + }, + }); + }) + ); + } + + const margins = { + top: process.env.PDF_MARGIN_TOP || "15px", + bottom: process.env.PDF_MARGIN_BOTTOM || "15px", + }; + + if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) { + processingPromises.push( + page + .pdf({ + width: "1366px", + height: "1931px", + printBackground: true, + margin: margins, + }) + .then(async (pdf) => { + if ( + Buffer.byteLength(pdf) > + 1024 * 1024 * Number(process.env.PDF_MAX_BUFFER || 2) + ) + return console.log( + "Error archiving as PDF: Buffer size exceeded" + ); + + await createFile({ + data: pdf, + filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`, + }); + + await prisma.link.update({ + where: { id: link.id }, + data: { + pdf: user.archiveAsPDF + ? `archives/${linkExists.collectionId}/${link.id}.pdf` + : undefined, + }, + }); + }) + ); + } + await Promise.allSettled(processingPromises); + } +}; + +const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => { + const timeoutPromise = new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, AUTOSCROLL_TIMEOUT * 1000); + }); + + const scrollingPromise = new Promise((resolve) => { + let totalHeight = 0; + let distance = 100; + let scrollDown = setInterval(() => { + let scrollHeight = document.body.scrollHeight; + window.scrollBy(0, distance); + totalHeight += distance; + if (totalHeight >= scrollHeight) { + clearInterval(scrollDown); + window.scroll(0, 0); + resolve(); + } + }, 100); + }); + + await Promise.race([scrollingPromise, timeoutPromise]); +}; + +export default handleScreenshotAndPdf; diff --git a/lib/api/preservationScheme/imageHandler.ts b/lib/api/preservationScheme/imageHandler.ts new file mode 100644 index 0000000..273c010 --- /dev/null +++ b/lib/api/preservationScheme/imageHandler.ts @@ -0,0 +1,38 @@ +import { Link } from "@prisma/client"; +import { prisma } from "../db"; +import createFile from "../storage/createFile"; +import generatePreview from "../generatePreview"; + +const imageHandler = async ({ url, id }: Link, extension: string) => { + const image = await fetch(url as string).then((res) => res.blob()); + + const buffer = Buffer.from(await image.arrayBuffer()); + + if ( + Buffer.byteLength(buffer) > + 1024 * 1024 * Number(process.env.SCREENSHOT_MAX_BUFFER || 2) + ) + return console.log("Error archiving as Screenshot: Buffer size exceeded"); + + const linkExists = await prisma.link.findUnique({ + where: { id }, + }); + + if (linkExists) { + await generatePreview(buffer, linkExists.collectionId, id); + + await createFile({ + data: buffer, + filePath: `archives/${linkExists.collectionId}/${id}.${extension}`, + }); + + await prisma.link.update({ + where: { id }, + data: { + image: `archives/${linkExists.collectionId}/${id}.${extension}`, + }, + }); + } +}; + +export default imageHandler; diff --git a/lib/api/preservationScheme/pdfHandler.ts b/lib/api/preservationScheme/pdfHandler.ts new file mode 100644 index 0000000..7216cef --- /dev/null +++ b/lib/api/preservationScheme/pdfHandler.ts @@ -0,0 +1,35 @@ +import { Link } from "@prisma/client"; +import { prisma } from "../db"; +import createFile from "../storage/createFile"; + +const pdfHandler = async ({ url, id }: Link) => { + const pdf = await fetch(url as string).then((res) => res.blob()); + + const buffer = Buffer.from(await pdf.arrayBuffer()); + + if ( + Buffer.byteLength(buffer) > + 1024 * 1024 * Number(process.env.PDF_MAX_BUFFER || 2) + ) + return console.log("Error archiving as PDF: Buffer size exceeded"); + + const linkExists = await prisma.link.findUnique({ + where: { id }, + }); + + if (linkExists) { + await createFile({ + data: buffer, + filePath: `archives/${linkExists.collectionId}/${id}.pdf`, + }); + + await prisma.link.update({ + where: { id }, + data: { + pdf: `archives/${linkExists.collectionId}/${id}.pdf`, + }, + }); + } +}; + +export default pdfHandler; diff --git a/lib/api/sendToWayback.ts b/lib/api/preservationScheme/sendToWayback.ts similarity index 100% rename from lib/api/sendToWayback.ts rename to lib/api/preservationScheme/sendToWayback.ts diff --git a/lib/api/setLinkCollection.ts b/lib/api/setLinkCollection.ts new file mode 100644 index 0000000..901e585 --- /dev/null +++ b/lib/api/setLinkCollection.ts @@ -0,0 +1,89 @@ +import { LinkIncludingShortenedCollectionAndTags } from "@/types/global"; +import { prisma } from "./db"; +import getPermission from "./getPermission"; +import { UsersAndCollections } from "@prisma/client"; + +const setLinkCollection = async ( + link: LinkIncludingShortenedCollectionAndTags, + userId: number +) => { + if (link?.collection?.id && typeof link?.collection?.id === "number") { + const existingCollection = await prisma.collection.findUnique({ + where: { + id: link.collection.id, + }, + }); + + if (!existingCollection) return null; + + const collectionIsAccessible = await getPermission({ + userId, + collectionId: existingCollection.id, + }); + + const memberHasAccess = collectionIsAccessible?.members.some( + (e: UsersAndCollections) => e.userId === userId && e.canCreate + ); + + if (!(collectionIsAccessible?.ownerId === userId || memberHasAccess)) + return null; + + return existingCollection; + } else if (link?.collection?.name) { + if (link.collection.name === "Unorganized") { + const firstTopLevelUnorganizedCollection = + await prisma.collection.findFirst({ + where: { + name: "Unorganized", + ownerId: userId, + parentId: null, + }, + }); + + if (firstTopLevelUnorganizedCollection) + return firstTopLevelUnorganizedCollection; + } + + const newCollection = await prisma.collection.create({ + data: { + name: link.collection.name.trim(), + ownerId: userId, + }, + }); + + await prisma.user.update({ + where: { + id: userId, + }, + data: { + collectionOrder: { + push: newCollection.id, + }, + }, + }); + + return newCollection; + } else { + const firstTopLevelUnorganizedCollection = + await prisma.collection.findFirst({ + where: { + name: "Unorganized", + ownerId: userId, + parentId: null, + }, + }); + + if (firstTopLevelUnorganizedCollection) + return firstTopLevelUnorganizedCollection; + else + return await prisma.collection.create({ + data: { + name: "Unorganized", + ownerId: userId, + parentId: null, + }, + }); + } +}; + +export default setLinkCollection; diff --git a/lib/api/storage/readFile.ts b/lib/api/storage/readFile.ts index 350726d..fc9d2e7 100644 --- a/lib/api/storage/readFile.ts +++ b/lib/api/storage/readFile.ts @@ -10,6 +10,7 @@ import util from "util"; type ReturnContentTypes = | "text/plain" + | "text/html" | "image/jpeg" | "image/png" | "application/pdf" @@ -61,6 +62,8 @@ export default async function readFile(filePath: string) { contentType = "image/png"; } else if (filePath.endsWith("_readability.json")) { contentType = "application/json"; + } else if (filePath.endsWith(".html")) { + contentType = "text/html"; } else { // if (filePath.endsWith(".jpg")) contentType = "image/jpeg"; @@ -88,6 +91,8 @@ export default async function readFile(filePath: string) { contentType = "image/png"; } else if (filePath.endsWith("_readability.json")) { contentType = "application/json"; + } else if (filePath.endsWith(".html")) { + contentType = "text/html"; } else { // if (filePath.endsWith(".jpg")) contentType = "image/jpeg"; diff --git a/lib/client/generateLinkHref.ts b/lib/client/generateLinkHref.ts index f012ab9..721c671 100644 --- a/lib/client/generateLinkHref.ts +++ b/lib/client/generateLinkHref.ts @@ -8,6 +8,7 @@ import { pdfAvailable, readabilityAvailable, screenshotAvailable, + monolithAvailable, } from "../shared/getArchiveValidity"; export const generateLinkHref = ( @@ -33,12 +34,15 @@ export const generateLinkHref = ( account.linksRouteTo === LinksRouteTo.SCREENSHOT || link.type === "image" ) { - console.log(link); if (!screenshotAvailable(link)) return link.url || ""; return `/preserved/${link?.id}?format=${ link?.image?.endsWith("png") ? ArchivedFormat.png : ArchivedFormat.jpeg }`; + } else if (account.linksRouteTo === LinksRouteTo.MONOLITH) { + if (!monolithAvailable(link)) return link.url || ""; + + return `/preserved/${link?.id}?format=${ArchivedFormat.monolith}`; } else { return link.url || ""; } diff --git a/lib/shared/getTitle.ts b/lib/shared/fetchTitleAndHeaders.ts similarity index 83% rename from lib/shared/getTitle.ts rename to lib/shared/fetchTitleAndHeaders.ts index 01488fd..a777964 100644 --- a/lib/shared/getTitle.ts +++ b/lib/shared/fetchTitleAndHeaders.ts @@ -2,7 +2,7 @@ import fetch from "node-fetch"; import https from "https"; import { SocksProxyAgent } from "socks-proxy-agent"; -export default async function getTitle(url: string) { +export default async function fetchTitleAndHeaders(url: string) { try { const httpsAgent = new https.Agent({ rejectUnauthorized: @@ -41,12 +41,16 @@ export default async function getTitle(url: string) { // regular expression to find the tag let match = text.match(/<title.*>([^<]*)<\/title>/); - if (match) return match[1]; - else return ""; + + const title = match[1] || ""; + const headers = (response as Response)?.headers || null; + + return { title, headers }; } else { - return ""; + return { title: "", headers: null }; } } catch (err) { console.log(err); + return { title: "", headers: null }; } } diff --git a/lib/shared/getArchiveValidity.ts b/lib/shared/getArchiveValidity.ts index 0da5504..8ec8c44 100644 --- a/lib/shared/getArchiveValidity.ts +++ b/lib/shared/getArchiveValidity.ts @@ -28,6 +28,17 @@ export function readabilityAvailable( ); } +export function monolithAvailable( + link: LinkIncludingShortenedCollectionAndTags +) { + return ( + link && + link.monolith && + link.monolith !== "pending" && + link.monolith !== "unavailable" + ); +} + export function previewAvailable(link: any) { return ( link && diff --git a/package.json b/package.json index 0045802..db8cf5d 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "next-i18next": "^15.3.0", "node-fetch": "^2.7.0", "nodemailer": "^6.9.3", - "playwright": "^1.43.1", + "playwright": "^1.45.0", "react": "18.2.0", "react-colorful": "^5.6.1", "react-dom": "18.2.0", @@ -77,7 +77,7 @@ "zustand": "^4.3.8" }, "devDependencies": { - "@playwright/test": "^1.43.1", + "@playwright/test": "^1.45.0", "@types/bcrypt": "^5.0.0", "@types/dompurify": "^3.0.4", "@types/jsdom": "^21.1.3", diff --git a/pages/api/v1/archives/[linkId].ts b/pages/api/v1/archives/[linkId].ts index 17e7518..faa3dd2 100644 --- a/pages/api/v1/archives/[linkId].ts +++ b/pages/api/v1/archives/[linkId].ts @@ -29,6 +29,7 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) { else if (format === ArchivedFormat.jpeg) suffix = ".jpeg"; else if (format === ArchivedFormat.pdf) suffix = ".pdf"; else if (format === ArchivedFormat.readability) suffix = "_readability.json"; + else if (format === ArchivedFormat.monolith) suffix = ".html"; //@ts-ignore if (!linkId || !suffix) @@ -84,21 +85,42 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) { linkId, }); - const memberHasAccess = collectionPermissions?.members.some( + if (!collectionPermissions) + return { response: "Collection is not accessible.", status: 400 }; + + const memberHasAccess = collectionPermissions.members.some( (e: UsersAndCollections) => e.userId === user.id && e.canCreate ); - if (!(collectionPermissions?.ownerId === user.id || memberHasAccess)) - return { response: "Collection is not accessible.", status: 401 }; + if (!(collectionPermissions.ownerId === user.id || memberHasAccess)) + return { response: "Collection is not accessible.", status: 400 }; // await uploadHandler(linkId, ) - const MAX_UPLOAD_SIZE = Number(process.env.NEXT_PUBLIC_MAX_FILE_SIZE); + const MAX_LINKS_PER_USER = Number(process.env.MAX_LINKS_PER_USER || 30000); + + const numberOfLinksTheUserHas = await prisma.link.count({ + where: { + collection: { + ownerId: user.id, + }, + }, + }); + + if (numberOfLinksTheUserHas > MAX_LINKS_PER_USER) + return { + response: `Each collection owner can only have a maximum of ${MAX_LINKS_PER_USER} Links.`, + status: 400, + }; + + const NEXT_PUBLIC_MAX_FILE_BUFFER = Number( + process.env.NEXT_PUBLIC_MAX_FILE_BUFFER || 10 + ); const form = formidable({ maxFields: 1, maxFiles: 1, - maxFileSize: MAX_UPLOAD_SIZE || 30 * 1048576, + maxFileSize: NEXT_PUBLIC_MAX_FILE_BUFFER * 1024 * 1024, }); form.parse(req, async (err, fields, files) => { @@ -116,18 +138,26 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) { !allowedMIMETypes.includes(files.file[0].mimetype || "") ) { // Handle parsing error - return res.status(500).json({ - response: `Sorry, we couldn't process your file. Please ensure it's a PDF, PNG, or JPG format and doesn't exceed ${MAX_UPLOAD_SIZE}MB.`, + return res.status(400).json({ + response: `Sorry, we couldn't process your file. Please ensure it's a PDF, PNG, or JPG format and doesn't exceed ${NEXT_PUBLIC_MAX_FILE_BUFFER}MB.`, }); } else { const fileBuffer = fs.readFileSync(files.file[0].filepath); + if ( + Buffer.byteLength(fileBuffer) > + 1024 * 1024 * Number(NEXT_PUBLIC_MAX_FILE_BUFFER) + ) + return res.status(400).json({ + response: `Sorry, we couldn't process your file. Please ensure it's a PDF, PNG, or JPG format and doesn't exceed ${NEXT_PUBLIC_MAX_FILE_BUFFER}MB.`, + }); + const linkStillExists = await prisma.link.findUnique({ where: { id: linkId }, }); if (linkStillExists && files.file[0].mimetype?.includes("image")) { - const collectionId = collectionPermissions?.id as number; + const collectionId = collectionPermissions.id as number; createFolder({ filePath: `archives/preview/${collectionId}`, }); @@ -137,9 +167,7 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) { if (linkStillExists) { await createFile({ - filePath: `archives/${collectionPermissions?.id}/${ - linkId + suffix - }`, + filePath: `archives/${collectionPermissions.id}/${linkId + suffix}`, data: fileBuffer, }); @@ -150,10 +178,10 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) { ? "unavailable" : undefined, image: files.file[0].mimetype?.includes("image") - ? `archives/${collectionPermissions?.id}/${linkId + suffix}` + ? `archives/${collectionPermissions.id}/${linkId + suffix}` : null, pdf: files.file[0].mimetype?.includes("pdf") - ? `archives/${collectionPermissions?.id}/${linkId + suffix}` + ? `archives/${collectionPermissions.id}/${linkId + suffix}` : null, lastPreserved: new Date().toISOString(), }, diff --git a/pages/api/v1/links/[id]/archive/index.ts b/pages/api/v1/links/[id]/archive/index.ts index 78353bb..ef91a4e 100644 --- a/pages/api/v1/links/[id]/archive/index.ts +++ b/pages/api/v1/links/[id]/archive/index.ts @@ -76,6 +76,7 @@ const deleteArchivedFiles = async (link: Link & { collection: Collection }) => { image: null, pdf: null, readable: null, + monolith: null, preview: null, }, }); diff --git a/pages/collections/[id].tsx b/pages/collections/[id].tsx index 58d474d..bdfbfe9 100644 --- a/pages/collections/[id].tsx +++ b/pages/collections/[id].tsx @@ -59,6 +59,7 @@ export default function Index() { username: "", image: "", archiveAsScreenshot: undefined as unknown as boolean, + archiveAsMonolith: undefined as unknown as boolean, archiveAsPDF: undefined as unknown as boolean, }); @@ -76,6 +77,7 @@ export default function Index() { username: account.username as string, image: account.image as string, archiveAsScreenshot: account.archiveAsScreenshot as boolean, + archiveAsMonolith: account.archiveAsScreenshot as boolean, archiveAsPDF: account.archiveAsPDF as boolean, }); } diff --git a/pages/preserved/[id].tsx b/pages/preserved/[id].tsx index 84e614a..99fe033 100644 --- a/pages/preserved/[id].tsx +++ b/pages/preserved/[id].tsx @@ -6,6 +6,7 @@ import { LinkIncludingShortenedCollectionAndTags, } from "@/types/global"; import ReadableView from "@/components/ReadableView"; +import getServerSideProps from "@/lib/client/getServerSideProps"; export default function Index() { const { links, getLink } = useLinkStore(); @@ -36,6 +37,12 @@ export default function Index() { {link && Number(router.query.format) === ArchivedFormat.readability && ( <ReadableView link={link} /> )} + {link && Number(router.query.format) === ArchivedFormat.monolith && ( + <iframe + src={`/api/v1/archives/${link.id}?format=${ArchivedFormat.monolith}`} + className="w-full h-screen border-none" + ></iframe> + )} {link && Number(router.query.format) === ArchivedFormat.pdf && ( <iframe src={`/api/v1/archives/${link.id}?format=${ArchivedFormat.pdf}`} @@ -59,3 +66,5 @@ export default function Index() { </div> ); } + +export { getServerSideProps }; diff --git a/pages/public/collections/[id].tsx b/pages/public/collections/[id].tsx index 8377518..47e074f 100644 --- a/pages/public/collections/[id].tsx +++ b/pages/public/collections/[id].tsx @@ -42,6 +42,7 @@ export default function PublicCollections() { username: "", image: "", archiveAsScreenshot: undefined as unknown as boolean, + archiveAsMonolith: undefined as unknown as boolean, archiveAsPDF: undefined as unknown as boolean, }); diff --git a/pages/public/preserved/[id].tsx b/pages/public/preserved/[id].tsx index 79710bf..fdb332a 100644 --- a/pages/public/preserved/[id].tsx +++ b/pages/public/preserved/[id].tsx @@ -6,6 +6,7 @@ import { LinkIncludingShortenedCollectionAndTags, } from "@/types/global"; import ReadableView from "@/components/ReadableView"; +import getServerSideProps from "@/lib/client/getServerSideProps"; export default function Index() { const { links, getLink } = useLinkStore(); @@ -61,3 +62,5 @@ export default function Index() { </div> ); } + +export { getServerSideProps }; diff --git a/pages/settings/preference.tsx b/pages/settings/preference.tsx index 7579c57..55c0167 100644 --- a/pages/settings/preference.tsx +++ b/pages/settings/preference.tsx @@ -25,14 +25,21 @@ export default function Appearance() { const [archiveAsPDF, setArchiveAsPDF] = useState<boolean>( account.archiveAsPDF ); + + const [archiveAsMonolith, setArchiveAsMonolith] = useState<boolean>( + account.archiveAsMonolith + ); + const [archiveAsWaybackMachine, setArchiveAsWaybackMachine] = useState<boolean>(account.archiveAsWaybackMachine); + const [linksRouteTo, setLinksRouteTo] = useState(account.linksRouteTo); useEffect(() => { setUser({ ...account, archiveAsScreenshot, + archiveAsMonolith, archiveAsPDF, archiveAsWaybackMachine, linksRouteTo, @@ -41,6 +48,7 @@ export default function Appearance() { }, [ account, archiveAsScreenshot, + archiveAsMonolith, archiveAsPDF, archiveAsWaybackMachine, linksRouteTo, @@ -54,6 +62,7 @@ export default function Appearance() { useEffect(() => { if (!objectIsEmpty(account)) { setArchiveAsScreenshot(account.archiveAsScreenshot); + setArchiveAsMonolith(account.archiveAsMonolith); setArchiveAsPDF(account.archiveAsPDF); setArchiveAsWaybackMachine(account.archiveAsWaybackMachine); setLinksRouteTo(account.linksRouteTo); @@ -125,6 +134,13 @@ export default function Appearance() { state={archiveAsScreenshot} onClick={() => setArchiveAsScreenshot(!archiveAsScreenshot)} /> + + <Checkbox + label={t("webpage")} + state={archiveAsMonolith} + onClick={() => setArchiveAsMonolith(!archiveAsMonolith)} + /> + <Checkbox label={t("pdf")} state={archiveAsPDF} @@ -204,6 +220,24 @@ export default function Appearance() { </span> </label> + <label + className="label cursor-pointer flex gap-2 justify-start w-fit" + tabIndex={0} + role="button" + > + <input + type="radio" + name="link-preference-radio" + className="radio checked:bg-primary" + value="Monolith" + checked={linksRouteTo === LinksRouteTo.MONOLITH} + onChange={() => setLinksRouteTo(LinksRouteTo.MONOLITH)} + /> + <span className="label-text"> + {t("open_webpage_if_available")} + </span> + </label> + <label className="label cursor-pointer flex gap-2 justify-start w-fit" tabIndex={0} diff --git a/prisma/migrations/20240309180643_add_singlefile_archive_format/migration.sql b/prisma/migrations/20240309180643_add_singlefile_archive_format/migration.sql new file mode 100644 index 0000000..07ad8a5 --- /dev/null +++ b/prisma/migrations/20240309180643_add_singlefile_archive_format/migration.sql @@ -0,0 +1,8 @@ +-- AlterEnum +ALTER TYPE "LinksRouteTo" ADD VALUE 'SINGLEFILE'; + +-- AlterTable +ALTER TABLE "User" ADD COLUMN "archiveAsSinglefile" BOOLEAN NOT NULL DEFAULT false; + +-- AlterTable +ALTER TABLE "Link" ADD COLUMN "singlefile" text; diff --git a/prisma/migrations/20240628013747_rename_fields/migration.sql b/prisma/migrations/20240628013747_rename_fields/migration.sql new file mode 100644 index 0000000..6c63edd --- /dev/null +++ b/prisma/migrations/20240628013747_rename_fields/migration.sql @@ -0,0 +1,21 @@ +/* + Warnings: + + - The values [SINGLEFILE] on the enum `LinksRouteTo` will be removed. If these variants are still used in the database, this will fail. + - You are about to drop the column `archiveAsSinglefile` on the `User` table. All the data in the column will be lost. + +*/ +-- AlterEnum +BEGIN; +CREATE TYPE "LinksRouteTo_new" AS ENUM ('ORIGINAL', 'PDF', 'READABLE', 'MONOLITH', 'SCREENSHOT'); +ALTER TABLE "User" ALTER COLUMN "linksRouteTo" DROP DEFAULT; +ALTER TABLE "User" ALTER COLUMN "linksRouteTo" TYPE "LinksRouteTo_new" USING ("linksRouteTo"::text::"LinksRouteTo_new"); +ALTER TYPE "LinksRouteTo" RENAME TO "LinksRouteTo_old"; +ALTER TYPE "LinksRouteTo_new" RENAME TO "LinksRouteTo"; +DROP TYPE "LinksRouteTo_old"; +ALTER TABLE "User" ALTER COLUMN "linksRouteTo" SET DEFAULT 'ORIGINAL'; +COMMIT; + +-- AlterTable +ALTER TABLE "User" DROP COLUMN "archiveAsSinglefile", +ADD COLUMN "archiveAsMonolith" BOOLEAN NOT NULL DEFAULT true; diff --git a/prisma/migrations/20240628014129_rename_field/migration.sql b/prisma/migrations/20240628014129_rename_field/migration.sql new file mode 100644 index 0000000..d6336e8 --- /dev/null +++ b/prisma/migrations/20240628014129_rename_field/migration.sql @@ -0,0 +1,9 @@ +/* + Warnings: + + - You are about to drop the column `singlefile` on the `Link` table. All the data in the column will be lost. + +*/ +-- AlterTable +ALTER TABLE "Link" DROP COLUMN "singlefile", +ADD COLUMN "monolith" TEXT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 37c544a..71b5b10 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -47,6 +47,7 @@ model User { linksRouteTo LinksRouteTo @default(ORIGINAL) preventDuplicateLinks Boolean @default(false) archiveAsScreenshot Boolean @default(true) + archiveAsMonolith Boolean @default(true) archiveAsPDF Boolean @default(true) archiveAsWaybackMachine Boolean @default(false) isPrivate Boolean @default(false) @@ -58,6 +59,7 @@ enum LinksRouteTo { ORIGINAL PDF READABLE + MONOLITH SCREENSHOT } @@ -137,6 +139,7 @@ model Link { image String? pdf String? readable String? + monolith String? lastPreserved DateTime? importDate DateTime? createdAt DateTime @default(now()) diff --git a/public/locales/en/common.json b/public/locales/en/common.json index bb6cf70..f1b5c72 100644 --- a/public/locales/en/common.json +++ b/public/locales/en/common.json @@ -166,6 +166,7 @@ "open_pdf_if_available": "Open PDF, if available", "open_readable_if_available": "Open Readable, if available", "open_screenshot_if_available": "Open Screenshot, if available", + "open_webpage_if_available": "Open Webpage copy, if available", "tag_renamed": "Tag renamed!", "tag_deleted": "Tag deleted!", "rename_tag": "Rename Tag", @@ -221,8 +222,9 @@ "github": "GitHub", "twitter": "Twitter", "mastodon": "Mastodon", - "link_preservation_in_queue": "LThe Link preservation is currently in the queue", + "link_preservation_in_queue": "The Link preservation is currently in the queue", "check_back_later": "Please check back later to see the result", + "there_are_more_formats": "There are more preserved formats in the queue", "settings": "Settings", "switch_to": "Switch to {{theme}}", "logout": "Logout", @@ -360,5 +362,6 @@ "show_link_details": "Show Link Details", "hide_link_details": "Hide Link Details", "link_pinned": "Link Pinned!", - "link_unpinned": "Link Unpinned!" + "link_unpinned": "Link Unpinned!", + "webpage": "Webpage" } \ No newline at end of file diff --git a/scripts/worker.ts b/scripts/worker.ts index 6b646c5..ca7a6ca 100644 --- a/scripts/worker.ts +++ b/scripts/worker.ts @@ -38,6 +38,13 @@ async function processBatch() { { readable: "pending", }, + /////////////////////// + { + monolith: null, + }, + { + monolith: "pending", + }, ], }, take: archiveTakeCount, @@ -75,6 +82,13 @@ async function processBatch() { { readable: "pending", }, + /////////////////////// + { + monolith: null, + }, + { + monolith: "pending", + }, ], }, take: archiveTakeCount, diff --git a/types/enviornment.d.ts b/types/enviornment.d.ts deleted file mode 100644 index 32bc29d..0000000 --- a/types/enviornment.d.ts +++ /dev/null @@ -1,423 +0,0 @@ -declare global { - namespace NodeJS { - interface ProcessEnv { - NEXTAUTH_SECRET: string; - DATABASE_URL: string; - NEXTAUTH_URL: string; - NEXT_PUBLIC_DISABLE_REGISTRATION?: string; - PAGINATION_TAKE_COUNT?: string; - STORAGE_FOLDER?: string; - AUTOSCROLL_TIMEOUT?: string; - RE_ARCHIVE_LIMIT?: string; - NEXT_PUBLIC_MAX_FILE_SIZE?: string; - MAX_LINKS_PER_USER?: string; - ARCHIVE_TAKE_COUNT?: string; - IGNORE_UNAUTHORIZED_CA?: string; - IGNORE_URL_SIZE_LIMIT?: string; - ADMINISTRATOR?: string; - - SPACES_KEY?: string; - SPACES_SECRET?: string; - SPACES_ENDPOINT?: string; - SPACES_BUCKET_NAME?: string; - SPACES_REGION?: string; - SPACES_FORCE_PATH_STYLE?: string; - - NEXT_PUBLIC_CREDENTIALS_ENABLED?: string; - DISABLE_NEW_SSO_USERS?: string; - - NEXT_PUBLIC_EMAIL_PROVIDER?: string; - EMAIL_FROM?: string; - EMAIL_SERVER?: string; - - BASE_URL?: string; // Used for email and stripe - - NEXT_PUBLIC_STRIPE?: string; - STRIPE_SECRET_KEY?: string; - MONTHLY_PRICE_ID?: string; - YEARLY_PRICE_ID?: string; - NEXT_PUBLIC_STRIPE_BILLING_PORTAL_URL?: string; - NEXT_PUBLIC_TRIAL_PERIOD_DAYS?: string; - - // Proxy settings - PROXY?: string; - PROXY_USERNAME?: string; - PROXY_PASSWORD?: string; - PROXY_BYPASS?: string; - - // PDF archive settings - PDF_MARGIN_TOP?: string; - PDF_MARGIN_BOTTOM?: string; - - // - // SSO Providers - // - - // 42 School - NEXT_PUBLIC_FORTYTWO_ENABLED?: string; - FORTYTWO_CUSTOM_NAME?: string; - FORTYTWO_CLIENT_ID?: string; - FORTYTWO_CLIENT_SECRET?: string; - - // Apple - NEXT_PUBLIC_APPLE_ENABLED?: string; - APPLE_CUSTOM_NAME?: string; - APPLE_ID?: string; - APPLE_SECRET?: string; - - // Atlassian - NEXT_PUBLIC_ATLASSIAN_ENABLED?: string; - ATLASSIAN_CUSTOM_NAME?: string; - ATLASSIAN_CLIENT_ID?: string; - ATLASSIAN_CLIENT_SECRET?: string; - ATLASSIAN_SCOPE?: string; - - // Auth0 - NEXT_PUBLIC_AUTH0_ENABLED?: string; - AUTH0_CUSTOM_NAME?: string; - AUTH0_ISSUER?: string; - AUTH0_CLIENT_SECRET?: string; - AUTH0_CLIENT_ID?: string; - - // Authelia - NEXT_PUBLIC_AUTHELIA_ENABLED?: string; - AUTHELIA_CUSTOM_NAME?: string; - AUTHELIA_CLIENT_ID?: string; - AUTHELIA_CLIENT_SECRET?: string; - AUTHELIA_WELLKNOWN_URL?: string; - - // Authentik - NEXT_PUBLIC_AUTHENTIK_ENABLED?: string; - AUTHENTIK_CUSTOM_NAME?: string; - AUTHENTIK_ISSUER?: string; - AUTHENTIK_CLIENT_ID?: string; - AUTHENTIK_CLIENT_SECRET?: string; - - // TODO: Azure AD B2C - // TODO: Azure AD - - // Battle.net - NEXT_PUBLIC_BATTLENET_ENABLED?: string; - BATTLENET_CUSTOM_NAME?: string; - BATTLENET_CLIENT_ID?: string; - BATTLENET_CLIENT_SECRET?: string; - BATLLENET_ISSUER?: string; - - // Box - NEXT_PUBLIC_BOX_ENABLED?: string; - BOX_CUSTOM_NAME?: string; - BOX_CLIENT_ID?: string; - BOX_CLIENT_SECRET?: string; - - // TODO: BoxyHQ SAML - - // Bungie - NEXT_PUBLIC_BUNGIE_ENABLED?: string; - BUNGIE_CUSTOM_NAME?: string; - BUNGIE_CLIENT_ID?: string; - BUNGIE_CLIENT_SECRET?: string; - BUNGIE_API_KEY?: string; - - // Cognito - NEXT_PUBLIC_COGNITO_ENABLED?: string; - COGNITO_CUSTOM_NAME?: string; - COGNITO_CLIENT_ID?: string; - COGNITO_CLIENT_SECRET?: string; - COGNITO_ISSUER?: string; - - // Coinbase - NEXT_PUBLIC_COINBASE_ENABLED?: string; - COINBASE_CUSTOM_NAME?: string; - COINBASE_CLIENT_ID?: string; - COINBASE_CLIENT_SECRET?: string; - - // Discord - NEXT_PUBLIC_DISCORD_ENABLED?: string; - DISCORD_CUSTOM_NAME?: string; - DISCORD_CLIENT_ID?: string; - DISCORD_CLIENT_SECRET?: string; - - // Dropbox - NEXT_PUBLIC_DROPBOX_ENABLED?: string; - DROPBOX_CUSTOM_NAME?: string; - DROPBOX_CLIENT_ID?: string; - DROPBOX_CLIENT_SECRET?: string; - - // DuendeIndentityServer6 - NEXT_PUBLIC_DUENDE_IDS6_ENABLED?: string; - DUENDE_IDS6_CUSTOM_NAME?: string; - DUENDE_IDS6_CLIENT_ID?: string; - DUENDE_IDS6_CLIENT_SECRET?: string; - DUENDE_IDS6_ISSUER?: string; - - // EVE Online - NEXT_PUBLIC_EVEONLINE_ENABLED?: string; - EVEONLINE_CUSTOM_NAME?: string; - EVEONLINE_CLIENT_ID?: string; - EVEONLINE_CLIENT_SECRET?: string; - - // Facebook - NEXT_PUBLIC_FACEBOOK_ENABLED?: string; - FACEBOOK_CUSTOM_NAME?: string; - FACEBOOK_CLIENT_ID?: string; - FACEBOOK_CLIENT_SECRET?: string; - - // FACEIT - NEXT_PUBLIC_FACEIT_ENABLED?: string; - FACEIT_CUSTOM_NAME?: string; - FACEIT_CLIENT_ID?: string; - FACEIT_CLIENT_SECRET?: string; - - // Foursquare - NEXT_PUBLIC_FOURSQUARE_ENABLED?: string; - FOURSQUARE_CUSTOM_NAME?: string; - FOURSQUARE_CLIENT_ID?: string; - FOURSQUARE_CLIENT_SECRET?: string; - FOURSQUARE_APIVERSION?: string; - - // Freshbooks - NEXT_PUBLIC_FRESHBOOKS_ENABLED?: string; - FRESHBOOKS_CUSTOM_NAME?: string; - FRESHBOOKS_CLIENT_ID?: string; - FRESHBOOKS_CLIENT_SECRET?: string; - - // FusionAuth - NEXT_PUBLIC_FUSIONAUTH_ENABLED?: string; - FUSIONAUTH_CUSTOM_NAME?: string; - FUSIONAUTH_CLIENT_ID?: string; - FUSIONAUTH_CLIENT_SECRET?: string; - FUSIONAUTH_ISSUER?: string; - FUSIONAUTH_TENANT_ID?: string; - - // GitHub - NEXT_PUBLIC_GITHUB_ENABLED?: string; - GITHUB_CUSTOM_NAME?: string; - GITHUB_CLIENT_ID?: string; - GITHUB_CLIENT_SECRET?: string; - - // GitLab - NEXT_PUBLIC_GITLAB_ENABLED?: string; - GITLAB_CUSTOM_NAME?: string; - GITLAB_CLIENT_ID?: string; - GITLAB_CLIENT_SECRET?: string; - - // Google - NEXT_PUBLIC_GOOGLE_ENABLED?: string; - GOOGLE_CUSTOM_NAME?: string; - GOOGLE_CLIENT_ID?: string; - GOOGLE_CLIENT_SECRET?: string; - - // HubSpot - NEXT_PUBLIC_HUBSPOT_ENABLED?: string; - HUBSPOT_CUSTOM_NAME?: string; - HUBSPOT_CLIENT_ID?: string; - HUBSPOT_CLIENT_SECRET?: string; - - // IdentityServer4 - NEXT_PUBLIC_IDS4_ENABLED?: string; - IDS4_CUSTOM_NAME?: string; - IDS4_CLIENT_ID?: string; - IDS4_CLIENT_SECRET?: string; - IDS4_ISSUER?: string; - - // TODO: Instagram (Doesn't return email) - - // Kakao - NEXT_PUBLIC_KAKAO_ENABLED?: string; - KAKAO_CUSTOM_NAME?: string; - KAKAO_CLIENT_ID?: string; - KAKAO_CLIENT_SECRET?: string; - - // Keycloak - NEXT_PUBLIC_KEYCLOAK_ENABLED?: string; - KEYCLOAK_CUSTOM_NAME?: string; - KEYCLOAK_ISSUER?: string; - KEYCLOAK_CLIENT_ID?: string; - KEYCLOAK_CLIENT_SECRET?: string; - - // LINE - NEXT_PUBLIC_LINE_ENABLED?: string; - LINE_CUSTOM_NAME?: string; - LINE_CLIENT_ID?: string; - LINE_CLIENT_SECRET?: string; - - // LinkedIn - NEXT_PUBLIC_LINKEDIN_ENABLED?: string; - LINKEDIN_CUSTOM_NAME?: string; - LINKEDIN_CLIENT_ID?: string; - LINKEDIN_CLIENT_SECRET?: string; - - // Mailchimp - NEXT_PUBLIC_MAILCHIMP_ENABLED?: string; - MAILCHIMP_CUSTOM_NAME?: string; - MAILCHIMP_CLIENT_ID?: string; - MAILCHIMP_CLIENT_SECRET?: string; - - // Mail.ru - NEXT_PUBLIC_MAILRU_ENABLED?: string; - MAILRU_CUSTOM_NAME?: string; - MAILRU_CLIENT_ID?: string; - MAILRU_CLIENT_SECRET?: string; - - // TODO: Medium (Doesn't return email) - - // Naver - NEXT_PUBLIC_NAVER_ENABLED?: string; - NAVER_CUSTOM_NAME?: string; - NAVER_CLIENT_ID?: string; - NAVER_CLIENT_SECRET?: string; - - // Netlify - NEXT_PUBLIC_NETLIFY_ENABLED?: string; - NETLIFY_CUSTOM_NAME?: string; - NETLIFY_CLIENT_ID?: string; - NETLIFY_CLIENT_SECRET?: string; - - // Okta - NEXT_PUBLIC_OKTA_ENABLED?: string; - OKTA_CUSTOM_NAME?: string; - OKTA_CLIENT_ID?: string; - OKTA_CLIENT_SECRET?: string; - OKTA_ISSUER?: string; - - // OneLogin - NEXT_PUBLIC_ONELOGIN_ENABLED?: string; - ONELOGIN_CUSTOM_NAME?: string; - ONELOGIN_CLIENT_ID?: string; - ONELOGIN_CLIENT_SECRET?: string; - ONELOGIN_ISSUER?: string; - - // Osso - NEXT_PUBLIC_OSSO_ENABLED?: string; - OSSO_CUSTOM_NAME?: string; - OSSO_CLIENT_ID?: string; - OSSO_CLIENT_SECRET?: string; - OSSO_ISSUER?: string; - - // osu! - NEXT_PUBLIC_OSU_ENABLED?: string; - OSU_CUSTOM_NAME?: string; - OSU_CLIENT_ID?: string; - OSU_CLIENT_SECRET?: string; - - // Patreon - NEXT_PUBLIC_PATREON_ENABLED?: string; - PATREON_CUSTOM_NAME?: string; - PATREON_CLIENT_ID?: string; - PATREON_CLIENT_SECRET?: string; - - // Pinterest - NEXT_PUBLIC_PINTEREST_ENABLED?: string; - PINTEREST_CUSTOM_NAME?: string; - PINTEREST_CLIENT_ID?: string; - PINTEREST_CLIENT_SECRET?: string; - - // Pipedrive - NEXT_PUBLIC_PIPEDRIVE_ENABLED?: string; - PIPEDRIVE_CUSTOM_NAME?: string; - PIPEDRIVE_CLIENT_ID?: string; - PIPEDRIVE_CLIENT_SECRET?: string; - - // Reddit - // TODO (1h tokens) - NEXT_PUBLIC_REDDIT_ENABLED?: string; - REDDIT_CUSTOM_NAME?: string; - REDDIT_CLIENT_ID?: string; - REDDIT_CLIENT_SECRET?: string; - - // Salesforce - NEXT_PUBLIC_SALESFORCE_ENABLED?: string; - SALESFORCE_CUSTOM_NAME?: string; - SALESFORCE_CLIENT_ID?: string; - SALESFORCE_CLIENT_SECRET?: string; - - // Slack - NEXT_PUBLIC_SLACK_ENABLED?: string; - SLACK_CUSTOM_NAME?: string; - SLACK_CLIENT_ID?: string; - SLACK_CLIENT_SECRET?: string; - - // Spotify - NEXT_PUBLIC_SPOTIFY_ENABLED?: string; - SPOTIFY_CUSTOM_NAME?: string; - SPOTIFY_CLIENT_ID?: string; - SPOTIFY_CLIENT_SECRET?: string; - - // Strava - NEXT_PUBLIC_STRAVA_ENABLED?: string; - STRAVA_CUSTOM_NAME?: string; - STRAVA_CLIENT_ID?: string; - STRAVA_CLIENT_SECRET?: string; - - // Todoist - NEXT_PUBLIC_TODOIST_ENABLED?: string; - TODOIST_CUSTOM_NAME?: string; - TODOIST_CLIENT_ID?: string; - TODOIST_CLIENT_SECRET?: string; - - // TODO: Trakt (Doesn't return email) - - // Twitch - NEXT_PUBLIC_TWITCH_ENABLED?: string; - TWITCH_CUSTOM_NAME?: string; - TWITCH_CLIENT_ID?: string; - TWITCH_CLIENT_SECRET?: string; - - // TODO: Twitter (OAuth 1.0) - - // United Effects - NEXT_PUBLIC_UNITED_EFFECTS_ENABLED?: string; - UNITED_EFFECTS_CUSTOM_NAME?: string; - UNITED_EFFECTS_CLIENT_ID?: string; - UNITED_EFFECTS_CLIENT_SECRET?: string; - UNITED_EFFECTS_ISSUER?: string; - - // VK - NEXT_PUBLIC_VK_ENABLED?: string; - VK_CUSTOM_NAME?: string; - VK_CLIENT_ID?: string; - VK_CLIENT_SECRET?: string; - - // Wikimedia - NEXT_PUBLIC_WIKIMEDIA_ENABLED?: string; - WIKIMEDIA_CUSTOM_NAME?: string; - WIKIMEDIA_CLIENT_ID?: string; - WIKIMEDIA_CLIENT_SECRET?: string; - - // Wordpress.com - NEXT_PUBLIC_WORDPRESS_ENABLED?: string; - WORDPRESS_CUSTOM_NAME?: string; - WORDPRESS_CLIENT_ID?: string; - WORDPRESS_CLIENT_SECRET?: string; - - // TODO: WorkOS (Custom flow) - - // Yandex - NEXT_PUBLIC_YANDEX_ENABLED?: string; - YANDEX_CUSTOM_NAME?: string; - YANDEX_CLIENT_ID?: string; - YANDEX_CLIENT_SECRET?: string; - - // Zitadel - NEXT_PUBLIC_ZITADEL_ENABLED?: string; - ZITADEL_CUSTOM_NAME?: string; - ZITADEL_CLIENT_ID?: string; - ZITADEL_CLIENT_SECRET?: string; - ZITADEL_ISSUER?: string; - - // Zoho - NEXT_PUBLIC_ZOHO_ENABLED?: string; - ZOHO_CUSTOM_NAME?: string; - ZOHO_CLIENT_ID?: string; - ZOHO_CLIENT_SECRET?: string; - - // Zoom - NEXT_PUBLIC_ZOOM_ENABLED?: string; - ZOOM_CUSTOM_NAME?: string; - ZOOM_CLIENT_ID?: string; - ZOOM_CLIENT_SECRET?: string; - } - } -} - -export {}; diff --git a/types/global.ts b/types/global.ts index fbdaea2..90c9aef 100644 --- a/types/global.ts +++ b/types/global.ts @@ -137,12 +137,14 @@ export enum ArchivedFormat { jpeg, pdf, readability, + monolith, } export enum LinkType { url, pdf, image, + monolith, } export enum TokenExpiry { diff --git a/yarn.lock b/yarn.lock index 55074b6..943096e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1293,12 +1293,12 @@ tiny-glob "^0.2.9" tslib "^2.4.0" -"@playwright/test@^1.43.1": - version "1.43.1" - resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.43.1.tgz#16728a59eb8ce0f60472f98d8886d6cab0fa3e42" - integrity sha512-HgtQzFgNEEo4TE22K/X7sYTYNqEMMTZmFS8kTq6m8hXj+m1D8TgwgIbumHddJa9h4yl4GkKb8/bgAl2+g7eDgA== +"@playwright/test@^1.45.0": + version "1.45.0" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.45.0.tgz#790a66165a46466c0d7099dd260881802f5aba7e" + integrity sha512-TVYsfMlGAaxeUllNkywbwek67Ncf8FRGn8ZlRdO291OL3NjG9oMbfVhyP82HQF0CZLMrYsvesqoUekxdWuF9Qw== dependencies: - playwright "1.43.1" + playwright "1.45.0" "@prisma/client@^4.16.2": version "4.16.2" @@ -5005,17 +5005,17 @@ pixelmatch@^4.0.2: dependencies: pngjs "^3.0.0" -playwright-core@1.43.1: - version "1.43.1" - resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.43.1.tgz#0eafef9994c69c02a1a3825a4343e56c99c03b02" - integrity sha512-EI36Mto2Vrx6VF7rm708qSnesVQKbxEWvPrfA1IPY6HgczBplDx7ENtx+K2n4kJ41sLLkuGfmb0ZLSSXlDhqPg== +playwright-core@1.45.0: + version "1.45.0" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.45.0.tgz#5741a670b7c9060ce06852c0051d84736fb94edc" + integrity sha512-lZmHlFQ0VYSpAs43dRq1/nJ9G/6SiTI7VPqidld9TDefL9tX87bTKExWZZUF5PeRyqtXqd8fQi2qmfIedkwsNQ== -playwright@1.43.1, playwright@^1.43.1: - version "1.43.1" - resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.43.1.tgz#8ad08984ac66c9ef3d0db035be54dd7ec9f1c7d9" - integrity sha512-V7SoH0ai2kNt1Md9E3Gwas5B9m8KR2GVvwZnAI6Pg0m3sh7UvgiYhRrhsziCmqMJNouPckiOhk8T+9bSAK0VIA== +playwright@1.45.0, playwright@^1.45.0: + version "1.45.0" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.45.0.tgz#400c709c64438690f13705cb9c88ef93089c5c27" + integrity sha512-4z3ac3plDfYzGB6r0Q3LF8POPR20Z8D0aXcxbJvmfMgSSq1hkcgvFRXJk9rUq5H/MJ0Ktal869hhOdI/zUTeLA== dependencies: - playwright-core "1.43.1" + playwright-core "1.45.0" optionalDependencies: fsevents "2.3.2"