xwx.moe/el.xwx.moe
xwx.moe/ el.xwx.moe
1
0
Disbranĉigi 0
Fontkodo Issues Tirpetoj Actions Packages Projects Eldonoj Wiki Activity

Merge pull request #619 from linkwarden/feat/single-file 

Feat/Monolith + Optimizations
This commit is contained in:
Daniel 2024-06-30 01:28:22 +03:30 committed by GitHub
parent 6003c6c449 78c80a5fea
commit 9ab01da369
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
59 changed files with 947 additions and 919 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.env.sample
View File

@ -15,7 +15,6 @@ NEXT_PUBLIC_DISABLE_REGISTRATION=
NEXT_PUBLIC_CREDENTIALS_ENABLED=
DISABLE_NEW_SSO_USERS=
RE_ARCHIVE_LIMIT=
NEXT_PUBLIC_MAX_FILE_SIZE=
MAX_LINKS_PER_USER=
ARCHIVE_TAKE_COUNT=
BROWSER_TIMEOUT=
@ -23,6 +22,13 @@ IGNORE_UNAUTHORIZED_CA=
IGNORE_HTTPS_ERRORS=
IGNORE_URL_SIZE_LIMIT=
ADMINISTRATOR=
NEXT_PUBLIC_MAX_FILE_BUFFER=
MONOLITH_MAX_BUFFER=
MONOLITH_CUSTOM_OPTIONS=
PDF_MAX_BUFFER=
SCREENSHOT_MAX_BUFFER=
READABILITY_MAX_BUFFER=
PREVIEW_MAX_BUFFER=
# AWS S3 Settings
SPACES_KEY=
@ -48,9 +54,9 @@ PROXY_BYPASS=
PDF_MARGIN_TOP=
PDF_MARGIN_BOTTOM=
#
# SSO Providers
#
#################
# SSO Providers #
#################
# 42 School
NEXT_PUBLIC_FORTYTWO_ENABLED=

19
Dockerfile
View File

@ -8,13 +8,30 @@ WORKDIR /data
COPY ./package.json ./yarn.lock ./playwright.config.ts ./
# Increase timeout to pass github actions arm64 build
RUN --mount=type=cache,sharing=locked,target=/usr/local/share/.cache/yarn yarn install --network-timeout 10000000
RUN apt-get update
RUN apt-get install -y \
build-essential \
curl \
libssl-dev \
pkg-config
RUN apt-get update
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
RUN cargo install monolith
RUN npx playwright install-deps && \
apt-get clean && \
yarn cache clean
RUN yarn playwright install
COPY . .
RUN yarn prisma generate && \

2
README.md
View File

@ -57,7 +57,7 @@ We've forked the old version from the current repository into [this repo](https:
## Features
- 📸 Auto capture a screenshot, PDF, and readable view of each webpage.
- 📸 Auto capture a screenshot, PDF, single html file, and readable view of each webpage.
- 🏛️ Send your webpage to Wayback Machine ([archive.org](https://archive.org)) for a snapshot. (Optional)
- 📂 Organize links by collection, sub-collection, name, description and multiple tags.
- 👥 Collaborate on gathering links in a collection.

4
components/Announcement.tsx
View File

@ -10,8 +10,8 @@ export default function Announcement({ toggleAnnouncementBar }: Props) {
const announcementId = localStorage.getItem("announcementId");
return (
<div className="fixed left-0 right-0 bottom-20 sm:bottom-10 w-full p-5 z-30">
<div className="mx-auto w-full p-2 flex justify-between gap-2 items-center border border-primary shadow-xl rounded-xl bg-base-300 backdrop-blur-sm bg-opacity-80 max-w-md">
<div className="fixed mx-auto bottom-20 sm:bottom-10 w-full pointer-events-none p-5 z-30">
<div className="mx-auto pointer-events-auto p-2 flex justify-between gap-2 items-center border border-primary shadow-xl rounded-xl bg-base-300 backdrop-blur-sm bg-opacity-80 max-w-md">
<i className="bi-stars text-2xl text-yellow-600 dark:text-yellow-500"></i>
<p className="w-4/5 text-center text-sm sm:text-base">
<Trans

2
components/CollectionCard.tsx
View File

@ -39,6 +39,7 @@ export default function CollectionCard({ collection, className }: Props) {
username: "",
image: "",
archiveAsScreenshot: undefined as unknown as boolean,
archiveAsMonolith: undefined as unknown as boolean,
archiveAsPDF: undefined as unknown as boolean,
});
@ -54,6 +55,7 @@ export default function CollectionCard({ collection, className }: Props) {
username: account.username as string,
image: account.image as string,
archiveAsScreenshot: account.archiveAsScreenshot as boolean,
archiveAsMonolith: account.archiveAsMonolith as boolean,
archiveAsPDF: account.archiveAsPDF as boolean,
});
}

9
components/LinkViews/LinkComponents/LinkIcon.tsx
View File

@ -70,7 +70,14 @@ export default function LinkIcon({
size={size}
icon="bi-file-earmark-image"
/>
) : undefined}
) : // : link.type === "monolith" ? (
// <LinkPlaceholderIcon
// iconClasses={iconClasses + dimension}
// size={size}
// icon="bi-filetype-html"
// />
// )
undefined}
</>
);
}

1
components/ModalContent/EditCollectionSharingModal.tsx
View File

@ -68,6 +68,7 @@ export default function EditCollectionSharingModal({
username: "",
image: "",
archiveAsScreenshot: undefined as unknown as boolean,
archiveAsMonolith: undefined as unknown as boolean,
archiveAsPDF: undefined as unknown as boolean,
});

1
components/ModalContent/NewLinkModal.tsx
View File

@ -30,6 +30,7 @@ export default function NewLinkModal({ onClose }: Props) {
image: "",
pdf: "",
readable: "",
monolith: "",
textContent: "",
collection: {
name: "",

127
components/ModalContent/PreservedFormatsModal.tsx
View File

@ -12,12 +12,14 @@ import { useSession } from "next-auth/react";
import {
pdfAvailable,
readabilityAvailable,
monolithAvailable,
screenshotAvailable,
} from "@/lib/shared/getArchiveValidity";
import PreservedFormatRow from "@/components/PreserverdFormatRow";
import useAccountStore from "@/store/account";
import getPublicUserData from "@/lib/client/getPublicUserData";
import { useTranslation } from "next-i18next";
import { BeatLoader } from "react-spinners";
type Props = {
onClose: Function;
@ -41,6 +43,7 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {
username: "",
image: "",
archiveAsScreenshot: undefined as unknown as boolean,
archiveAsMonolith: undefined as unknown as boolean,
archiveAsPDF: undefined as unknown as boolean,
});
@ -58,6 +61,7 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {
username: account.username as string,
image: account.image as string,
archiveAsScreenshot: account.archiveAsScreenshot as boolean,
archiveAsMonolith: account.archiveAsScreenshot as boolean,
archiveAsPDF: account.archiveAsPDF as boolean,
});
}
@ -72,6 +76,9 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {
(collectionOwner.archiveAsScreenshot === true
? link.pdf && link.pdf !== "pending"
: true) &&
(collectionOwner.archiveAsMonolith === true
? link.monolith && link.monolith !== "pending"
: true) &&
(collectionOwner.archiveAsPDF === true
? link.pdf && link.pdf !== "pending"
: true) &&
@ -80,6 +87,15 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {
);
};
const atLeastOneFormatAvailable = () => {
return (
screenshotAvailable(link) ||
pdfAvailable(link) ||
readabilityAvailable(link) ||
monolithAvailable(link)
);
};
useEffect(() => {
(async () => {
const data = await getLink(link.id as number, isPublic);
@ -108,7 +124,7 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {
clearInterval(interval);
}
};
}, [link, getLink]);
}, [link?.monolith]);
const updateArchive = async () => {
const load = toast.loading(t("sending_request"));
@ -133,56 +149,81 @@ export default function PreservedFormatsModal({ onClose, activeLink }: Props) {
<Modal toggleModal={onClose}>
<p className="text-xl font-thin">{t("preserved_formats")}</p>
<div className="divider mb-2 mt-1"></div>
{isReady() &&
(screenshotAvailable(link) ||
pdfAvailable(link) ||
readabilityAvailable(link)) ? (
{screenshotAvailable(link) ||
pdfAvailable(link) ||
readabilityAvailable(link) ||
monolithAvailable(link) ? (
<p className="mb-3">{t("available_formats")}</p>
) : (
""
)}
<div className="flex flex-col gap-3">
{isReady() ? (
<>
{screenshotAvailable(link) ? (
<PreservedFormatRow
name={t("screenshot")}
icon={"bi-file-earmark-image"}
format={
link?.image?.endsWith("png")
? ArchivedFormat.png
: ArchivedFormat.jpeg
}
activeLink={link}
downloadable={true}
/>
) : undefined}
{pdfAvailable(link) ? (
<PreservedFormatRow
name={t("pdf")}
icon="bi-file-earmark-pdf"
format={ArchivedFormat.pdf}
activeLink={link}
downloadable={true}
/>
) : undefined}
{readabilityAvailable(link) ? (
<PreservedFormatRow
name={t("readable")}
icon="bi-file-earmark-text"
format={ArchivedFormat.readability}
activeLink={link}
/>
) : undefined}
</>
) : (
<div className="w-full h-full flex flex-col justify-center p-10 skeleton bg-base-200">
<i className="bi-stack drop-shadow text-primary text-8xl mx-auto mb-5"></i>
<div className={`flex flex-col gap-3`}>
{monolithAvailable(link) ? (
<PreservedFormatRow
name={t("webpage")}
icon={"bi-filetype-html"}
format={ArchivedFormat.monolith}
activeLink={link}
downloadable={true}
/>
) : undefined}
{screenshotAvailable(link) ? (
<PreservedFormatRow
name={t("screenshot")}
icon={"bi-file-earmark-image"}
format={
link?.image?.endsWith("png")
? ArchivedFormat.png
: ArchivedFormat.jpeg
}
activeLink={link}
downloadable={true}
/>
) : undefined}
{pdfAvailable(link) ? (
<PreservedFormatRow
name={t("pdf")}
icon={"bi-file-earmark-pdf"}
format={ArchivedFormat.pdf}
activeLink={link}
downloadable={true}
/>
) : undefined}
{readabilityAvailable(link) ? (
<PreservedFormatRow
name={t("readable")}
icon={"bi-file-earmark-text"}
format={ArchivedFormat.readability}
activeLink={link}
/>
) : undefined}
{!isReady() && !atLeastOneFormatAvailable() ? (
<div className={`w-full h-full flex flex-col justify-center p-10`}>
<BeatLoader
color="oklch(var(--p))"
className="mx-auto mb-3"
size={30}
/>
<p className="text-center text-2xl">{t("preservation_in_queue")}</p>
<p className="text-center text-lg">{t("check_back_later")}</p>
</div>
)}
) : !isReady() && atLeastOneFormatAvailable() ? (
<div className={`w-full h-full flex flex-col justify-center p-5`}>
<BeatLoader
color="oklch(var(--p))"
className="mx-auto mb-3"
size={20}
/>
<p className="text-center">{t("there_are_more_formats")}</p>
<p className="text-center text-sm">{t("check_back_later")}</p>
</div>
) : undefined}
<div
className={`flex flex-col sm:flex-row gap-3 items-center justify-center ${

28
components/ModalContent/UploadFileModal.tsx
View File

@ -5,7 +5,10 @@ import TextInput from "@/components/TextInput";
import unescapeString from "@/lib/client/unescapeString";
import useCollectionStore from "@/store/collections";
import useLinkStore from "@/store/links";
import { LinkIncludingShortenedCollectionAndTags } from "@/types/global";
import {
LinkIncludingShortenedCollectionAndTags,
ArchivedFormat,
} from "@/types/global";
import { useSession } from "next-auth/react";
import { useRouter } from "next/router";
import toast from "react-hot-toast";
@ -30,6 +33,7 @@ export default function UploadFileModal({ onClose }: Props) {
image: "",
pdf: "",
readable: "",
monolith: "",
textContent: "",
collection: {
name: "",
@ -92,6 +96,24 @@ export default function UploadFileModal({ onClose }: Props) {
const submit = async () => {
if (!submitLoader && file) {
let fileType: ArchivedFormat | null = null;
let linkType: "url" | "image" | "monolith" | "pdf" | null = null;
if (file?.type === "image/jpg" || file.type === "image/jpeg") {
fileType = ArchivedFormat.jpeg;
linkType = "image";
} else if (file.type === "image/png") {
fileType = ArchivedFormat.png;
linkType = "image";
} else if (file.type === "application/pdf") {
fileType = ArchivedFormat.pdf;
linkType = "pdf";
}
// else if (file.type === "text/html") {
// fileType = ArchivedFormat.monolith;
// linkType = "monolith";
// }
setSubmitLoader(true);
const load = toast.loading(t("creating"));
@ -122,14 +144,14 @@ export default function UploadFileModal({ onClose }: Props) {
<label className="btn h-10 btn-sm w-full border border-neutral-content hover:border-neutral-content flex justify-between">
<input
type="file"
accept=".pdf,.png,.jpg,.jpeg"
accept=".pdf,.png,.jpg,.jpeg,.html"
className="cursor-pointer custom-file-input"
onChange={(e) => e.target.files && setFile(e.target.files[0])}
/>
</label>
<p className="text-xs font-semibold mt-2">
{t("file_types", {
size: process.env.NEXT_PUBLIC_MAX_FILE_SIZE || 30,
size: process.env.NEXT_PUBLIC_MAX_FILE_BUFFER || 10,
})}
</p>
</div>

16
components/PreserverdFormatRow.tsx
View File

@ -4,6 +4,7 @@ import {
ArchivedFormat,
LinkIncludingShortenedCollectionAndTags,
} from "@/types/global";
import toast from "react-hot-toast";
import Link from "next/link";
import { useRouter } from "next/router";
import { useSession } from "next-auth/react";
@ -60,7 +61,7 @@ export default function PreservedFormatRow({
clearInterval(interval);
}
};
}, [link?.image, link?.pdf, link?.readable]);
}, [link?.image, link?.pdf, link?.readable, link?.monolith]);
const handleDownload = () => {
const path = `/api/v1/archives/${link?.id}?format=${format}`;
@ -68,10 +69,15 @@ export default function PreservedFormatRow({
.then((response) => {
if (response.ok) {
// Create a temporary link and click it to trigger the download
const link = document.createElement("a");
link.href = path;
link.download = format === ArchivedFormat.pdf ? "PDF" : "Screenshot";
link.click();
const anchorElement = document.createElement("a");
anchorElement.href = path;
anchorElement.download =
format === ArchivedFormat.monolith
? "Webpage"
: format === ArchivedFormat.pdf
? "PDF"
: "Screenshot";
anchorElement.click();
} else {
console.error("Failed to download file");
}

6
components/ReadableView.tsx
View File

@ -81,9 +81,11 @@ export default function ReadableView({ link }: Props) {
(link?.image === "pending" ||
link?.pdf === "pending" ||
link?.readable === "pending" ||
link?.monolith === "pending" ||
!link?.image ||
!link?.pdf ||
!link?.readable)
!link?.readable ||
!link?.monolith)
) {
interval = setInterval(() => getLink(link.id as number), 5000);
} else {
@ -97,7 +99,7 @@ export default function ReadableView({ link }: Props) {
clearInterval(interval);
}
};
}, [link?.image, link?.pdf, link?.readable]);
}, [link?.image, link?.pdf, link?.readable, link?.monolith]);
const rgbToHex = (r: number, g: number, b: number): string =>
"#" +

304
lib/api/archiveHandler.ts
View File

@ -1,15 +1,16 @@
import { LaunchOptions, chromium, devices } from "playwright";
import { prisma } from "./db";
import createFile from "./storage/createFile";
import sendToWayback from "./sendToWayback";
import { Readability } from "@mozilla/readability";
import { JSDOM } from "jsdom";
import DOMPurify from "dompurify";
import sendToWayback from "./preservationScheme/sendToWayback";
import { Collection, Link, User } from "@prisma/client";
import validateUrlSize from "./validateUrlSize";
import fetchHeaders from "./fetchHeaders";
import createFolder from "./storage/createFolder";
import generatePreview from "./generatePreview";
import { removeFiles } from "./manageLinkFiles";
import handleMonolith from "./preservationScheme/handleMonolith";
import handleReadablility from "./preservationScheme/handleReadablility";
import handleArchivePreview from "./preservationScheme/handleArchivePreview";
import handleScreenshotAndPdf from "./preservationScheme/handleScreenshotAndPdf";
import imageHandler from "./preservationScheme/imageHandler";
import pdfHandler from "./preservationScheme/pdfHandler";
type LinksAndCollectionAndOwner = Link & {
collection: Collection & {
@ -20,6 +21,18 @@ type LinksAndCollectionAndOwner = Link & {
const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;
export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
const timeoutPromise = new Promise((_, reject) => {
setTimeout(
() =>
reject(
new Error(
`Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`
)
),
BROWSER_TIMEOUT * 60000
);
});
// allow user to configure a proxy
let browserOptions: LaunchOptions = {};
if (process.env.PROXY) {
@ -39,18 +52,6 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
const page = await context.newPage();
const timeoutPromise = new Promise((_, reject) => {
setTimeout(
() =>
reject(
new Error(
`Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`
)
),
BROWSER_TIMEOUT * 60000
);
});
createFolder({
filePath: `archives/preview/${link.collectionId}`,
});
@ -62,17 +63,11 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
try {
await Promise.race([
(async () => {
const validatedUrl = link.url
? await validateUrlSize(link.url)
: undefined;
const user = link.collection?.owner;
if (
validatedUrl === null &&
process.env.IGNORE_URL_SIZE_LIMIT !== "true"
)
throw "Something went wrong while retrieving the file size.";
const header = link.url ? await fetchHeaders(link.url) : undefined;
const contentType = validatedUrl?.get("content-type");
const contentType = header?.get("content-type");
let linkType = "url";
let imageExtension = "png";
@ -84,12 +79,7 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
else if (contentType.includes("image/png")) imageExtension = "png";
}
const user = link.collection?.owner;
// send to archive.org
if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url);
const targetLink = await prisma.link.update({
await prisma.link.update({
where: { id: link.id },
data: {
type: linkType,
@ -104,6 +94,9 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
readable: !link.readable?.startsWith("archive")
? "pending"
: undefined,
monolith: !link.monolith?.startsWith("archive")
? "pending"
: undefined,
preview: !link.readable?.startsWith("archive")
? "pending"
: undefined,
@ -111,6 +104,9 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
},
});
// send to archive.org
if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url);
if (linkType === "image" && !link.image?.startsWith("archive")) {
await imageHandler(link, imageExtension); // archive image (jpeg/png)
return;
@ -124,151 +120,37 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
const content = await page.content();
// TODO single file
// const session = await page.context().newCDPSession(page);
// const doc = await session.send("Page.captureSnapshot", {
// format: "mhtml",
// });
// const saveDocLocally = (doc: any) => {
// console.log(doc);
// return createFile({
// data: doc,
// filePath: `archives/${targetLink.collectionId}/${link.id}.mhtml`,
// });
// };
// saveDocLocally(doc.data);
// Preview
if (
!link.preview?.startsWith("archives") &&
!link.preview?.startsWith("unavailable")
)
await handleArchivePreview(link, page);
// Readability
const window = new JSDOM("").window;
const purify = DOMPurify(window);
const cleanedUpContent = purify.sanitize(content);
const dom = new JSDOM(cleanedUpContent, { url: link.url || "" });
const article = new Readability(dom.window.document).parse();
const articleText = article?.textContent
.replace(/ +(?= )/g, "") // strip out multiple spaces
.replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaks
if (
articleText &&
articleText !== "" &&
!link.readable?.startsWith("archive")
) {
await createFile({
data: JSON.stringify(article),
filePath: `archives/${targetLink.collectionId}/${link.id}_readability.json`,
});
await prisma.link.update({
where: { id: link.id },
data: {
readable: `archives/${targetLink.collectionId}/${link.id}_readability.json`,
textContent: articleText,
},
});
}
// Preview
const ogImageUrl = await page.evaluate(() => {
const metaTag = document.querySelector('meta[property="og:image"]');
return metaTag ? (metaTag as any).content : null;
});
if (ogImageUrl) {
console.log("Found og:image URL:", ogImageUrl);
// Download the image
const imageResponse = await page.goto(ogImageUrl);
// Check if imageResponse is not null
if (imageResponse && !link.preview?.startsWith("archive")) {
const buffer = await imageResponse.body();
await generatePreview(buffer, link.collectionId, link.id);
}
await page.goBack();
} else if (!link.preview?.startsWith("archive")) {
console.log("No og:image found");
await page
.screenshot({ type: "jpeg", quality: 20 })
.then((screenshot) => {
return createFile({
data: screenshot,
filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,
});
})
.then(() => {
return prisma.link.update({
where: { id: link.id },
data: {
preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,
},
});
});
}
!link.readable?.startsWith("archives") &&
!link.readable?.startsWith("unavailable")
)
await handleReadablility(content, link);
// Screenshot/PDF
await page.evaluate(
autoScroll,
Number(process.env.AUTOSCROLL_TIMEOUT) || 30
);
if (
(!link.image?.startsWith("archives") &&
!link.image?.startsWith("unavailable")) ||
(!link.pdf?.startsWith("archives") &&
!link.pdf?.startsWith("unavailable"))
)
await handleScreenshotAndPdf(link, page, user);
// Check if the user hasn't deleted the link by the time we're done scrolling
const linkExists = await prisma.link.findUnique({
where: { id: link.id },
});
if (linkExists) {
const processingPromises = [];
if (
user.archiveAsScreenshot &&
!link.image?.startsWith("archive")
) {
processingPromises.push(
page.screenshot({ fullPage: true }).then((screenshot) => {
return createFile({
data: screenshot,
filePath: `archives/${linkExists.collectionId}/${link.id}.png`,
});
})
);
}
// apply administrator's defined pdf margins or default to 15px
const margins = {
top: process.env.PDF_MARGIN_TOP || "15px",
bottom: process.env.PDF_MARGIN_BOTTOM || "15px",
};
if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {
processingPromises.push(
page
.pdf({
width: "1366px",
height: "1931px",
printBackground: true,
margin: margins,
})
.then((pdf) => {
return createFile({
data: pdf,
filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`,
});
})
);
}
await Promise.allSettled(processingPromises);
await prisma.link.update({
where: { id: link.id },
data: {
image: user.archiveAsScreenshot
? `archives/${linkExists.collectionId}/${link.id}.png`
: undefined,
pdf: user.archiveAsPDF
? `archives/${linkExists.collectionId}/${link.id}.pdf`
: undefined,
},
});
}
// Monolith
if (
!link.monolith?.startsWith("archive") &&
!link.monolith?.startsWith("unavailable") &&
user.archiveAsMonolith &&
link.url
)
await handleMonolith(link, content);
}
})(),
timeoutPromise,
@ -293,6 +175,9 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
image: !finalLink.image?.startsWith("archives")
? "unavailable"
: undefined,
monolith: !finalLink.monolith?.startsWith("archives")
? "unavailable"
: undefined,
pdf: !finalLink.pdf?.startsWith("archives")
? "unavailable"
: undefined,
@ -308,76 +193,3 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
await browser.close();
}
}
const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => {
const timeoutPromise = new Promise<void>((_, reject) => {
setTimeout(() => {
reject(new Error(`Webpage was too long to be archived.`));
}, AUTOSCROLL_TIMEOUT * 1000);
});
const scrollingPromise = new Promise<void>((resolve) => {
let totalHeight = 0;
let distance = 100;
let scrollDown = setInterval(() => {
let scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if (totalHeight >= scrollHeight) {
clearInterval(scrollDown);
window.scroll(0, 0);
resolve();
}
}, 100);
});
await Promise.race([scrollingPromise, timeoutPromise]);
};
const imageHandler = async ({ url, id }: Link, extension: string) => {
const image = await fetch(url as string).then((res) => res.blob());
const buffer = Buffer.from(await image.arrayBuffer());
const linkExists = await prisma.link.findUnique({
where: { id },
});
if (linkExists) {
await createFile({
data: buffer,
filePath: `archives/${linkExists.collectionId}/${id}.${extension}`,
});
await prisma.link.update({
where: { id },
data: {
image: `archives/${linkExists.collectionId}/${id}.${extension}`,
},
});
}
};
const pdfHandler = async ({ url, id }: Link) => {
const pdf = await fetch(url as string).then((res) => res.blob());
const buffer = Buffer.from(await pdf.arrayBuffer());
const linkExists = await prisma.link.findUnique({
where: { id },
});
if (linkExists) {
await createFile({
data: buffer,
filePath: `archives/${linkExists.collectionId}/${id}.pdf`,
});
await prisma.link.update({
where: { id },
data: {
pdf: `archives/${linkExists.collectionId}/${id}.pdf`,
},
});
}
};

4
lib/api/controllers/collections/collectionId/deleteCollectionById.ts
View File

@ -1,6 +1,6 @@
import { prisma } from "@/lib/api/db";
import getPermission from "@/lib/api/getPermission";
import { Collection, UsersAndCollections } from "@prisma/client";
import { UsersAndCollections } from "@prisma/client";
import removeFolder from "@/lib/api/storage/removeFolder";
export default async function deleteCollection(
@ -58,6 +58,7 @@ export default async function deleteCollection(
});
await removeFolder({ filePath: `archives/${collectionId}` });
await removeFolder({ filePath: `archives/preview/${collectionId}` });
await removeFromOrders(userId, collectionId);
@ -100,6 +101,7 @@ async function deleteSubCollections(collectionId: number) {
});
await removeFolder({ filePath: `archives/${subCollection.id}` });
await removeFolder({ filePath: `archives/preview/${subCollection.id}` });
}
}

3
lib/api/controllers/links/getLinks.ts
View File

@ -2,7 +2,8 @@ import { prisma } from "@/lib/api/db";
import { LinkRequestQuery, Sort } from "@/types/global";
export default async function getLink(userId: number, query: LinkRequestQuery) {
const POSTGRES_IS_ENABLED = process.env.DATABASE_URL.startsWith("postgresql");
const POSTGRES_IS_ENABLED =
process.env.DATABASE_URL?.startsWith("postgresql");
let order: any;
if (query.sort === Sort.DateNewestFirst) order = { id: "desc" };

1
lib/api/controllers/links/linkId/deleteLinkById.ts
View File

@ -1,7 +1,6 @@
import { prisma } from "@/lib/api/db";
import { Link, UsersAndCollections } from "@prisma/client";
import getPermission from "@/lib/api/getPermission";
import removeFile from "@/lib/api/storage/removeFile";
import { removeFiles } from "@/lib/api/manageLinkFiles";
export default async function deleteLink(userId: number, linkId: number) {

122
lib/api/controllers/links/postLink.ts
View File

@ -1,10 +1,8 @@
import { prisma } from "@/lib/api/db";
import { LinkIncludingShortenedCollectionAndTags } from "@/types/global";
import getTitle from "@/lib/shared/getTitle";
import { UsersAndCollections } from "@prisma/client";
import getPermission from "@/lib/api/getPermission";
import fetchTitleAndHeaders from "@/lib/shared/fetchTitleAndHeaders";
import createFolder from "@/lib/api/storage/createFolder";
import validateUrlSize from "../../validateUrlSize";
import setLinkCollection from "../../setLinkCollection";
const MAX_LINKS_PER_USER = Number(process.env.MAX_LINKS_PER_USER) || 30000;
@ -24,93 +22,10 @@ export default async function postLink(
}
}
if (!link.collection.id && link.collection.name) {
link.collection.name = link.collection.name.trim();
const linkCollection = await setLinkCollection(link, userId);
// find the collection with the name and the user's id
const findCollection = await prisma.collection.findFirst({
where: {
name: link.collection.name,
ownerId: userId,
parentId: link.collection.parentId,
},
});
if (findCollection) {
const collectionIsAccessible = await getPermission({
userId,
collectionId: findCollection.id,
});
const memberHasAccess = collectionIsAccessible?.members.some(
(e: UsersAndCollections) => e.userId === userId && e.canCreate
);
if (!(collectionIsAccessible?.ownerId === userId || memberHasAccess))
return { response: "Collection is not accessible.", status: 401 };
link.collection.id = findCollection.id;
link.collection.ownerId = findCollection.ownerId;
} else {
const collection = await prisma.collection.create({
data: {
name: link.collection.name,
ownerId: userId,
},
});
link.collection.id = collection.id;
await prisma.user.update({
where: {
id: userId,
},
data: {
collectionOrder: {
push: link.collection.id,
},
},
});
}
} else if (link.collection.id) {
const collectionIsAccessible = await getPermission({
userId,
collectionId: link.collection.id,
});
const memberHasAccess = collectionIsAccessible?.members.some(
(e: UsersAndCollections) => e.userId === userId && e.canCreate
);
if (!(collectionIsAccessible?.ownerId === userId || memberHasAccess))
return { response: "Collection is not accessible.", status: 401 };
} else if (!link.collection.id) {
link.collection.name = "Unorganized";
link.collection.parentId = null;
// find the collection with the name "Unorganized" and the user's id
const unorganizedCollection = await prisma.collection.findFirst({
where: {
name: "Unorganized",
ownerId: userId,
},
});
link.collection.id = unorganizedCollection?.id;
await prisma.user.update({
where: {
id: userId,
},
data: {
collectionOrder: {
push: link.collection.id,
},
},
});
} else {
return { response: "Uncaught error.", status: 500 };
}
if (!linkCollection)
return { response: "Collection is not accessible.", status: 400 };
const user = await prisma.user.findUnique({
where: {
@ -124,8 +39,6 @@ export default async function postLink(
const urlWithoutWww = hasWwwPrefix ? url?.replace(`://www.`, "://") : url;
const urlWithWww = hasWwwPrefix ? url : url?.replace("://", `://www.`);
console.log(url, urlWithoutWww, urlWithWww);
const existingLink = await prisma.link.findFirst({
where: {
OR: [{ url: urlWithWww }, { url: urlWithoutWww }],
@ -135,8 +48,6 @@ export default async function postLink(
},
});
console.log(url, urlWithoutWww, urlWithWww, "DONE!");
if (existingLink)
return {
response: "Link already exists",
@ -147,30 +58,23 @@ export default async function postLink(
const numberOfLinksTheUserHas = await prisma.link.count({
where: {
collection: {
ownerId: userId,
ownerId: linkCollection.ownerId,
},
},
});
if (numberOfLinksTheUserHas + 1 > MAX_LINKS_PER_USER)
if (numberOfLinksTheUserHas > MAX_LINKS_PER_USER)
return {
response: `Error: Each user can only have a maximum of ${MAX_LINKS_PER_USER} Links.`,
response: `Each collection owner can only have a maximum of ${MAX_LINKS_PER_USER} Links.`,
status: 400,
};
link.collection.name = link.collection.name.trim();
const title =
!(link.name && link.name !== "") && link.url
? await getTitle(link.url)
: "";
const { title, headers } = await fetchTitleAndHeaders(link.url || "");
const name =
link.name && link.name !== "" ? link.name : link.url ? title : "";
const validatedUrl = link.url ? await validateUrlSize(link.url) : undefined;
const contentType = validatedUrl?.get("content-type");
const contentType = headers?.get("content-type");
let linkType = "url";
let imageExtension = "png";
@ -190,7 +94,7 @@ export default async function postLink(
type: linkType,
collection: {
connect: {
id: link.collection.id,
id: linkCollection.id,
},
},
tags: {
@ -198,14 +102,14 @@ export default async function postLink(
where: {
name_ownerId: {
name: tag.name.trim(),
ownerId: link.collection.ownerId,
ownerId: linkCollection.ownerId,
},
},
create: {
name: tag.name.trim(),
owner: {
connect: {
id: link.collection.ownerId,
id: linkCollection.ownerId,
},
},
},

2
lib/api/controllers/migration/importFromHTMLFile.ts
View File

@ -31,7 +31,7 @@ export default async function importFromHTMLFile(
if (totalImports + numberOfLinksTheUserHas > MAX_LINKS_PER_USER)
return {
response: `Error: Each user can only have a maximum of ${MAX_LINKS_PER_USER} Links.`,
response: `Each collection owner can only have a maximum of ${MAX_LINKS_PER_USER} Links.`,
status: 400,
};

2
lib/api/controllers/migration/importFromLinkwarden.ts
View File

@ -26,7 +26,7 @@ export default async function importFromLinkwarden(
if (totalImports + numberOfLinksTheUserHas > MAX_LINKS_PER_USER)
return {
response: `Error: Each user can only have a maximum of ${MAX_LINKS_PER_USER} Links.`,
response: `Each collection owner can only have a maximum of ${MAX_LINKS_PER_USER} Links.`,
status: 400,
};

2
lib/api/controllers/migration/importFromWallabag.ts
View File

@ -47,7 +47,7 @@ export default async function importFromWallabag(
if (totalImports + numberOfLinksTheUserHas > MAX_LINKS_PER_USER)
return {
response: `Error: Each user can only have a maximum of ${MAX_LINKS_PER_USER} Links.`,
response: `Each collection owner can only have a maximum of ${MAX_LINKS_PER_USER} Links.`,
status: 400,
};

3
lib/api/controllers/public/links/getPublicLinksUnderCollection.ts
View File

@ -4,7 +4,8 @@ import { LinkRequestQuery, Sort } from "@/types/global";
export default async function getLink(
query: Omit<LinkRequestQuery, "tagId" | "pinnedOnly">
) {
const POSTGRES_IS_ENABLED = process.env.DATABASE_URL.startsWith("postgresql");
const POSTGRES_IS_ENABLED =
process.env.DATABASE_URL?.startsWith("postgresql");
let order: any;
if (query.sort === Sort.DateNewestFirst) order = { id: "desc" };

1
lib/api/controllers/public/users/getPublicUser.ts
View File

@ -75,6 +75,7 @@ export default async function getPublicUser(
username: lessSensitiveInfo.username,
image: lessSensitiveInfo.image,
archiveAsScreenshot: lessSensitiveInfo.archiveAsScreenshot,
archiveAsMonolith: lessSensitiveInfo.archiveAsMonolith,
archiveAsPDF: lessSensitiveInfo.archiveAsPDF,
};

4
lib/api/controllers/session/createSession.ts
View File

@ -21,12 +21,12 @@ export default async function createSession(
jti: crypto.randomUUID(),
},
maxAge: expiryDateSecond || 604800,
secret: process.env.NEXTAUTH_SECRET,
secret: process.env.NEXTAUTH_SECRET as string,
});
const tokenBody = await decode({
token,
secret: process.env.NEXTAUTH_SECRET,
secret: process.env.NEXTAUTH_SECRET as string,
});
const createToken = await prisma.accessToken.create({

4
lib/api/controllers/tokens/postToken.ts
View File

@ -65,12 +65,12 @@ export default async function postToken(
jti: crypto.randomUUID(),
},
maxAge: expiryDateSecond || 604800,
secret: process.env.NEXTAUTH_SECRET,
secret: process.env.NEXTAUTH_SECRET as string,
});
const tokenBody = await decode({
token,
secret: process.env.NEXTAUTH_SECRET,
secret: process.env.NEXTAUTH_SECRET as string,
});
const createToken = await prisma.accessToken.create({

2
lib/api/controllers/users/userId/deleteUserById.ts
View File

@ -80,7 +80,7 @@ export default async function deleteUserById(
});
// Delete archive folders
removeFolder({ filePath: `archives/${collection.id}` });
await removeFolder({ filePath: `archives/${collection.id}` });
await removeFolder({
filePath: `archives/preview/${collection.id}`,

1
lib/api/controllers/users/userId/updateUserById.ts
View File

@ -207,6 +207,7 @@ export default async function updateUserById(
),
locale: i18n.locales.includes(data.locale) ? data.locale : "en",
archiveAsScreenshot: data.archiveAsScreenshot,
archiveAsMonolith: data.archiveAsMonolith,
archiveAsPDF: data.archiveAsPDF,
archiveAsWaybackMachine: data.archiveAsWaybackMachine,
linksRouteTo: data.linksRouteTo,

18
lib/api/validateUrlSize.ts → lib/api/fetchHeaders.ts
View File

@ -2,7 +2,7 @@ import fetch from "node-fetch";
import https from "https";
import { SocksProxyAgent } from "socks-proxy-agent";
export default async function validateUrlSize(url: string) {
export default async function fetchHeaders(url: string) {
if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null;
try {
@ -29,13 +29,17 @@ export default async function validateUrlSize(url: string) {
};
}
const response = await fetch(url, fetchOpts);
const responsePromise = fetch(url, fetchOpts);
const totalSizeMB =
Number(response.headers.get("content-length")) / Math.pow(1024, 2);
if (totalSizeMB > (Number(process.env.NEXT_PUBLIC_MAX_FILE_SIZE) || 30))
return null;
else return response.headers;
const timeoutPromise = new Promise((_, reject) => {
setTimeout(() => {
reject(new Error("Fetch header timeout"));
}, 10 * 1000); // Stop after 10 seconds
});
const response = await Promise.race([responsePromise, timeoutPromise]);
return (response as Response)?.headers || null;
} catch (err) {
console.log(err);
return null;

49
lib/api/generatePreview.ts
View File

@ -1,7 +1,6 @@
import Jimp from "jimp";
import { prisma } from "./db";
import createFile from "./storage/createFile";
import createFolder from "./storage/createFolder";
const generatePreview = async (
buffer: Buffer,
@ -9,27 +8,39 @@ const generatePreview = async (
linkId: number
) => {
if (buffer && collectionId && linkId) {
// Load the image using Jimp
await Jimp.read(buffer, async (err, image) => {
if (image && !err) {
image?.resize(1280, Jimp.AUTO).quality(20);
const processedBuffer = await image?.getBufferAsync(Jimp.MIME_JPEG);
try {
const image = await Jimp.read(buffer);
createFile({
data: processedBuffer,
filePath: `archives/preview/${collectionId}/${linkId}.jpeg`,
}).then(() => {
return prisma.link.update({
where: { id: linkId },
data: {
preview: `archives/preview/${collectionId}/${linkId}.jpeg`,
},
});
});
if (!image) {
console.log("Error generating preview: Image not found");
return;
}
}).catch((err) => {
image.resize(1280, Jimp.AUTO).quality(20);
const processedBuffer = await image.getBufferAsync(Jimp.MIME_JPEG);
if (
Buffer.byteLength(processedBuffer) >
1024 * 1024 * Number(process.env.PREVIEW_MAX_BUFFER || 0.1)
) {
console.log("Error generating preview: Buffer size exceeded");
return;
}
await createFile({
data: processedBuffer,
filePath: `archives/preview/${collectionId}/${linkId}.jpeg`,
});
await prisma.link.update({
where: { id: linkId },
data: {
preview: `archives/preview/${collectionId}/${linkId}.jpeg`,
},
});
} catch (err) {
console.error("Error processing the image:", err);
});
}
}
};

7
lib/api/getPermission.ts
View File

@ -3,14 +3,12 @@ import { prisma } from "@/lib/api/db";
type Props = {
userId: number;
collectionId?: number;
collectionName?: string;
linkId?: number;
};
export default async function getPermission({
userId,
collectionId,
collectionName,
linkId,
}: Props) {
if (linkId) {
@ -26,11 +24,10 @@ export default async function getPermission({
});
return check;
} else if (collectionId || collectionName) {
} else if (collectionId) {
const check = await prisma.collection.findFirst({
where: {
id: collectionId || undefined,
name: collectionName || undefined,
id: collectionId,
OR: [{ ownerId: userId }, { members: { some: { userId } } }],
},
include: { members: true },

9
lib/api/manageLinkFiles.ts
View File

@ -16,6 +16,10 @@ const removeFiles = async (linkId: number, collectionId: number) => {
await removeFile({
filePath: `archives/${collectionId}/${linkId}.jpg`,
});
// HTML
await removeFile({
filePath: `archives/${collectionId}/${linkId}.html`,
});
// Preview
await removeFile({
filePath: `archives/preview/${collectionId}/${linkId}.jpeg`,
@ -47,6 +51,11 @@ const moveFiles = async (linkId: number, from: number, to: number) => {
`archives/${to}/${linkId}.jpg`
);
await moveFile(
`archives/${from}/${linkId}.html`,
`archives/${to}/${linkId}.html`
);
await moveFile(
`archives/preview/${from}/${linkId}.jpeg`,
`archives/preview/${to}/${linkId}.jpeg`

61
lib/api/preservationScheme/handleArchivePreview.ts Normal file
View File

@ -0,0 +1,61 @@
import { Collection, Link, User } from "@prisma/client";
import { Page } from "playwright";
import generatePreview from "../generatePreview";
import createFile from "../storage/createFile";
import { prisma } from "../db";
type LinksAndCollectionAndOwner = Link & {
collection: Collection & {
owner: User;
};
};
const handleArchivePreview = async (
link: LinksAndCollectionAndOwner,
page: Page
) => {
const ogImageUrl = await page.evaluate(() => {
const metaTag = document.querySelector('meta[property="og:image"]');
return metaTag ? (metaTag as any).content : null;
});
if (ogImageUrl) {
console.log("Found og:image URL:", ogImageUrl);
// Download the image
const imageResponse = await page.goto(ogImageUrl);
// Check if imageResponse is not null
if (imageResponse && !link.preview?.startsWith("archive")) {
const buffer = await imageResponse.body();
generatePreview(buffer, link.collectionId, link.id);
}
await page.goBack();
} else if (!link.preview?.startsWith("archive")) {
console.log("No og:image found");
await page
.screenshot({ type: "jpeg", quality: 20 })
.then(async (screenshot) => {
if (
Buffer.byteLength(screenshot) >
1024 * 1024 * Number(process.env.PREVIEW_MAX_BUFFER || 0.1)
)
return console.log("Error generating preview: Buffer size exceeded");
await createFile({
data: screenshot,
filePath: `archives/preview/${link.collectionId}/${link.id}.jpeg`,
});
await prisma.link.update({
where: { id: link.id },
data: {
preview: `archives/preview/${link.collectionId}/${link.id}.jpeg`,
},
});
});
}
};
export default handleArchivePreview;

46
lib/api/preservationScheme/handleMonolith.ts Normal file
View File

@ -0,0 +1,46 @@
import { execSync } from "child_process";
import createFile from "../storage/createFile";
import { prisma } from "../db";
import { Link } from "@prisma/client";
const handleMonolith = async (link: Link, content: string) => {
if (!link.url) return;
try {
let html = execSync(
`monolith - -I -b ${link.url} ${
process.env.MONOLITH_CUSTOM_OPTIONS || "-j -F -s"
} -o -`,
{
timeout: 120000,
maxBuffer: 1024 * 1024 * Number(process.env.MONOLITH_MAX_BUFFER || 5),
input: content,
}
);
if (!html?.length)
return console.error("Error archiving as Monolith: Empty buffer");
if (
Buffer.byteLength(html) >
1024 * 1024 * Number(process.env.MONOLITH_MAX_BUFFER || 6)
)
return console.error("Error archiving as Monolith: Buffer size exceeded");
await createFile({
data: html,
filePath: `archives/${link.collectionId}/${link.id}.html`,
}).then(async () => {
await prisma.link.update({
where: { id: link.id },
data: {
monolith: `archives/${link.collectionId}/${link.id}.html`,
},
});
});
} catch (err) {
console.log("Error running MONOLITH:", err);
}
};
export default handleMonolith;

51
lib/api/preservationScheme/handleReadablility.ts Normal file
View File

@ -0,0 +1,51 @@
import { Readability } from "@mozilla/readability";
import { JSDOM } from "jsdom";
import DOMPurify from "dompurify";
import { prisma } from "../db";
import createFile from "../storage/createFile";
import { Link } from "@prisma/client";
const handleReadablility = async (content: string, link: Link) => {
const window = new JSDOM("").window;
const purify = DOMPurify(window);
const cleanedUpContent = purify.sanitize(content);
const dom = new JSDOM(cleanedUpContent, { url: link.url || "" });
const article = new Readability(dom.window.document).parse();
const articleText = article?.textContent
.replace(/ +(?= )/g, "") // strip out multiple spaces
.replace(/(\r\n|\n|\r)/gm, " "); // strip out line breaks
if (articleText && articleText !== "") {
const collectionId = (
await prisma.link.findUnique({
where: { id: link.id },
select: { collectionId: true },
})
)?.collectionId;
const data = JSON.stringify(article);
if (
Buffer.byteLength(data, "utf8") >
1024 * 1024 * Number(process.env.READABILITY_MAX_BUFFER || 1)
)
return console.error(
"Error archiving as Readability: Buffer size exceeded"
);
await createFile({
data,
filePath: `archives/${collectionId}/${link.id}_readability.json`,
});
await prisma.link.update({
where: { id: link.id },
data: {
readable: `archives/${collectionId}/${link.id}_readability.json`,
textContent: articleText,
},
});
}
};
export default handleReadablility;

122
lib/api/preservationScheme/handleScreenshotAndPdf.ts Normal file
View File

@ -0,0 +1,122 @@
import { Collection, Link, User } from "@prisma/client";
import { Page } from "playwright";
import createFile from "../storage/createFile";
import { prisma } from "../db";
type LinksAndCollectionAndOwner = Link & {
collection: Collection & {
owner: User;
};
};
const handleScreenshotAndPdf = async (
link: LinksAndCollectionAndOwner,
page: Page,
user: User
) => {
await page.evaluate(autoScroll, Number(process.env.AUTOSCROLL_TIMEOUT) || 30);
// Check if the user hasn't deleted the link by the time we're done scrolling
const linkExists = await prisma.link.findUnique({
where: { id: link.id },
});
if (linkExists) {
const processingPromises = [];
if (user.archiveAsScreenshot && !link.image?.startsWith("archive")) {
processingPromises.push(
page
.screenshot({ fullPage: true, type: "jpeg" })
.then(async (screenshot) => {
if (
Buffer.byteLength(screenshot) >
1024 * 1024 * Number(process.env.SCREENSHOT_MAX_BUFFER || 2)
)
return console.log(
"Error archiving as Screenshot: Buffer size exceeded"
);
await createFile({
data: screenshot,
filePath: `archives/${linkExists.collectionId}/${link.id}.jpeg`,
});
await prisma.link.update({
where: { id: link.id },
data: {
image: user.archiveAsScreenshot
? `archives/${linkExists.collectionId}/${link.id}.jpeg`
: undefined,
},
});
})
);
}
const margins = {
top: process.env.PDF_MARGIN_TOP || "15px",
bottom: process.env.PDF_MARGIN_BOTTOM || "15px",
};
if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {
processingPromises.push(
page
.pdf({
width: "1366px",
height: "1931px",
printBackground: true,
margin: margins,
})
.then(async (pdf) => {
if (
Buffer.byteLength(pdf) >
1024 * 1024 * Number(process.env.PDF_MAX_BUFFER || 2)
)
return console.log(
"Error archiving as PDF: Buffer size exceeded"
);
await createFile({
data: pdf,
filePath: `archives/${linkExists.collectionId}/${link.id}.pdf`,
});
await prisma.link.update({
where: { id: link.id },
data: {
pdf: user.archiveAsPDF
? `archives/${linkExists.collectionId}/${link.id}.pdf`
: undefined,
},
});
})
);
}
await Promise.allSettled(processingPromises);
}
};
const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => {
const timeoutPromise = new Promise<void>((resolve) => {
setTimeout(() => {
resolve();
}, AUTOSCROLL_TIMEOUT * 1000);
});
const scrollingPromise = new Promise<void>((resolve) => {
let totalHeight = 0;
let distance = 100;
let scrollDown = setInterval(() => {
let scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if (totalHeight >= scrollHeight) {
clearInterval(scrollDown);
window.scroll(0, 0);
resolve();
}
}, 100);
});
await Promise.race([scrollingPromise, timeoutPromise]);
};
export default handleScreenshotAndPdf;

38
lib/api/preservationScheme/imageHandler.ts Normal file
View File

@ -0,0 +1,38 @@
import { Link } from "@prisma/client";
import { prisma } from "../db";
import createFile from "../storage/createFile";
import generatePreview from "../generatePreview";
const imageHandler = async ({ url, id }: Link, extension: string) => {
const image = await fetch(url as string).then((res) => res.blob());
const buffer = Buffer.from(await image.arrayBuffer());
if (
Buffer.byteLength(buffer) >
1024 * 1024 * Number(process.env.SCREENSHOT_MAX_BUFFER || 2)
)
return console.log("Error archiving as Screenshot: Buffer size exceeded");
const linkExists = await prisma.link.findUnique({
where: { id },
});
if (linkExists) {
await generatePreview(buffer, linkExists.collectionId, id);
await createFile({
data: buffer,
filePath: `archives/${linkExists.collectionId}/${id}.${extension}`,
});
await prisma.link.update({
where: { id },
data: {
image: `archives/${linkExists.collectionId}/${id}.${extension}`,
},
});
}
};
export default imageHandler;

35
lib/api/preservationScheme/pdfHandler.ts Normal file
View File

@ -0,0 +1,35 @@
import { Link } from "@prisma/client";
import { prisma } from "../db";
import createFile from "../storage/createFile";
const pdfHandler = async ({ url, id }: Link) => {
const pdf = await fetch(url as string).then((res) => res.blob());
const buffer = Buffer.from(await pdf.arrayBuffer());
if (
Buffer.byteLength(buffer) >
1024 * 1024 * Number(process.env.PDF_MAX_BUFFER || 2)
)
return console.log("Error archiving as PDF: Buffer size exceeded");
const linkExists = await prisma.link.findUnique({
where: { id },
});
if (linkExists) {
await createFile({
data: buffer,
filePath: `archives/${linkExists.collectionId}/${id}.pdf`,
});
await prisma.link.update({
where: { id },
data: {
pdf: `archives/${linkExists.collectionId}/${id}.pdf`,
},
});
}
};
export default pdfHandler;

0
lib/api/sendToWayback.ts → lib/api/preservationScheme/sendToWayback.ts
View File

89
lib/api/setLinkCollection.ts Normal file
View File

@ -0,0 +1,89 @@
import { LinkIncludingShortenedCollectionAndTags } from "@/types/global";
import { prisma } from "./db";
import getPermission from "./getPermission";
import { UsersAndCollections } from "@prisma/client";
const setLinkCollection = async (
link: LinkIncludingShortenedCollectionAndTags,
userId: number
) => {
if (link?.collection?.id && typeof link?.collection?.id === "number") {
const existingCollection = await prisma.collection.findUnique({
where: {
id: link.collection.id,
},
});
if (!existingCollection) return null;
const collectionIsAccessible = await getPermission({
userId,
collectionId: existingCollection.id,
});
const memberHasAccess = collectionIsAccessible?.members.some(
(e: UsersAndCollections) => e.userId === userId && e.canCreate
);
if (!(collectionIsAccessible?.ownerId === userId || memberHasAccess))
return null;
return existingCollection;
} else if (link?.collection?.name) {
if (link.collection.name === "Unorganized") {
const firstTopLevelUnorganizedCollection =
await prisma.collection.findFirst({
where: {
name: "Unorganized",
ownerId: userId,
parentId: null,
},
});
if (firstTopLevelUnorganizedCollection)
return firstTopLevelUnorganizedCollection;
}
const newCollection = await prisma.collection.create({
data: {
name: link.collection.name.trim(),
ownerId: userId,
},
});
await prisma.user.update({
where: {
id: userId,
},
data: {
collectionOrder: {
push: newCollection.id,
},
},
});
return newCollection;
} else {
const firstTopLevelUnorganizedCollection =
await prisma.collection.findFirst({
where: {
name: "Unorganized",
ownerId: userId,
parentId: null,
},
});
if (firstTopLevelUnorganizedCollection)
return firstTopLevelUnorganizedCollection;
else
return await prisma.collection.create({
data: {
name: "Unorganized",
ownerId: userId,
parentId: null,
},
});
}
};
export default setLinkCollection;

5
lib/api/storage/readFile.ts
View File

@ -10,6 +10,7 @@ import util from "util";
type ReturnContentTypes =
| "text/plain"
| "text/html"
| "image/jpeg"
| "image/png"
| "application/pdf"
@ -61,6 +62,8 @@ export default async function readFile(filePath: string) {
contentType = "image/png";
} else if (filePath.endsWith("_readability.json")) {
contentType = "application/json";
} else if (filePath.endsWith(".html")) {
contentType = "text/html";
} else {
// if (filePath.endsWith(".jpg"))
contentType = "image/jpeg";
@ -88,6 +91,8 @@ export default async function readFile(filePath: string) {
contentType = "image/png";
} else if (filePath.endsWith("_readability.json")) {
contentType = "application/json";
} else if (filePath.endsWith(".html")) {
contentType = "text/html";
} else {
// if (filePath.endsWith(".jpg"))
contentType = "image/jpeg";

6
lib/client/generateLinkHref.ts
View File

@ -8,6 +8,7 @@ import {
pdfAvailable,
readabilityAvailable,
screenshotAvailable,
monolithAvailable,
} from "../shared/getArchiveValidity";
export const generateLinkHref = (
@ -33,12 +34,15 @@ export const generateLinkHref = (
account.linksRouteTo === LinksRouteTo.SCREENSHOT ||
link.type === "image"
) {
console.log(link);
if (!screenshotAvailable(link)) return link.url || "";
return `/preserved/${link?.id}?format=${
link?.image?.endsWith("png") ? ArchivedFormat.png : ArchivedFormat.jpeg
}`;
} else if (account.linksRouteTo === LinksRouteTo.MONOLITH) {
if (!monolithAvailable(link)) return link.url || "";
return `/preserved/${link?.id}?format=${ArchivedFormat.monolith}`;
} else {
return link.url || "";
}

12
lib/shared/getTitle.ts → lib/shared/fetchTitleAndHeaders.ts
View File

@ -2,7 +2,7 @@ import fetch from "node-fetch";
import https from "https";
import { SocksProxyAgent } from "socks-proxy-agent";
export default async function getTitle(url: string) {
export default async function fetchTitleAndHeaders(url: string) {
try {
const httpsAgent = new https.Agent({
rejectUnauthorized:
@ -41,12 +41,16 @@ export default async function getTitle(url: string) {
// regular expression to find the <title> tag
let match = text.match(/<title.*>([^<]*)<\/title>/);
if (match) return match[1];
else return "";
const title = match[1] || "";
const headers = (response as Response)?.headers || null;
return { title, headers };
} else {
return "";
return { title: "", headers: null };
}
} catch (err) {
console.log(err);
return { title: "", headers: null };
}
}

11
lib/shared/getArchiveValidity.ts
View File

@ -28,6 +28,17 @@ export function readabilityAvailable(
);
}
export function monolithAvailable(
link: LinkIncludingShortenedCollectionAndTags
) {
return (
link &&
link.monolith &&
link.monolith !== "pending" &&
link.monolith !== "unavailable"
);
}
export function previewAvailable(link: any) {
return (
link &&

4
package.json
View File

@ -60,7 +60,7 @@
"next-i18next": "^15.3.0",
"node-fetch": "^2.7.0",
"nodemailer": "^6.9.3",
"playwright": "^1.43.1",
"playwright": "^1.45.0",
"react": "18.2.0",
"react-colorful": "^5.6.1",
"react-dom": "18.2.0",
@ -77,7 +77,7 @@
"zustand": "^4.3.8"
},
"devDependencies": {
"@playwright/test": "^1.43.1",
"@playwright/test": "^1.45.0",
"@types/bcrypt": "^5.0.0",
"@types/dompurify": "^3.0.4",
"@types/jsdom": "^21.1.3",

54
pages/api/v1/archives/[linkId].ts
View File

@ -29,6 +29,7 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) {
else if (format === ArchivedFormat.jpeg) suffix = ".jpeg";
else if (format === ArchivedFormat.pdf) suffix = ".pdf";
else if (format === ArchivedFormat.readability) suffix = "_readability.json";
else if (format === ArchivedFormat.monolith) suffix = ".html";
//@ts-ignore
if (!linkId || !suffix)
@ -84,21 +85,42 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) {
linkId,
});
const memberHasAccess = collectionPermissions?.members.some(
if (!collectionPermissions)
return { response: "Collection is not accessible.", status: 400 };
const memberHasAccess = collectionPermissions.members.some(
(e: UsersAndCollections) => e.userId === user.id && e.canCreate
);
if (!(collectionPermissions?.ownerId === user.id || memberHasAccess))
return { response: "Collection is not accessible.", status: 401 };
if (!(collectionPermissions.ownerId === user.id || memberHasAccess))
return { response: "Collection is not accessible.", status: 400 };
// await uploadHandler(linkId, )
const MAX_UPLOAD_SIZE = Number(process.env.NEXT_PUBLIC_MAX_FILE_SIZE);
const MAX_LINKS_PER_USER = Number(process.env.MAX_LINKS_PER_USER || 30000);
const numberOfLinksTheUserHas = await prisma.link.count({
where: {
collection: {
ownerId: user.id,
},
},
});
if (numberOfLinksTheUserHas > MAX_LINKS_PER_USER)
return {
response: `Each collection owner can only have a maximum of ${MAX_LINKS_PER_USER} Links.`,
status: 400,
};
const NEXT_PUBLIC_MAX_FILE_BUFFER = Number(
process.env.NEXT_PUBLIC_MAX_FILE_BUFFER || 10
);
const form = formidable({
maxFields: 1,
maxFiles: 1,
maxFileSize: MAX_UPLOAD_SIZE || 30 * 1048576,
maxFileSize: NEXT_PUBLIC_MAX_FILE_BUFFER * 1024 * 1024,
});
form.parse(req, async (err, fields, files) => {
@ -116,18 +138,26 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) {
!allowedMIMETypes.includes(files.file[0].mimetype || "")
) {
// Handle parsing error
return res.status(500).json({
response: `Sorry, we couldn't process your file. Please ensure it's a PDF, PNG, or JPG format and doesn't exceed ${MAX_UPLOAD_SIZE}MB.`,
return res.status(400).json({
response: `Sorry, we couldn't process your file. Please ensure it's a PDF, PNG, or JPG format and doesn't exceed ${NEXT_PUBLIC_MAX_FILE_BUFFER}MB.`,
});
} else {
const fileBuffer = fs.readFileSync(files.file[0].filepath);
if (
Buffer.byteLength(fileBuffer) >
1024 * 1024 * Number(NEXT_PUBLIC_MAX_FILE_BUFFER)
)
return res.status(400).json({
response: `Sorry, we couldn't process your file. Please ensure it's a PDF, PNG, or JPG format and doesn't exceed ${NEXT_PUBLIC_MAX_FILE_BUFFER}MB.`,
});
const linkStillExists = await prisma.link.findUnique({
where: { id: linkId },
});
if (linkStillExists && files.file[0].mimetype?.includes("image")) {
const collectionId = collectionPermissions?.id as number;
const collectionId = collectionPermissions.id as number;
createFolder({
filePath: `archives/preview/${collectionId}`,
});
@ -137,9 +167,7 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) {
if (linkStillExists) {
await createFile({
filePath: `archives/${collectionPermissions?.id}/${
linkId + suffix
}`,
filePath: `archives/${collectionPermissions.id}/${linkId + suffix}`,
data: fileBuffer,
});
@ -150,10 +178,10 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) {
? "unavailable"
: undefined,
image: files.file[0].mimetype?.includes("image")
? `archives/${collectionPermissions?.id}/${linkId + suffix}`
? `archives/${collectionPermissions.id}/${linkId + suffix}`
: null,
pdf: files.file[0].mimetype?.includes("pdf")
? `archives/${collectionPermissions?.id}/${linkId + suffix}`
? `archives/${collectionPermissions.id}/${linkId + suffix}`
: null,
lastPreserved: new Date().toISOString(),
},

1
pages/api/v1/links/[id]/archive/index.ts
View File

@ -76,6 +76,7 @@ const deleteArchivedFiles = async (link: Link & { collection: Collection }) => {
image: null,
pdf: null,
readable: null,
monolith: null,
preview: null,
},
});

2
pages/collections/[id].tsx
View File

@ -59,6 +59,7 @@ export default function Index() {
username: "",
image: "",
archiveAsScreenshot: undefined as unknown as boolean,
archiveAsMonolith: undefined as unknown as boolean,
archiveAsPDF: undefined as unknown as boolean,
});
@ -76,6 +77,7 @@ export default function Index() {
username: account.username as string,
image: account.image as string,
archiveAsScreenshot: account.archiveAsScreenshot as boolean,
archiveAsMonolith: account.archiveAsScreenshot as boolean,
archiveAsPDF: account.archiveAsPDF as boolean,
});
}

9
pages/preserved/[id].tsx
View File

@ -6,6 +6,7 @@ import {
LinkIncludingShortenedCollectionAndTags,
} from "@/types/global";
import ReadableView from "@/components/ReadableView";
import getServerSideProps from "@/lib/client/getServerSideProps";
export default function Index() {
const { links, getLink } = useLinkStore();
@ -36,6 +37,12 @@ export default function Index() {
{link && Number(router.query.format) === ArchivedFormat.readability && (
<ReadableView link={link} />
)}
{link && Number(router.query.format) === ArchivedFormat.monolith && (
<iframe
src={`/api/v1/archives/${link.id}?format=${ArchivedFormat.monolith}`}
className="w-full h-screen border-none"
></iframe>
)}
{link && Number(router.query.format) === ArchivedFormat.pdf && (
<iframe
src={`/api/v1/archives/${link.id}?format=${ArchivedFormat.pdf}`}
@ -59,3 +66,5 @@ export default function Index() {
</div>
);
}
export { getServerSideProps };

1
pages/public/collections/[id].tsx
View File

@ -42,6 +42,7 @@ export default function PublicCollections() {
username: "",
image: "",
archiveAsScreenshot: undefined as unknown as boolean,
archiveAsMonolith: undefined as unknown as boolean,
archiveAsPDF: undefined as unknown as boolean,
});

3
pages/public/preserved/[id].tsx
View File

@ -6,6 +6,7 @@ import {
LinkIncludingShortenedCollectionAndTags,
} from "@/types/global";
import ReadableView from "@/components/ReadableView";
import getServerSideProps from "@/lib/client/getServerSideProps";
export default function Index() {
const { links, getLink } = useLinkStore();
@ -61,3 +62,5 @@ export default function Index() {
</div>
);
}
export { getServerSideProps };

34
pages/settings/preference.tsx
View File

@ -25,14 +25,21 @@ export default function Appearance() {
const [archiveAsPDF, setArchiveAsPDF] = useState<boolean>(
account.archiveAsPDF
);
const [archiveAsMonolith, setArchiveAsMonolith] = useState<boolean>(
account.archiveAsMonolith
);
const [archiveAsWaybackMachine, setArchiveAsWaybackMachine] =
useState<boolean>(account.archiveAsWaybackMachine);
const [linksRouteTo, setLinksRouteTo] = useState(account.linksRouteTo);
useEffect(() => {
setUser({
...account,
archiveAsScreenshot,
archiveAsMonolith,
archiveAsPDF,
archiveAsWaybackMachine,
linksRouteTo,
@ -41,6 +48,7 @@ export default function Appearance() {
}, [
account,
archiveAsScreenshot,
archiveAsMonolith,
archiveAsPDF,
archiveAsWaybackMachine,
linksRouteTo,
@ -54,6 +62,7 @@ export default function Appearance() {
useEffect(() => {
if (!objectIsEmpty(account)) {
setArchiveAsScreenshot(account.archiveAsScreenshot);
setArchiveAsMonolith(account.archiveAsMonolith);
setArchiveAsPDF(account.archiveAsPDF);
setArchiveAsWaybackMachine(account.archiveAsWaybackMachine);
setLinksRouteTo(account.linksRouteTo);
@ -125,6 +134,13 @@ export default function Appearance() {
state={archiveAsScreenshot}
onClick={() => setArchiveAsScreenshot(!archiveAsScreenshot)}
/>
<Checkbox
label={t("webpage")}
state={archiveAsMonolith}
onClick={() => setArchiveAsMonolith(!archiveAsMonolith)}
/>
<Checkbox
label={t("pdf")}
state={archiveAsPDF}
@ -204,6 +220,24 @@ export default function Appearance() {
</span>
</label>
<label
className="label cursor-pointer flex gap-2 justify-start w-fit"
tabIndex={0}
role="button"
>
<input
type="radio"
name="link-preference-radio"
className="radio checked:bg-primary"
value="Monolith"
checked={linksRouteTo === LinksRouteTo.MONOLITH}
onChange={() => setLinksRouteTo(LinksRouteTo.MONOLITH)}
/>
<span className="label-text">
{t("open_webpage_if_available")}
</span>
</label>
<label
className="label cursor-pointer flex gap-2 justify-start w-fit"
tabIndex={0}

8
prisma/migrations/20240309180643_add_singlefile_archive_format/migration.sql Normal file
View File

@ -0,0 +1,8 @@
-- AlterEnum
ALTER TYPE "LinksRouteTo" ADD VALUE 'SINGLEFILE';
-- AlterTable
ALTER TABLE "User" ADD COLUMN "archiveAsSinglefile" BOOLEAN NOT NULL DEFAULT false;
-- AlterTable
ALTER TABLE "Link" ADD COLUMN "singlefile" text;

21
prisma/migrations/20240628013747_rename_fields/migration.sql Normal file
View File

@ -0,0 +1,21 @@
/*
Warnings:
- The values [SINGLEFILE] on the enum `LinksRouteTo` will be removed. If these variants are still used in the database, this will fail.
- You are about to drop the column `archiveAsSinglefile` on the `User` table. All the data in the column will be lost.
*/
-- AlterEnum
BEGIN;
CREATE TYPE "LinksRouteTo_new" AS ENUM ('ORIGINAL', 'PDF', 'READABLE', 'MONOLITH', 'SCREENSHOT');
ALTER TABLE "User" ALTER COLUMN "linksRouteTo" DROP DEFAULT;
ALTER TABLE "User" ALTER COLUMN "linksRouteTo" TYPE "LinksRouteTo_new" USING ("linksRouteTo"::text::"LinksRouteTo_new");
ALTER TYPE "LinksRouteTo" RENAME TO "LinksRouteTo_old";
ALTER TYPE "LinksRouteTo_new" RENAME TO "LinksRouteTo";
DROP TYPE "LinksRouteTo_old";
ALTER TABLE "User" ALTER COLUMN "linksRouteTo" SET DEFAULT 'ORIGINAL';
COMMIT;
-- AlterTable
ALTER TABLE "User" DROP COLUMN "archiveAsSinglefile",
ADD COLUMN "archiveAsMonolith" BOOLEAN NOT NULL DEFAULT true;

9
prisma/migrations/20240628014129_rename_field/migration.sql Normal file
View File

@ -0,0 +1,9 @@
/*
Warnings:
- You are about to drop the column `singlefile` on the `Link` table. All the data in the column will be lost.
*/
-- AlterTable
ALTER TABLE "Link" DROP COLUMN "singlefile",
ADD COLUMN "monolith" TEXT;

3
prisma/schema.prisma
View File

@ -47,6 +47,7 @@ model User {
linksRouteTo LinksRouteTo @default(ORIGINAL)
preventDuplicateLinks Boolean @default(false)
archiveAsScreenshot Boolean @default(true)
archiveAsMonolith Boolean @default(true)
archiveAsPDF Boolean @default(true)
archiveAsWaybackMachine Boolean @default(false)
isPrivate Boolean @default(false)
@ -58,6 +59,7 @@ enum LinksRouteTo {
ORIGINAL
PDF
READABLE
MONOLITH
SCREENSHOT
}
@ -137,6 +139,7 @@ model Link {
image String?
pdf String?
readable String?
monolith String?
lastPreserved DateTime?
importDate DateTime?
createdAt DateTime @default(now())

7
public/locales/en/common.json
View File

@ -166,6 +166,7 @@
"open_pdf_if_available": "Open PDF, if available",
"open_readable_if_available": "Open Readable, if available",
"open_screenshot_if_available": "Open Screenshot, if available",
"open_webpage_if_available": "Open Webpage copy, if available",
"tag_renamed": "Tag renamed!",
"tag_deleted": "Tag deleted!",
"rename_tag": "Rename Tag",
@ -221,8 +222,9 @@
"github": "GitHub",
"twitter": "Twitter",
"mastodon": "Mastodon",
"link_preservation_in_queue": "LThe Link preservation is currently in the queue",
"link_preservation_in_queue": "The Link preservation is currently in the queue",
"check_back_later": "Please check back later to see the result",
"there_are_more_formats": "There are more preserved formats in the queue",
"settings": "Settings",
"switch_to": "Switch to {{theme}}",
"logout": "Logout",
@ -360,5 +362,6 @@
"show_link_details": "Show Link Details",
"hide_link_details": "Hide Link Details",
"link_pinned": "Link Pinned!",
"link_unpinned": "Link Unpinned!"
"link_unpinned": "Link Unpinned!",
"webpage": "Webpage"
}

14
scripts/worker.ts
View File

@ -38,6 +38,13 @@ async function processBatch() {
{
readable: "pending",
},
///////////////////////
{
monolith: null,
},
{
monolith: "pending",
},
],
},
take: archiveTakeCount,
@ -75,6 +82,13 @@ async function processBatch() {
{
readable: "pending",
},
///////////////////////
{
monolith: null,
},
{
monolith: "pending",
},
],
},
take: archiveTakeCount,

423
types/enviornment.d.ts vendored
View File

@ -1,423 +0,0 @@
declare global {
namespace NodeJS {
interface ProcessEnv {
NEXTAUTH_SECRET: string;
DATABASE_URL: string;
NEXTAUTH_URL: string;
NEXT_PUBLIC_DISABLE_REGISTRATION?: string;
PAGINATION_TAKE_COUNT?: string;
STORAGE_FOLDER?: string;
AUTOSCROLL_TIMEOUT?: string;
RE_ARCHIVE_LIMIT?: string;
NEXT_PUBLIC_MAX_FILE_SIZE?: string;
MAX_LINKS_PER_USER?: string;
ARCHIVE_TAKE_COUNT?: string;
IGNORE_UNAUTHORIZED_CA?: string;
IGNORE_URL_SIZE_LIMIT?: string;
ADMINISTRATOR?: string;
SPACES_KEY?: string;
SPACES_SECRET?: string;
SPACES_ENDPOINT?: string;
SPACES_BUCKET_NAME?: string;
SPACES_REGION?: string;
SPACES_FORCE_PATH_STYLE?: string;
NEXT_PUBLIC_CREDENTIALS_ENABLED?: string;
DISABLE_NEW_SSO_USERS?: string;
NEXT_PUBLIC_EMAIL_PROVIDER?: string;
EMAIL_FROM?: string;
EMAIL_SERVER?: string;
BASE_URL?: string; // Used for email and stripe
NEXT_PUBLIC_STRIPE?: string;
STRIPE_SECRET_KEY?: string;
MONTHLY_PRICE_ID?: string;
YEARLY_PRICE_ID?: string;
NEXT_PUBLIC_STRIPE_BILLING_PORTAL_URL?: string;
NEXT_PUBLIC_TRIAL_PERIOD_DAYS?: string;
// Proxy settings
PROXY?: string;
PROXY_USERNAME?: string;
PROXY_PASSWORD?: string;
PROXY_BYPASS?: string;
// PDF archive settings
PDF_MARGIN_TOP?: string;
PDF_MARGIN_BOTTOM?: string;
//
// SSO Providers
//
// 42 School
NEXT_PUBLIC_FORTYTWO_ENABLED?: string;
FORTYTWO_CUSTOM_NAME?: string;
FORTYTWO_CLIENT_ID?: string;
FORTYTWO_CLIENT_SECRET?: string;
// Apple
NEXT_PUBLIC_APPLE_ENABLED?: string;
APPLE_CUSTOM_NAME?: string;
APPLE_ID?: string;
APPLE_SECRET?: string;
// Atlassian
NEXT_PUBLIC_ATLASSIAN_ENABLED?: string;
ATLASSIAN_CUSTOM_NAME?: string;
ATLASSIAN_CLIENT_ID?: string;
ATLASSIAN_CLIENT_SECRET?: string;
ATLASSIAN_SCOPE?: string;
// Auth0
NEXT_PUBLIC_AUTH0_ENABLED?: string;
AUTH0_CUSTOM_NAME?: string;
AUTH0_ISSUER?: string;
AUTH0_CLIENT_SECRET?: string;
AUTH0_CLIENT_ID?: string;
// Authelia
NEXT_PUBLIC_AUTHELIA_ENABLED?: string;
AUTHELIA_CUSTOM_NAME?: string;
AUTHELIA_CLIENT_ID?: string;
AUTHELIA_CLIENT_SECRET?: string;
AUTHELIA_WELLKNOWN_URL?: string;
// Authentik
NEXT_PUBLIC_AUTHENTIK_ENABLED?: string;
AUTHENTIK_CUSTOM_NAME?: string;
AUTHENTIK_ISSUER?: string;
AUTHENTIK_CLIENT_ID?: string;
AUTHENTIK_CLIENT_SECRET?: string;
// TODO: Azure AD B2C
// TODO: Azure AD
// Battle.net
NEXT_PUBLIC_BATTLENET_ENABLED?: string;
BATTLENET_CUSTOM_NAME?: string;
BATTLENET_CLIENT_ID?: string;
BATTLENET_CLIENT_SECRET?: string;
BATLLENET_ISSUER?: string;
// Box
NEXT_PUBLIC_BOX_ENABLED?: string;
BOX_CUSTOM_NAME?: string;
BOX_CLIENT_ID?: string;
BOX_CLIENT_SECRET?: string;
// TODO: BoxyHQ SAML
// Bungie
NEXT_PUBLIC_BUNGIE_ENABLED?: string;
BUNGIE_CUSTOM_NAME?: string;
BUNGIE_CLIENT_ID?: string;
BUNGIE_CLIENT_SECRET?: string;
BUNGIE_API_KEY?: string;
// Cognito
NEXT_PUBLIC_COGNITO_ENABLED?: string;
COGNITO_CUSTOM_NAME?: string;
COGNITO_CLIENT_ID?: string;
COGNITO_CLIENT_SECRET?: string;
COGNITO_ISSUER?: string;
// Coinbase
NEXT_PUBLIC_COINBASE_ENABLED?: string;
COINBASE_CUSTOM_NAME?: string;
COINBASE_CLIENT_ID?: string;
COINBASE_CLIENT_SECRET?: string;
// Discord
NEXT_PUBLIC_DISCORD_ENABLED?: string;
DISCORD_CUSTOM_NAME?: string;
DISCORD_CLIENT_ID?: string;
DISCORD_CLIENT_SECRET?: string;
// Dropbox
NEXT_PUBLIC_DROPBOX_ENABLED?: string;
DROPBOX_CUSTOM_NAME?: string;
DROPBOX_CLIENT_ID?: string;
DROPBOX_CLIENT_SECRET?: string;
// DuendeIndentityServer6
NEXT_PUBLIC_DUENDE_IDS6_ENABLED?: string;
DUENDE_IDS6_CUSTOM_NAME?: string;
DUENDE_IDS6_CLIENT_ID?: string;
DUENDE_IDS6_CLIENT_SECRET?: string;
DUENDE_IDS6_ISSUER?: string;
// EVE Online
NEXT_PUBLIC_EVEONLINE_ENABLED?: string;
EVEONLINE_CUSTOM_NAME?: string;
EVEONLINE_CLIENT_ID?: string;
EVEONLINE_CLIENT_SECRET?: string;
// Facebook
NEXT_PUBLIC_FACEBOOK_ENABLED?: string;
FACEBOOK_CUSTOM_NAME?: string;
FACEBOOK_CLIENT_ID?: string;
FACEBOOK_CLIENT_SECRET?: string;
// FACEIT
NEXT_PUBLIC_FACEIT_ENABLED?: string;
FACEIT_CUSTOM_NAME?: string;
FACEIT_CLIENT_ID?: string;
FACEIT_CLIENT_SECRET?: string;
// Foursquare
NEXT_PUBLIC_FOURSQUARE_ENABLED?: string;
FOURSQUARE_CUSTOM_NAME?: string;
FOURSQUARE_CLIENT_ID?: string;
FOURSQUARE_CLIENT_SECRET?: string;
FOURSQUARE_APIVERSION?: string;
// Freshbooks
NEXT_PUBLIC_FRESHBOOKS_ENABLED?: string;
FRESHBOOKS_CUSTOM_NAME?: string;
FRESHBOOKS_CLIENT_ID?: string;
FRESHBOOKS_CLIENT_SECRET?: string;
// FusionAuth
NEXT_PUBLIC_FUSIONAUTH_ENABLED?: string;
FUSIONAUTH_CUSTOM_NAME?: string;
FUSIONAUTH_CLIENT_ID?: string;
FUSIONAUTH_CLIENT_SECRET?: string;
FUSIONAUTH_ISSUER?: string;
FUSIONAUTH_TENANT_ID?: string;
// GitHub
NEXT_PUBLIC_GITHUB_ENABLED?: string;
GITHUB_CUSTOM_NAME?: string;
GITHUB_CLIENT_ID?: string;
GITHUB_CLIENT_SECRET?: string;
// GitLab
NEXT_PUBLIC_GITLAB_ENABLED?: string;
GITLAB_CUSTOM_NAME?: string;
GITLAB_CLIENT_ID?: string;
GITLAB_CLIENT_SECRET?: string;
// Google
NEXT_PUBLIC_GOOGLE_ENABLED?: string;
GOOGLE_CUSTOM_NAME?: string;
GOOGLE_CLIENT_ID?: string;
GOOGLE_CLIENT_SECRET?: string;
// HubSpot
NEXT_PUBLIC_HUBSPOT_ENABLED?: string;
HUBSPOT_CUSTOM_NAME?: string;
HUBSPOT_CLIENT_ID?: string;
HUBSPOT_CLIENT_SECRET?: string;
// IdentityServer4
NEXT_PUBLIC_IDS4_ENABLED?: string;
IDS4_CUSTOM_NAME?: string;
IDS4_CLIENT_ID?: string;
IDS4_CLIENT_SECRET?: string;
IDS4_ISSUER?: string;
// TODO: Instagram (Doesn't return email)
// Kakao
NEXT_PUBLIC_KAKAO_ENABLED?: string;
KAKAO_CUSTOM_NAME?: string;
KAKAO_CLIENT_ID?: string;
KAKAO_CLIENT_SECRET?: string;
// Keycloak
NEXT_PUBLIC_KEYCLOAK_ENABLED?: string;
KEYCLOAK_CUSTOM_NAME?: string;
KEYCLOAK_ISSUER?: string;
KEYCLOAK_CLIENT_ID?: string;
KEYCLOAK_CLIENT_SECRET?: string;
// LINE
NEXT_PUBLIC_LINE_ENABLED?: string;
LINE_CUSTOM_NAME?: string;
LINE_CLIENT_ID?: string;
LINE_CLIENT_SECRET?: string;
// LinkedIn
NEXT_PUBLIC_LINKEDIN_ENABLED?: string;
LINKEDIN_CUSTOM_NAME?: string;
LINKEDIN_CLIENT_ID?: string;
LINKEDIN_CLIENT_SECRET?: string;
// Mailchimp
NEXT_PUBLIC_MAILCHIMP_ENABLED?: string;
MAILCHIMP_CUSTOM_NAME?: string;
MAILCHIMP_CLIENT_ID?: string;
MAILCHIMP_CLIENT_SECRET?: string;
// Mail.ru
NEXT_PUBLIC_MAILRU_ENABLED?: string;
MAILRU_CUSTOM_NAME?: string;
MAILRU_CLIENT_ID?: string;
MAILRU_CLIENT_SECRET?: string;
// TODO: Medium (Doesn't return email)
// Naver
NEXT_PUBLIC_NAVER_ENABLED?: string;
NAVER_CUSTOM_NAME?: string;
NAVER_CLIENT_ID?: string;
NAVER_CLIENT_SECRET?: string;
// Netlify
NEXT_PUBLIC_NETLIFY_ENABLED?: string;
NETLIFY_CUSTOM_NAME?: string;
NETLIFY_CLIENT_ID?: string;
NETLIFY_CLIENT_SECRET?: string;
// Okta
NEXT_PUBLIC_OKTA_ENABLED?: string;
OKTA_CUSTOM_NAME?: string;
OKTA_CLIENT_ID?: string;
OKTA_CLIENT_SECRET?: string;
OKTA_ISSUER?: string;
// OneLogin
NEXT_PUBLIC_ONELOGIN_ENABLED?: string;
ONELOGIN_CUSTOM_NAME?: string;
ONELOGIN_CLIENT_ID?: string;
ONELOGIN_CLIENT_SECRET?: string;
ONELOGIN_ISSUER?: string;
// Osso
NEXT_PUBLIC_OSSO_ENABLED?: string;
OSSO_CUSTOM_NAME?: string;
OSSO_CLIENT_ID?: string;
OSSO_CLIENT_SECRET?: string;
OSSO_ISSUER?: string;
// osu!
NEXT_PUBLIC_OSU_ENABLED?: string;
OSU_CUSTOM_NAME?: string;
OSU_CLIENT_ID?: string;
OSU_CLIENT_SECRET?: string;
// Patreon
NEXT_PUBLIC_PATREON_ENABLED?: string;
PATREON_CUSTOM_NAME?: string;
PATREON_CLIENT_ID?: string;
PATREON_CLIENT_SECRET?: string;
// Pinterest
NEXT_PUBLIC_PINTEREST_ENABLED?: string;
PINTEREST_CUSTOM_NAME?: string;
PINTEREST_CLIENT_ID?: string;
PINTEREST_CLIENT_SECRET?: string;
// Pipedrive
NEXT_PUBLIC_PIPEDRIVE_ENABLED?: string;
PIPEDRIVE_CUSTOM_NAME?: string;
PIPEDRIVE_CLIENT_ID?: string;
PIPEDRIVE_CLIENT_SECRET?: string;
// Reddit
// TODO (1h tokens)
NEXT_PUBLIC_REDDIT_ENABLED?: string;
REDDIT_CUSTOM_NAME?: string;
REDDIT_CLIENT_ID?: string;
REDDIT_CLIENT_SECRET?: string;
// Salesforce
NEXT_PUBLIC_SALESFORCE_ENABLED?: string;
SALESFORCE_CUSTOM_NAME?: string;
SALESFORCE_CLIENT_ID?: string;
SALESFORCE_CLIENT_SECRET?: string;
// Slack
NEXT_PUBLIC_SLACK_ENABLED?: string;
SLACK_CUSTOM_NAME?: string;
SLACK_CLIENT_ID?: string;
SLACK_CLIENT_SECRET?: string;
// Spotify
NEXT_PUBLIC_SPOTIFY_ENABLED?: string;
SPOTIFY_CUSTOM_NAME?: string;
SPOTIFY_CLIENT_ID?: string;
SPOTIFY_CLIENT_SECRET?: string;
// Strava
NEXT_PUBLIC_STRAVA_ENABLED?: string;
STRAVA_CUSTOM_NAME?: string;
STRAVA_CLIENT_ID?: string;
STRAVA_CLIENT_SECRET?: string;
// Todoist
NEXT_PUBLIC_TODOIST_ENABLED?: string;
TODOIST_CUSTOM_NAME?: string;
TODOIST_CLIENT_ID?: string;
TODOIST_CLIENT_SECRET?: string;
// TODO: Trakt (Doesn't return email)
// Twitch
NEXT_PUBLIC_TWITCH_ENABLED?: string;
TWITCH_CUSTOM_NAME?: string;
TWITCH_CLIENT_ID?: string;
TWITCH_CLIENT_SECRET?: string;
// TODO: Twitter (OAuth 1.0)
// United Effects
NEXT_PUBLIC_UNITED_EFFECTS_ENABLED?: string;
UNITED_EFFECTS_CUSTOM_NAME?: string;
UNITED_EFFECTS_CLIENT_ID?: string;
UNITED_EFFECTS_CLIENT_SECRET?: string;
UNITED_EFFECTS_ISSUER?: string;
// VK
NEXT_PUBLIC_VK_ENABLED?: string;
VK_CUSTOM_NAME?: string;
VK_CLIENT_ID?: string;
VK_CLIENT_SECRET?: string;
// Wikimedia
NEXT_PUBLIC_WIKIMEDIA_ENABLED?: string;
WIKIMEDIA_CUSTOM_NAME?: string;
WIKIMEDIA_CLIENT_ID?: string;
WIKIMEDIA_CLIENT_SECRET?: string;
// Wordpress.com
NEXT_PUBLIC_WORDPRESS_ENABLED?: string;
WORDPRESS_CUSTOM_NAME?: string;
WORDPRESS_CLIENT_ID?: string;
WORDPRESS_CLIENT_SECRET?: string;
// TODO: WorkOS (Custom flow)
// Yandex
NEXT_PUBLIC_YANDEX_ENABLED?: string;
YANDEX_CUSTOM_NAME?: string;
YANDEX_CLIENT_ID?: string;
YANDEX_CLIENT_SECRET?: string;
// Zitadel
NEXT_PUBLIC_ZITADEL_ENABLED?: string;
ZITADEL_CUSTOM_NAME?: string;
ZITADEL_CLIENT_ID?: string;
ZITADEL_CLIENT_SECRET?: string;
ZITADEL_ISSUER?: string;
// Zoho
NEXT_PUBLIC_ZOHO_ENABLED?: string;
ZOHO_CUSTOM_NAME?: string;
ZOHO_CLIENT_ID?: string;
ZOHO_CLIENT_SECRET?: string;
// Zoom
NEXT_PUBLIC_ZOOM_ENABLED?: string;
ZOOM_CUSTOM_NAME?: string;
ZOOM_CLIENT_ID?: string;
ZOOM_CLIENT_SECRET?: string;
}
}
}
export {};

2
types/global.ts
View File

@ -137,12 +137,14 @@ export enum ArchivedFormat {
jpeg,
pdf,
readability,
monolith,
}
export enum LinkType {
url,
pdf,
image,
monolith,
}
export enum TokenExpiry {

28
yarn.lock
View File

@ -1293,12 +1293,12 @@
tiny-glob "^0.2.9"
tslib "^2.4.0"
"@playwright/test@^1.43.1":
version "1.43.1"
resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.43.1.tgz#16728a59eb8ce0f60472f98d8886d6cab0fa3e42"
integrity sha512-HgtQzFgNEEo4TE22K/X7sYTYNqEMMTZmFS8kTq6m8hXj+m1D8TgwgIbumHddJa9h4yl4GkKb8/bgAl2+g7eDgA==
"@playwright/test@^1.45.0":
version "1.45.0"
resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.45.0.tgz#790a66165a46466c0d7099dd260881802f5aba7e"
integrity sha512-TVYsfMlGAaxeUllNkywbwek67Ncf8FRGn8ZlRdO291OL3NjG9oMbfVhyP82HQF0CZLMrYsvesqoUekxdWuF9Qw==
dependencies:
playwright "1.43.1"
playwright "1.45.0"
"@prisma/client@^4.16.2":
version "4.16.2"
@ -5005,17 +5005,17 @@ pixelmatch@^4.0.2:
dependencies:
pngjs "^3.0.0"
playwright-core@1.43.1:
version "1.43.1"
resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.43.1.tgz#0eafef9994c69c02a1a3825a4343e56c99c03b02"
integrity sha512-EI36Mto2Vrx6VF7rm708qSnesVQKbxEWvPrfA1IPY6HgczBplDx7ENtx+K2n4kJ41sLLkuGfmb0ZLSSXlDhqPg==
playwright-core@1.45.0:
version "1.45.0"
resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.45.0.tgz#5741a670b7c9060ce06852c0051d84736fb94edc"
integrity sha512-lZmHlFQ0VYSpAs43dRq1/nJ9G/6SiTI7VPqidld9TDefL9tX87bTKExWZZUF5PeRyqtXqd8fQi2qmfIedkwsNQ==
playwright@1.43.1, playwright@^1.43.1:
version "1.43.1"
resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.43.1.tgz#8ad08984ac66c9ef3d0db035be54dd7ec9f1c7d9"
integrity sha512-V7SoH0ai2kNt1Md9E3Gwas5B9m8KR2GVvwZnAI6Pg0m3sh7UvgiYhRrhsziCmqMJNouPckiOhk8T+9bSAK0VIA==
playwright@1.45.0, playwright@^1.45.0:
version "1.45.0"
resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.45.0.tgz#400c709c64438690f13705cb9c88ef93089c5c27"
integrity sha512-4z3ac3plDfYzGB6r0Q3LF8POPR20Z8D0aXcxbJvmfMgSSq1hkcgvFRXJk9rUq5H/MJ0Ktal869hhOdI/zUTeLA==
dependencies:
playwright-core "1.43.1"
playwright-core "1.45.0"
optionalDependencies:
fsevents "2.3.2"