xwx.moe/el.xwx.moe
xwx.moe/ el.xwx.moe
1
0
Disbranĉigi 0
Fontkodo Issues Tirpetoj Actions Packages Projects Eldonoj Wiki Activity

rearchive protection

This commit is contained in:
daniel31x13 2023-10-31 15:44:58 -04:00
parent ccafc997fc
commit 56a281ae3d
14 changed files with 246 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.env.sample
View File

@ -9,6 +9,7 @@ STORAGE_FOLDER=
AUTOSCROLL_TIMEOUT=
NEXT_PUBLIC_DISABLE_REGISTRATION=
IMPORT_SIZE_LIMIT=
RE_ARCHIVE_LIMIT=
# AWS S3 Settings
SPACES_KEY=

4
components/Dropdown.tsx
View File

@ -21,7 +21,6 @@ type Props = {
items: MenuItem[];
points?: { x: number; y: number };
style?: React.CSSProperties;
width?: number; // in rem
};
export default function Dropdown({
@ -29,7 +28,6 @@ export default function Dropdown({
onClickOutside,
className,
items,
width,
}: Props) {
const [pos, setPos] = useState<{ x: number; y: number }>();
const [dropdownHeight, setDropdownHeight] = useState<number>();
@ -60,7 +58,7 @@ export default function Dropdown({
setPos({ x: finalX, y: finalY });
}
}, [points, width, dropdownHeight]);
}, [points, dropdownHeight]);
return (
(!points || pos) && (

2
components/LinkCard.tsx
View File

@ -254,7 +254,7 @@ export default function LinkCard({ link, count, className }: Props) {
: undefined,
permissions === true
? {
name: "Update Archive",
name: "Refresh Formats",
onClick: updateArchive,
}
: undefined,

22
components/Modal/Link/PreservedFormats.tsx
View File

@ -55,7 +55,7 @@ export default function PreservedFormats() {
if (response.ok) {
toast.success(`Link is being archived...`);
getLink(link?.id as number);
} else toast.error(data);
} else toast.error(data.response);
};
const handleDownload = (format: "png" | "pdf") => {
@ -152,11 +152,18 @@ export default function PreservedFormats() {
<div className="flex flex-col-reverse sm:flex-row gap-5 items-center justify-center">
{link?.collection.ownerId === session.data?.user.id ? (
<div
className="w-full text-center bg-sky-600 p-1 rounded-md cursor-pointer select-none mt-3"
className={`w-full text-center bg-sky-700 p-1 rounded-md cursor-pointer select-none hover:bg-sky-600 duration-100 ${
link?.pdfPath &&
link?.screenshotPath &&
link?.pdfPath !== "pending" &&
link?.screenshotPath !== "pending"
? "mt-3"
: ""
}`}
onClick={() => updateArchive()}
>
<p>Update Preserved Formats</p>
<p className="text-xs">(re-fetch)</p>
<p className="text-xs">(Refresh Formats)</p>
</div>
) : undefined}
<Link
@ -165,7 +172,14 @@ export default function PreservedFormats() {
""
)}`}
target="_blank"
className="sm:mt-3 text-gray-500 dark:text-gray-300 duration-100 hover:opacity-60 flex gap-2 w-fit items-center text-sm"
className={`text-gray-500 dark:text-gray-300 duration-100 hover:opacity-60 flex gap-2 w-fit items-center text-sm ${
link?.pdfPath &&
link?.screenshotPath &&
link?.pdfPath !== "pending" &&
link?.screenshotPath !== "pending"
? "sm:mt-3"
: ""
}`}
>
<FontAwesomeIcon
icon={faArrowUpRightFromSquare}

27
components/Modal/Link/index.tsx
View File

@ -30,11 +30,15 @@ export default function LinkModal({
}: Props) {
return (
<div className={className}>
{/* {method === "CREATE" && (
<p className="text-xl text-black dark:text-white text-center">
New Link
</p>
)} */}
{method === "CREATE" ? (
<>
<p className="ml-10 mt-[0.1rem] text-xl mb-3 font-thin">
Create a New Link
</p>
<AddOrEditLink toggleLinkModal={toggleLinkModal} method="CREATE" />
</>
) : undefined}
{activeLink && method === "UPDATE" ? (
<>
<p className="ml-10 mt-[0.1rem] text-xl mb-3 font-thin">Edit Link</p>
@ -46,19 +50,10 @@ export default function LinkModal({
</>
) : undefined}
{method === "CREATE" ? (
{method === "FORMATS" ? (
<>
<p className="ml-10 mt-[0.1rem] text-xl mb-3 font-thin">
Create a New Link
</p>
<AddOrEditLink toggleLinkModal={toggleLinkModal} method="CREATE" />
</>
) : undefined}
{activeLink && method === "FORMATS" ? (
<>
<p className="ml-10 mt-[0.1rem] text-xl mb-3 font-thin">
Manage Preserved Formats
Preserved Formats
</p>
<PreservedFormats />
</>

112
layouts/LinkLayout.tsx
View File

@ -8,6 +8,18 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faBars, faChevronLeft } from "@fortawesome/free-solid-svg-icons";
import Link from "next/link";
import useWindowDimensions from "@/hooks/useWindowDimensions";
import {
faPen,
faBoxesStacked,
faTrashCan,
} from "@fortawesome/free-solid-svg-icons";
import useLinkStore from "@/store/links";
import {
CollectionIncludingMembersAndLinkCount,
LinkIncludingShortenedCollectionAndTags,
} from "@/types/global";
import { useSession } from "next-auth/react";
import useCollectionStore from "@/store/collections";
interface Props {
children: ReactNode;
@ -40,6 +52,28 @@ export default function LinkLayout({ children }: Props) {
setSidebar(!sidebar);
};
const session = useSession();
const userId = session.data?.user.id;
const { setModal } = useModalStore();
const { links, removeLink } = useLinkStore();
const { collections } = useCollectionStore();
const [linkCollection, setLinkCollection] =
useState<CollectionIncludingMembersAndLinkCount>();
const [link, setLink] = useState<LinkIncludingShortenedCollectionAndTags>();
useEffect(() => {
if (links) setLink(links.find((e) => e.id === Number(router.query.id)));
}, [links]);
useEffect(() => {
if (link)
setLinkCollection(collections.find((e) => e.id === link?.collection.id));
}, [link]);
return (
<>
<ModalManagement />
@ -50,21 +84,91 @@ export default function LinkLayout({ children }: Props) {
</div>
<div className="w-full flex flex-col min-h-screen max-w-screen-md mx-auto p-5">
<div className="flex gap-3 mb-5 duration-100">
<div
<div className="flex gap-3 mb-5 duration-100 items-center justify-between">
{/* <div
onClick={toggleSidebar}
className="inline-flex lg:hidden gap-1 items-center select-none cursor-pointer p-2 text-gray-500 dark:text-gray-300 rounded-md duration-100 hover:bg-slate-200 dark:hover:bg-neutral-700"
>
<FontAwesomeIcon icon={faBars} className="w-5 h-5" />
</div>
</div> */}
<div
onClick={() => router.back()}
className="inline-flex gap-1 lg:hover:opacity-60 items-center select-none cursor-pointer p-2 lg:p-0 lg:px-1 lg:my-2 text-gray-500 dark:text-gray-300 rounded-md duration-100 hover:bg-slate-200 dark:hover:bg-neutral-700 lg:hover:bg-transparent lg:dark:hover:bg-transparent"
className="inline-flex gap-1 hover:opacity-60 items-center select-none cursor-pointer p-2 lg:p-0 lg:px-1 lg:my-2 text-gray-500 dark:text-gray-300 rounded-md duration-100"
>
<FontAwesomeIcon icon={faChevronLeft} className="w-4 h-4" />
Back
</div>
<div className="lg:hidden">
<div className="flex gap-5">
{link?.collection.ownerId === userId ||
linkCollection?.members.some(
(e) => e.userId === userId && e.canUpdate
) ? (
<div
title="Edit"
onClick={() => {
link
? setModal({
modal: "LINK",
state: true,
active: link,
method: "UPDATE",
})
: undefined;
}}
className={`hover:opacity-60 duration-100 py-2 px-2 cursor-pointer flex items-center gap-4 w-full rounded-md h-8`}
>
<FontAwesomeIcon
icon={faPen}
className="w-6 h-6 text-gray-500 dark:text-gray-300"
/>
</div>
) : undefined}
<div
onClick={() => {
link
? setModal({
modal: "LINK",
state: true,
active: link,
method: "FORMATS",
})
: undefined;
}}
title="Preserved Formats"
className={`hover:opacity-60 duration-100 py-2 px-2 cursor-pointer flex items-center gap-4 w-full rounded-md h-8`}
>
<FontAwesomeIcon
icon={faBoxesStacked}
className="w-6 h-6 text-gray-500 dark:text-gray-300"
/>
</div>
{link?.collection.ownerId === userId ||
linkCollection?.members.some(
(e) => e.userId === userId && e.canDelete
) ? (
<div
onClick={() => {
if (link?.id) {
removeLink(link.id);
router.back();
}
}}
title="Delete"
className={`hover:opacity-60 duration-100 py-2 px-2 cursor-pointer flex items-center gap-4 w-full rounded-md h-8`}
>
<FontAwesomeIcon
icon={faTrashCan}
className="w-6 h-6 text-gray-500 dark:text-gray-300"
/>
</div>
) : undefined}
</div>
</div>
</div>
{children}

96
lib/api/archive.ts
View File

@ -11,36 +11,15 @@ export default async function archive(
url: string,
userId: number
) {
const user = await prisma.user.findUnique({
where: {
id: userId,
},
});
// const checkExistingLink = await prisma.link.findFirst({
// where: {
// id: linkId,
// OR: [
// {
// screenshotPath: "pending",
// },
// {
// pdfPath: "pending",
// },
// ],
// },
// });
// if (checkExistingLink) return "A request has already been made.";
const user = await prisma.user.findUnique({ where: { id: userId } });
const targetLink = await prisma.link.update({
where: {
id: linkId,
},
where: { id: linkId },
data: {
screenshotPath: user?.archiveAsScreenshot ? "pending" : null,
pdfPath: user?.archiveAsPDF ? "pending" : null,
readabilityPath: "pending",
lastPreserved: new Date().toISOString(),
},
});
@ -56,7 +35,6 @@ export default async function archive(
try {
await page.goto(url, { waitUntil: "domcontentloaded" });
await page.goto(url);
const content = await page.content();
// Readability
@ -64,46 +42,35 @@ export default async function archive(
const window = new JSDOM("").window;
const purify = DOMPurify(window);
const cleanedUpContent = purify.sanitize(content);
const dom = new JSDOM(cleanedUpContent, {
url: url,
});
const dom = new JSDOM(cleanedUpContent, { url: url });
const article = new Readability(dom.window.document).parse();
await prisma.link.update({
where: {
id: linkId,
},
data: {
readabilityPath: `archives/${targetLink.collectionId}/${linkId}_readability.json`,
},
});
await createFile({
data: JSON.stringify(article),
filePath: `archives/${targetLink.collectionId}/${linkId}_readability.json`,
});
// Screenshot/PDF
let faulty = true;
await page
.evaluate(autoScroll, Number(process.env.AUTOSCROLL_TIMEOUT) || 30)
.catch((e) => (faulty = false));
const linkExists = await prisma.link.findUnique({
where: {
id: linkId,
await prisma.link.update({
where: { id: linkId },
data: {
readabilityPath: `archives/${targetLink.collectionId}/${linkId}_readability.json`,
},
});
if (linkExists && faulty) {
if (user.archiveAsScreenshot) {
const screenshot = await page.screenshot({
fullPage: true,
});
// Screenshot/PDF
let faulty = false;
await page
.evaluate(autoScroll, Number(process.env.AUTOSCROLL_TIMEOUT) || 30)
.catch((e) => (faulty = true));
const linkExists = await prisma.link.findUnique({
where: { id: linkId },
});
if (linkExists && !faulty) {
if (user.archiveAsScreenshot) {
const screenshot = await page.screenshot({ fullPage: true });
await createFile({
data: screenshot,
filePath: `archives/${linkExists.collectionId}/${linkId}.png`,
@ -124,10 +91,8 @@ export default async function archive(
});
}
const updateLink = await prisma.link.update({
where: {
id: linkId,
},
await prisma.link.update({
where: { id: linkId },
data: {
screenshotPath: user.archiveAsScreenshot
? `archives/${linkExists.collectionId}/${linkId}.png`
@ -139,21 +104,18 @@ export default async function archive(
});
} else if (faulty) {
await prisma.link.update({
where: {
id: linkId,
},
where: { id: linkId },
data: {
screenshotPath: null,
pdfPath: null,
},
});
}
await browser.close();
} catch (err) {
console.log(err);
throw err;
} finally {
await browser.close();
return err;
}
}
}
@ -161,11 +123,7 @@ export default async function archive(
const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => {
const timeoutPromise = new Promise<void>((_, reject) => {
setTimeout(() => {
reject(
new Error(
`Auto scroll took too long (more than ${AUTOSCROLL_TIMEOUT} seconds).`
)
);
reject(new Error(`Webpage was too long to be archived.`));
}, AUTOSCROLL_TIMEOUT * 1000);
});

1
lib/api/controllers/links/postLink.ts
View File

@ -56,6 +56,7 @@ export default async function postLink(
url: link.url,
name: link.name,
description,
readabilityPath: "pending",
collection: {
connectOrCreate: {
where: {

27
pages/api/v1/links/[id]/archive/index.ts
View File

@ -4,6 +4,8 @@ import { authOptions } from "@/pages/api/v1/auth/[...nextauth]";
import archive from "@/lib/api/archive";
import { prisma } from "@/lib/api/db";
const RE_ARCHIVE_LIMIT = Number(process.env.RE_ARCHIVE_LIMIT) || 5;
export default async function links(req: NextApiRequest, res: NextApiResponse) {
const session = await getServerSession(req, res, authOptions);
@ -33,6 +35,20 @@ export default async function links(req: NextApiRequest, res: NextApiResponse) {
});
if (req.method === "PUT") {
if (
link?.lastPreserved &&
getTimezoneDifferenceInMinutes(new Date(), link?.lastPreserved) <
RE_ARCHIVE_LIMIT
)
return res.status(400).json({
response: `This link is currently being saved or has already been preserved. Please retry in ${
RE_ARCHIVE_LIMIT -
Math.floor(
getTimezoneDifferenceInMinutes(new Date(), link?.lastPreserved)
)
} minutes or create a new one.`,
});
archive(link.id, link.url, session.user.id);
return res.status(200).json({
response: "Link is being archived.",
@ -42,3 +58,14 @@ export default async function links(req: NextApiRequest, res: NextApiResponse) {
// Later?
// else if (req.method === "DELETE") {}
}
const getTimezoneDifferenceInMinutes = (future: Date, past: Date) => {
const date1 = new Date(future);
const date2 = new Date(past);
const diffInMilliseconds = Math.abs(date1.getTime() - date2.getTime());
const diffInMinutes = diffInMilliseconds / (1000 * 60);
return diffInMinutes;
};

2
pages/collections/[id].tsx
View File

@ -220,7 +220,7 @@ export default function Index() {
if (target.id !== "expand-dropdown")
setExpandDropdown(false);
}}
className="absolute top-8 right-0 z-10 w-40"
className="absolute top-8 right-0 z-10 w-44"
/>
) : null}
</div>

79
pages/links/[id].tsx
View File

@ -76,7 +76,11 @@ export default function Index() {
useEffect(() => {
let interval: NodeJS.Timer | undefined;
if (link?.screenshotPath === "pending" || link?.pdfPath === "pending") {
if (
link?.screenshotPath === "pending" ||
link?.pdfPath === "pending" ||
link?.readabilityPath === "pending"
) {
interval = setInterval(() => getLink(link.id as number), 5000);
} else {
if (interval) {
@ -190,6 +194,7 @@ export default function Index() {
<p>•</p>
<Link
href={link?.url || ""}
title={link?.url}
target="_blank"
className="hover:opacity-60 duration-100 break-all"
>
@ -240,46 +245,52 @@ export default function Index() {
</div>
<div className="flex flex-col gap-5 h-full">
{link &&
link?.readabilityPath &&
link?.readabilityPath !== "pending" ? (
{link?.readabilityPath?.startsWith("archives") ? (
<div
className="line-break px-5"
className="line-break px-3"
dangerouslySetInnerHTML={{
__html: DOMPurify.sanitize(linkContent?.content || "") || "",
}}
></div>
) : (
<div className="border border-solid border-sky-100 dark:border-neutral-700 w-full h-full flex flex-col justify-center p-10 rounded-2xl bg-gray-50 dark:bg-neutral-800">
<p className="text-center text-2xl text-black dark:text-white">
There is no reader view for this webpage
</p>
<p className="text-center text-sm text-black dark:text-white">
{link?.collection.ownerId === userId
? "You can update (refetch) the preserved formats by managing them below"
: "The collections owners can refetch the preserved formats"}
</p>
{link?.collection.ownerId === userId ? (
<div
onClick={() =>
link
? setModal({
modal: "LINK",
state: true,
active: link,
method: "FORMATS",
})
: undefined
}
className="mt-4 flex gap-2 w-fit mx-auto relative items-center font-semibold select-none cursor-pointer p-2 px-3 rounded-md dark:hover:bg-sky-600 text-white bg-sky-700 hover:bg-sky-600 duration-100"
>
<FontAwesomeIcon
icon={faBoxesStacked}
className="w-5 h-5 duration-100"
/>
<p>Manage preserved formats</p>
</div>
) : undefined}
{link?.readabilityPath === "pending" ? (
<p className="text-center">
Generating readable format, please wait...
</p>
) : (
<>
<p className="text-center text-2xl text-black dark:text-white">
There is no reader view for this webpage
</p>
<p className="text-center text-sm text-black dark:text-white">
{link?.collection.ownerId === userId
? "You can update (refetch) the preserved formats by managing them below"
: "The collections owners can refetch the preserved formats"}
</p>
{link?.collection.ownerId === userId ? (
<div
onClick={() =>
link
? setModal({
modal: "LINK",
state: true,
active: link,
method: "FORMATS",
})
: undefined
}
className="mt-4 flex gap-2 w-fit mx-auto relative items-center font-semibold select-none cursor-pointer p-2 px-3 rounded-md dark:hover:bg-sky-600 text-white bg-sky-700 hover:bg-sky-600 duration-100"
>
<FontAwesomeIcon
icon={faBoxesStacked}
className="w-5 h-5 duration-100"
/>
<p>Manage preserved formats</p>
</div>
) : undefined}
</>
)}
</div>
)}
</div>

2
prisma/migrations/20231031100017_add_last_preserved_field/migration.sql Normal file
View File

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "Link" ADD COLUMN "lastPreserved" TIMESTAMP(3);

2
prisma/schema.prisma
View File

@ -106,6 +106,8 @@ model Link {
screenshotPath String?
pdfPath String?
readabilityPath String?
lastPreserved DateTime?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt @default(now())

1
types/enviornment.d.ts vendored
View File

@ -9,6 +9,7 @@ declare global {
STORAGE_FOLDER?: string;
AUTOSCROLL_TIMEOUT?: string;
IMPORT_SIZE_LIMIT?: string;
RE_ARCHIVE_LIMIT?: string;
SPACES_KEY?: string;
SPACES_SECRET?: string;