From b88fa446bec102fa54f66d8b40fa08f3f25fd5ed Mon Sep 17 00:00:00 2001 From: daniel31x13 Date: Sat, 25 Nov 2023 03:19:02 -0500 Subject: [PATCH 1/2] webpages can now be a image or pdf --- components/Modal/Link/PreservedFormats.tsx | 11 +++-- lib/api/controllers/links/postLink.ts | 39 ++++++++++++++-- lib/api/imageHandler.ts | 37 ++++++++++++++++ lib/api/pdfHandler.ts | 44 +++++++++++++++++++ lib/api/storage/readFile.ts | 2 +- lib/api/{archive.ts => urlHandler.ts} | 2 +- lib/api/validateUrlSize.ts | 13 ++++++ pages/api/v1/archives/[linkId].ts | 4 +- pages/api/v1/links/[id]/archive/index.ts | 4 +- .../migration.sql | 3 ++ prisma/schema.prisma | 3 +- types/global.ts | 9 +++- 12 files changed, 156 insertions(+), 15 deletions(-) create mode 100644 lib/api/imageHandler.ts create mode 100644 lib/api/pdfHandler.ts rename lib/api/{archive.ts => urlHandler.ts} (99%) create mode 100644 lib/api/validateUrlSize.ts create mode 100644 prisma/migrations/20231125043215_add_link_type_field/migration.sql diff --git a/components/Modal/Link/PreservedFormats.tsx b/components/Modal/Link/PreservedFormats.tsx index 410d3a1..914accf 100644 --- a/components/Modal/Link/PreservedFormats.tsx +++ b/components/Modal/Link/PreservedFormats.tsx @@ -76,8 +76,7 @@ export default function PreservedFormats() { // Create a temporary link and click it to trigger the download const link = document.createElement("a"); link.href = path; - link.download = - format === ArchivedFormat.screenshot ? "Screenshot" : "PDF"; + link.download = format === ArchivedFormat.png ? "Screenshot" : "PDF"; link.click(); } else { console.error("Failed to download file"); @@ -102,7 +101,7 @@ export default function PreservedFormats() {
handleDownload(ArchivedFormat.screenshot)} + onClick={() => handleDownload(ArchivedFormat.png)} className="cursor-pointer hover:opacity-60 duration-100 p-2 rounded-md" > diff --git a/lib/api/controllers/links/postLink.ts b/lib/api/controllers/links/postLink.ts index b93eeef..c6755b4 100644 --- a/lib/api/controllers/links/postLink.ts +++ b/lib/api/controllers/links/postLink.ts @@ -1,17 +1,20 @@ import { prisma } from "@/lib/api/db"; import { LinkIncludingShortenedCollectionAndTags } from "@/types/global"; import getTitle from "@/lib/api/getTitle"; -import archive from "@/lib/api/archive"; +import urlHandler from "@/lib/api/urlHandler"; import { UsersAndCollections } from "@prisma/client"; import getPermission from "@/lib/api/getPermission"; import createFolder from "@/lib/api/storage/createFolder"; +import pdfHandler from "../../pdfHandler"; +import validateUrlSize from "../../validateUrlSize"; +import imageHandler from "../../imageHandler"; export default async function postLink( link: LinkIncludingShortenedCollectionAndTags, userId: number ) { try { - new URL(link.url); + if (link.url) new URL(link.url); } catch (error) { return { response: @@ -45,13 +48,33 @@ export default async function postLink( const description = link.description && link.description !== "" ? link.description - : await getTitle(link.url); + : link.url + ? await getTitle(link.url) + : undefined; + + const validatedUrl = link.url ? await validateUrlSize(link.url) : undefined; + + if (validatedUrl === null) + return { response: "File is too large to be stored.", status: 400 }; + + const contentType = validatedUrl?.get("content-type"); + let linkType = "url"; + let imageExtension = "png"; + + if (!link.url) linkType = link.type; + else if (contentType === "application/pdf") linkType = "pdf"; + else if (contentType?.startsWith("image")) { + linkType = "image"; + if (contentType === "image/jpeg") imageExtension = "jpeg"; + else if (contentType === "image/png") imageExtension = "png"; + } const newLink = await prisma.link.create({ data: { url: link.url, name: link.name, description, + type: linkType, readabilityPath: "pending", collection: { connectOrCreate: { @@ -91,7 +114,15 @@ export default async function postLink( createFolder({ filePath: `archives/${newLink.collectionId}` }); - archive(newLink.id, newLink.url, userId); + newLink.url && linkType === "url" + ? urlHandler(newLink.id, newLink.url, userId) + : undefined; + + linkType === "pdf" ? pdfHandler(newLink.id, newLink.url) : undefined; + + linkType === "image" + ? imageHandler(newLink.id, newLink.url, imageExtension) + : undefined; return { response: newLink, status: 200 }; } diff --git a/lib/api/imageHandler.ts b/lib/api/imageHandler.ts new file mode 100644 index 0000000..b9437dd --- /dev/null +++ b/lib/api/imageHandler.ts @@ -0,0 +1,37 @@ +import { prisma } from "@/lib/api/db"; +import createFile from "@/lib/api/storage/createFile"; +import fs from "fs"; +import path from "path"; + +export default async function imageHandler( + linkId: number, + url: string | null, + extension: string, + file?: string +) { + const pdf = await fetch(url as string).then((res) => res.blob()); + + const buffer = Buffer.from(await pdf.arrayBuffer()); + + const linkExists = await prisma.link.findUnique({ + where: { id: linkId }, + }); + + linkExists + ? await createFile({ + data: buffer, + filePath: `archives/${linkExists.collectionId}/${linkId}.${extension}`, + }) + : undefined; + + await prisma.link.update({ + where: { id: linkId }, + data: { + screenshotPath: linkExists + ? `archives/${linkExists.collectionId}/${linkId}.${extension}` + : null, + pdfPath: null, + readabilityPath: null, + }, + }); +} diff --git a/lib/api/pdfHandler.ts b/lib/api/pdfHandler.ts new file mode 100644 index 0000000..37dbef7 --- /dev/null +++ b/lib/api/pdfHandler.ts @@ -0,0 +1,44 @@ +import { prisma } from "@/lib/api/db"; +import createFile from "@/lib/api/storage/createFile"; +import fs from "fs"; +import path from "path"; + +export default async function pdfHandler( + linkId: number, + url: string | null, + file?: string +) { + const targetLink = await prisma.link.update({ + where: { id: linkId }, + data: { + pdfPath: "pending", + lastPreserved: new Date().toISOString(), + }, + }); + + const pdf = await fetch(url as string).then((res) => res.blob()); + + const buffer = Buffer.from(await pdf.arrayBuffer()); + + const linkExists = await prisma.link.findUnique({ + where: { id: linkId }, + }); + + linkExists + ? await createFile({ + data: buffer, + filePath: `archives/${linkExists.collectionId}/${linkId}.pdf`, + }) + : undefined; + + await prisma.link.update({ + where: { id: linkId }, + data: { + pdfPath: linkExists + ? `archives/${linkExists.collectionId}/${linkId}.pdf` + : null, + readabilityPath: null, + screenshotPath: null, + }, + }); +} diff --git a/lib/api/storage/readFile.ts b/lib/api/storage/readFile.ts index 64ff8d7..350726d 100644 --- a/lib/api/storage/readFile.ts +++ b/lib/api/storage/readFile.ts @@ -97,7 +97,7 @@ export default async function readFile(filePath: string) { return { file: "File not found.", contentType: "text/plain", - status: 400, + status: 404, }; else { const file = fs.readFileSync(creationPath); diff --git a/lib/api/archive.ts b/lib/api/urlHandler.ts similarity index 99% rename from lib/api/archive.ts rename to lib/api/urlHandler.ts index e55d86d..8795d76 100644 --- a/lib/api/archive.ts +++ b/lib/api/urlHandler.ts @@ -6,7 +6,7 @@ import { Readability } from "@mozilla/readability"; import { JSDOM } from "jsdom"; import DOMPurify from "dompurify"; -export default async function archive( +export default async function urlHandler( linkId: number, url: string, userId: number diff --git a/lib/api/validateUrlSize.ts b/lib/api/validateUrlSize.ts new file mode 100644 index 0000000..cde7e33 --- /dev/null +++ b/lib/api/validateUrlSize.ts @@ -0,0 +1,13 @@ +export default async function validateUrlSize(url: string) { + try { + const response = await fetch(url, { method: "HEAD" }); + + const totalSizeMB = + Number(response.headers.get("content-length")) / Math.pow(1024, 2); + if (totalSizeMB > 50) return null; + else return response.headers; + } catch (err) { + console.log(err); + return null; + } +} diff --git a/pages/api/v1/archives/[linkId].ts b/pages/api/v1/archives/[linkId].ts index 74f7738..faade60 100644 --- a/pages/api/v1/archives/[linkId].ts +++ b/pages/api/v1/archives/[linkId].ts @@ -10,7 +10,8 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) { let suffix; - if (format === ArchivedFormat.screenshot) suffix = ".png"; + if (format === ArchivedFormat.png) suffix = ".png"; + else if (format === ArchivedFormat.jpeg) suffix = ".jpeg"; else if (format === ArchivedFormat.pdf) suffix = ".pdf"; else if (format === ArchivedFormat.readability) suffix = "_readability.json"; @@ -43,6 +44,7 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) { const { file, contentType, status } = await readFile( `archives/${collectionIsAccessible.id}/${linkId + suffix}` ); + res.setHeader("Content-Type", contentType).status(status as number); return res.send(file); diff --git a/pages/api/v1/links/[id]/archive/index.ts b/pages/api/v1/links/[id]/archive/index.ts index 65da756..1fce3ff 100644 --- a/pages/api/v1/links/[id]/archive/index.ts +++ b/pages/api/v1/links/[id]/archive/index.ts @@ -1,5 +1,5 @@ import type { NextApiRequest, NextApiResponse } from "next"; -import archive from "@/lib/api/archive"; +import urlHandler from "@/lib/api/urlHandler"; import { prisma } from "@/lib/api/db"; import verifyUser from "@/lib/api/verifyUser"; @@ -41,7 +41,7 @@ export default async function links(req: NextApiRequest, res: NextApiResponse) { } minutes or create a new one.`, }); - archive(link.id, link.url, user.id); + urlHandler(link.id, link.url, user.id); return res.status(200).json({ response: "Link is being archived.", }); diff --git a/prisma/migrations/20231125043215_add_link_type_field/migration.sql b/prisma/migrations/20231125043215_add_link_type_field/migration.sql new file mode 100644 index 0000000..7a90c1c --- /dev/null +++ b/prisma/migrations/20231125043215_add_link_type_field/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "Link" ADD COLUMN "type" TEXT NOT NULL DEFAULT 'url', +ALTER COLUMN "url" DROP NOT NULL; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 70ef764..dd91892 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -103,12 +103,13 @@ model UsersAndCollections { model Link { id Int @id @default(autoincrement()) name String - url String + type String @default("url") description String @default("") pinnedBy User[] collection Collection @relation(fields: [collectionId], references: [id]) collectionId Int tags Tag[] + url String? textContent String? screenshotPath String? pdfPath String? diff --git a/types/global.ts b/types/global.ts index a423d45..c1d1893 100644 --- a/types/global.ts +++ b/types/global.ts @@ -117,7 +117,14 @@ export type DeleteUserBody = { }; export enum ArchivedFormat { - screenshot, + png, + jpeg, pdf, readability, } + +export enum LinkType { + url, + pdf, + image, +} From af80614b3a465ebad91903dbee0efb5a765c0c87 Mon Sep 17 00:00:00 2001 From: daniel31x13 Date: Sat, 25 Nov 2023 03:27:34 -0500 Subject: [PATCH 2/2] bug fixed --- components/LinkCard.tsx | 9 +++++---- components/LinkPreview.tsx | 2 +- components/Modal/Link/AddOrEditLink.tsx | 7 ++++--- components/PublicPage/PublicLinkCard.tsx | 8 ++++---- lib/{client => shared}/isValidUrl.ts | 0 pages/api/v1/links/[id]/archive/index.ts | 9 ++++++++- pages/links/[id].tsx | 2 +- pages/public/links/[id].tsx | 2 +- 8 files changed, 24 insertions(+), 15 deletions(-) rename lib/{client => shared}/isValidUrl.ts (100%) diff --git a/components/LinkCard.tsx b/components/LinkCard.tsx index a3c2d72..748cb2b 100644 --- a/components/LinkCard.tsx +++ b/components/LinkCard.tsx @@ -18,7 +18,7 @@ import useModalStore from "@/store/modals"; import { faCalendarDays } from "@fortawesome/free-regular-svg-icons"; import usePermissions from "@/hooks/usePermissions"; import { toast } from "react-hot-toast"; -import isValidUrl from "@/lib/client/isValidUrl"; +import isValidUrl from "@/lib/shared/isValidUrl"; import Link from "next/link"; import unescapeString from "@/lib/client/unescapeString"; import { useRouter } from "next/router"; @@ -54,7 +54,7 @@ export default function LinkCard({ link, count, className }: Props) { let shortendURL; try { - shortendURL = new URL(link.url).host.toLowerCase(); + shortendURL = new URL(link.url || "").host.toLowerCase(); } catch (error) { console.log(error); } @@ -124,7 +124,8 @@ export default function LinkCard({ link, count, className }: Props) { setExpandDropdown(false); }; - const url = isValidUrl(link.url) ? new URL(link.url) : undefined; + const url = + isValidUrl(link.url || "") && link.url ? new URL(link.url) : undefined; const formattedDate = new Date(link.createdAt as string).toLocaleString( "en-US", @@ -230,7 +231,7 @@ export default function LinkCard({ link, count, className }: Props) { ) : undefined} */} { e.stopPropagation(); diff --git a/components/LinkPreview.tsx b/components/LinkPreview.tsx index c81e46e..c07600e 100644 --- a/components/LinkPreview.tsx +++ b/components/LinkPreview.tsx @@ -2,7 +2,7 @@ import { faFolder, faLink } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import Image from "next/image"; import { faCalendarDays } from "@fortawesome/free-regular-svg-icons"; -import isValidUrl from "@/lib/client/isValidUrl"; +import isValidUrl from "@/lib/shared/isValidUrl"; import A from "next/link"; import unescapeString from "@/lib/client/unescapeString"; import { Link } from "@prisma/client"; diff --git a/components/Modal/Link/AddOrEditLink.tsx b/components/Modal/Link/AddOrEditLink.tsx index 79f6979..207c2b9 100644 --- a/components/Modal/Link/AddOrEditLink.tsx +++ b/components/Modal/Link/AddOrEditLink.tsx @@ -44,6 +44,7 @@ export default function AddOrEditLink({ activeLink || { name: "", url: "", + type: "", description: "", tags: [], screenshotPath: "", @@ -139,10 +140,10 @@ export default function AddOrEditLink({ {method === "UPDATE" ? (
- + {link.url}
@@ -153,7 +154,7 @@ export default function AddOrEditLink({

Address (URL)

setLink({ ...link, url: e.target.value })} placeholder="e.g. http://example.com/" /> diff --git a/components/PublicPage/PublicLinkCard.tsx b/components/PublicPage/PublicLinkCard.tsx index b31190e..eeb03ba 100644 --- a/components/PublicPage/PublicLinkCard.tsx +++ b/components/PublicPage/PublicLinkCard.tsx @@ -2,7 +2,7 @@ import { faChevronRight } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import Image from "next/image"; import { Link as LinkType, Tag } from "@prisma/client"; -import isValidUrl from "@/lib/client/isValidUrl"; +import isValidUrl from "@/lib/shared/isValidUrl"; import unescapeString from "@/lib/client/unescapeString"; import { TagIncludingLinkCount } from "@/types/global"; import Link from "next/link"; @@ -17,7 +17,7 @@ type Props = { }; export default function LinkCard({ link, count }: Props) { - const url = isValidUrl(link.url) ? new URL(link.url) : undefined; + const url = link.url && isValidUrl(link.url) ? new URL(link.url) : undefined; const formattedDate = new Date( link.createdAt as unknown as string @@ -68,10 +68,10 @@ export default function LinkCard({ link, count }: Props) {

{formattedDate}

ยท

{url ? url.host : link.url} diff --git a/lib/client/isValidUrl.ts b/lib/shared/isValidUrl.ts similarity index 100% rename from lib/client/isValidUrl.ts rename to lib/shared/isValidUrl.ts diff --git a/pages/api/v1/links/[id]/archive/index.ts b/pages/api/v1/links/[id]/archive/index.ts index 1fce3ff..6c82aed 100644 --- a/pages/api/v1/links/[id]/archive/index.ts +++ b/pages/api/v1/links/[id]/archive/index.ts @@ -2,6 +2,7 @@ import type { NextApiRequest, NextApiResponse } from "next"; import urlHandler from "@/lib/api/urlHandler"; import { prisma } from "@/lib/api/db"; import verifyUser from "@/lib/api/verifyUser"; +import isValidUrl from "@/lib/shared/isValidUrl"; const RE_ARCHIVE_LIMIT = Number(process.env.RE_ARCHIVE_LIMIT) || 5; @@ -41,7 +42,13 @@ export default async function links(req: NextApiRequest, res: NextApiResponse) { } minutes or create a new one.`, }); - urlHandler(link.id, link.url, user.id); + if (link.url && isValidUrl(link.url)) { + urlHandler(link.id, link.url, user.id); + return res.status(200).json({ + response: "Link is not a webpage to be archived.", + }); + } + return res.status(200).json({ response: "Link is being archived.", }); diff --git a/pages/links/[id].tsx b/pages/links/[id].tsx index a28b45d..5ab2206 100644 --- a/pages/links/[id].tsx +++ b/pages/links/[id].tsx @@ -11,7 +11,7 @@ import Image from "next/image"; import ColorThief, { RGBColor } from "colorthief"; import { useTheme } from "next-themes"; import unescapeString from "@/lib/client/unescapeString"; -import isValidUrl from "@/lib/client/isValidUrl"; +import isValidUrl from "@/lib/shared/isValidUrl"; import DOMPurify from "dompurify"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faBoxesStacked, faFolder } from "@fortawesome/free-solid-svg-icons"; diff --git a/pages/public/links/[id].tsx b/pages/public/links/[id].tsx index 15aec43..5f774a6 100644 --- a/pages/public/links/[id].tsx +++ b/pages/public/links/[id].tsx @@ -11,7 +11,7 @@ import Image from "next/image"; import ColorThief, { RGBColor } from "colorthief"; import { useTheme } from "next-themes"; import unescapeString from "@/lib/client/unescapeString"; -import isValidUrl from "@/lib/client/isValidUrl"; +import isValidUrl from "@/lib/shared/isValidUrl"; import DOMPurify from "dompurify"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faBoxesStacked, faFolder } from "@fortawesome/free-solid-svg-icons";