From d008c441b7a77859f827c8a1f744d1c50e3609c6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 10 Aug 2023 12:16:44 -0400 Subject: [PATCH] Feat/import export (#136) * added import/export functionality --- .env.sample | 1 + components/Modal/User/PrivacySettings.tsx | 88 +++++++++++++++++++ components/Sidebar.tsx | 12 +-- lib/api/archive.ts | 55 ++++++------ .../controllers/collections/postCollection.ts | 8 -- lib/api/controllers/data/getData.ts | 24 +++++ lib/api/controllers/data/postData.ts | 86 ++++++++++++++++++ lib/api/controllers/links/postLink.ts | 5 +- lib/api/storage/readFile.ts | 53 ++++++++--- lib/client/avatarExists.ts | 2 +- pages/api/archives/[...params].ts | 8 +- pages/api/avatar/[id].ts | 32 +++---- pages/api/data/index.ts | 31 +++++++ prisma/schema.prisma | 29 +++--- types/enviornment.d.ts | 1 + types/global.ts | 10 ++- 16 files changed, 352 insertions(+), 93 deletions(-) create mode 100644 lib/api/controllers/data/getData.ts create mode 100644 lib/api/controllers/data/postData.ts create mode 100644 pages/api/data/index.ts diff --git a/.env.sample b/.env.sample index 451abee..4bf7c56 100644 --- a/.env.sample +++ b/.env.sample @@ -6,6 +6,7 @@ NEXTAUTH_URL=http://localhost:3000 PAGINATION_TAKE_COUNT= STORAGE_FOLDER= +AUTOSCROLL_TIMEOUT= # AWS S3 Settings SPACES_KEY= diff --git a/components/Modal/User/PrivacySettings.tsx b/components/Modal/User/PrivacySettings.tsx index 1f66103..e209ae2 100644 --- a/components/Modal/User/PrivacySettings.tsx +++ b/components/Modal/User/PrivacySettings.tsx @@ -6,6 +6,9 @@ import { signOut, useSession } from "next-auth/react"; import { faPenToSquare } from "@fortawesome/free-regular-svg-icons"; import SubmitButton from "../../SubmitButton"; import { toast } from "react-hot-toast"; +import Link from "next/link"; +import ClickAwayHandler from "@/components/ClickAwayHandler"; +import useInitialData from "@/hooks/useInitialData"; type Props = { toggleSettingsModal: Function; @@ -21,6 +24,7 @@ export default function PrivacySettings({ const { update, data } = useSession(); const { account, updateAccount } = useAccountStore(); + const [importDropdown, setImportDropdown] = useState(false); const [submitLoader, setSubmitLoader] = useState(false); const [whitelistedUsersTextbox, setWhiteListedUsersTextbox] = useState( @@ -46,6 +50,38 @@ export default function PrivacySettings({ return wordsArray; }; + const postJSONFile = async (e: any) => { + const file: File = e.target.files[0]; + + if (file) { + var reader = new FileReader(); + reader.readAsText(file, "UTF-8"); + reader.onload = async function (e) { + const load = toast.loading("Importing..."); + + const response = await fetch("/api/data", { + method: "POST", + body: e.target?.result, + }); + + const data = await response.json(); + + toast.dismiss(load); + + toast.success("Imported the Bookmarks! Reloading the page..."); + + setImportDropdown(false); + + setTimeout(() => { + location.reload(); + }, 2000); + }; + reader.onerror = function (e) { + console.log("Error:", e); + }; + } + }; + const submit = async () => { setSubmitLoader(true); @@ -115,6 +151,58 @@ export default function PrivacySettings({ )} +
+

Import/Export Data

+ +
+
setImportDropdown(true)} + className="w-fit relative" + id="import-dropdown" + > +
+ Import From +
+ {importDropdown ? ( + { + const target = e.target as HTMLInputElement; + if (target.id !== "import-dropdown") setImportDropdown(false); + }} + className={`absolute top-7 left-0 w-36 py-1 shadow-md border border-sky-100 bg-gray-50 rounded-md flex flex-col z-20`} + > +
+ +
+ + ) : null} +
+ + +
+ Export Data +
+ +
+
+ -

Dashboard

+

+ Dashboard +

-

- All Links -

+

Links

-

- All Collections +

+ Collections

diff --git a/lib/api/archive.ts b/lib/api/archive.ts index 80aa655..1163f26 100644 --- a/lib/api/archive.ts +++ b/lib/api/archive.ts @@ -10,7 +10,10 @@ export default async function archive(linkId: number, url: string) { try { await page.goto(url, { waitUntil: "domcontentloaded" }); - await autoScroll(page); + await page.evaluate( + autoScroll, + Number(process.env.AUTOSCROLL_TIMEOUT) || 30 + ); const linkExists = await prisma.link.findUnique({ where: { @@ -47,29 +50,31 @@ export default async function archive(linkId: number, url: string) { } } -const autoScroll = async (page: Page) => { - await page.evaluate(async () => { - const timeoutPromise = new Promise((_, reject) => { - setTimeout(() => { - reject(new Error("Auto scroll took too long (more than 20 seconds).")); - }, 20000); - }); - - const scrollingPromise = new Promise((resolve) => { - let totalHeight = 0; - let distance = 100; - let scrollDown = setInterval(() => { - let scrollHeight = document.body.scrollHeight; - window.scrollBy(0, distance); - totalHeight += distance; - if (totalHeight >= scrollHeight) { - clearInterval(scrollDown); - window.scroll(0, 0); - resolve(); - } - }, 100); - }); - - await Promise.race([scrollingPromise, timeoutPromise]); +const autoScroll = async (AUTOSCROLL_TIMEOUT: number) => { + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject( + new Error( + `Auto scroll took too long (more than ${AUTOSCROLL_TIMEOUT} seconds).` + ) + ); + }, AUTOSCROLL_TIMEOUT * 1000); }); + + const scrollingPromise = new Promise((resolve) => { + let totalHeight = 0; + let distance = 100; + let scrollDown = setInterval(() => { + let scrollHeight = document.body.scrollHeight; + window.scrollBy(0, distance); + totalHeight += distance; + if (totalHeight >= scrollHeight) { + clearInterval(scrollDown); + window.scroll(0, 0); + resolve(); + } + }, 100); + }); + + await Promise.race([scrollingPromise, timeoutPromise]); }; diff --git a/lib/api/controllers/collections/postCollection.ts b/lib/api/controllers/collections/postCollection.ts index 917f02f..86ca379 100644 --- a/lib/api/controllers/collections/postCollection.ts +++ b/lib/api/controllers/collections/postCollection.ts @@ -40,14 +40,6 @@ export default async function postCollection( name: collection.name.trim(), description: collection.description, color: collection.color, - members: { - create: collection.members.map((e) => ({ - user: { connect: { id: e.user.id } }, - canCreate: e.canCreate, - canUpdate: e.canUpdate, - canDelete: e.canDelete, - })), - }, }, include: { _count: { diff --git a/lib/api/controllers/data/getData.ts b/lib/api/controllers/data/getData.ts new file mode 100644 index 0000000..6b9e2b0 --- /dev/null +++ b/lib/api/controllers/data/getData.ts @@ -0,0 +1,24 @@ +import { prisma } from "@/lib/api/db"; + +export default async function getData(userId: number) { + const user = await prisma.user.findUnique({ + where: { id: userId }, + include: { + collections: { + include: { + links: { + include: { + tags: true, + }, + }, + }, + }, + }, + }); + + if (!user) return { response: "User not found.", status: 404 }; + + const { password, id, image, ...userData } = user; + + return { response: userData, status: 200 }; +} diff --git a/lib/api/controllers/data/postData.ts b/lib/api/controllers/data/postData.ts new file mode 100644 index 0000000..b9af119 --- /dev/null +++ b/lib/api/controllers/data/postData.ts @@ -0,0 +1,86 @@ +import { prisma } from "@/lib/api/db"; +import { Backup } from "@/types/global"; +import createFolder from "@/lib/api/storage/createFolder"; + +export default async function getData(userId: number, data: Backup) { + // Import collections + try { + data.collections.forEach(async (e) => { + e.name = e.name.trim(); + + const findCollection = await prisma.user.findUnique({ + where: { + id: userId, + }, + select: { + collections: { + where: { + name: e.name, + }, + }, + }, + }); + + const checkIfCollectionExists = findCollection?.collections[0]; + + let collectionId = findCollection?.collections[0]?.id; + + if (!checkIfCollectionExists) { + const newCollection = await prisma.collection.create({ + data: { + owner: { + connect: { + id: userId, + }, + }, + name: e.name, + description: e.description, + color: e.color, + }, + }); + + createFolder({ filePath: `archives/${newCollection.id}` }); + + collectionId = newCollection.id; + } + + // Import Links + e.links.forEach(async (e) => { + const newLink = await prisma.link.create({ + data: { + url: e.url, + name: e.name, + description: e.description, + collection: { + connect: { + id: collectionId, + }, + }, + tags: { + connectOrCreate: e.tags.map((tag) => ({ + where: { + name_ownerId: { + name: tag.name.trim(), + ownerId: userId, + }, + }, + create: { + name: tag.name.trim(), + owner: { + connect: { + id: userId, + }, + }, + }, + })), + }, + }, + }); + }); + }); + } catch (err) { + console.log(err); + } + + return { response: "Success.", status: 200 }; +} diff --git a/lib/api/controllers/links/postLink.ts b/lib/api/controllers/links/postLink.ts index f81a59f..66d8e91 100644 --- a/lib/api/controllers/links/postLink.ts +++ b/lib/api/controllers/links/postLink.ts @@ -20,9 +20,6 @@ export default async function postLink( }; } - // This has to move above we assign link.collection.name - // Because if the link is null (write then delete text on collection) - // It will try to do trim on empty string and will throw and error, this prevents it. if (!link.collection.name) { link.collection.name = "Unnamed Collection"; } @@ -54,7 +51,7 @@ export default async function postLink( ? link.description : await getTitle(link.url); - const newLink: Link = await prisma.link.create({ + const newLink = await prisma.link.create({ data: { url: link.url, name: link.name, diff --git a/lib/api/storage/readFile.ts b/lib/api/storage/readFile.ts index 0cad83b..57dd950 100644 --- a/lib/api/storage/readFile.ts +++ b/lib/api/storage/readFile.ts @@ -9,12 +9,14 @@ import s3Client from "./s3Client"; import util from "util"; type ReturnContentTypes = - | "text/plain" + | "text/html" | "image/jpeg" | "image/png" | "application/pdf"; -export default async function readFile({ filePath }: { filePath: string }) { +export default async function readFile(filePath: string) { + const isRequestingAvatar = filePath.startsWith("uploads/avatar"); + let contentType: ReturnContentTypes; if (s3Client) { @@ -28,6 +30,7 @@ export default async function readFile({ filePath }: { filePath: string }) { | { file: Buffer | string; contentType: ReturnContentTypes; + status: number; } | undefined; @@ -38,11 +41,12 @@ export default async function readFile({ filePath }: { filePath: string }) { try { await headObjectAsync(bucketParams); } catch (err) { - contentType = "text/plain"; + contentType = "text/html"; returnObject = { - file: "File not found, it's possible that the file you're looking for either doesn't exist or hasn't been created yet.", + file: isRequestingAvatar ? "File not found." : fileNotFoundTemplate, contentType, + status: isRequestingAvatar ? 200 : 400, }; } @@ -60,14 +64,14 @@ export default async function readFile({ filePath }: { filePath: string }) { // if (filePath.endsWith(".jpg")) contentType = "image/jpeg"; } - returnObject = { file: data as Buffer, contentType }; + returnObject = { file: data as Buffer, contentType, status: 200 }; } return returnObject; } catch (err) { console.log("Error:", err); - contentType = "text/plain"; + contentType = "text/html"; return { file: "An internal occurred, please contact support.", contentType, @@ -77,13 +81,7 @@ export default async function readFile({ filePath }: { filePath: string }) { const storagePath = process.env.STORAGE_FOLDER || "data"; const creationPath = path.join(process.cwd(), storagePath + "/" + filePath); - const file = fs.existsSync(creationPath) - ? fs.readFileSync(creationPath) - : "File not found, it's possible that the file you're looking for either doesn't exist or hasn't been created yet."; - - if (file.toString().startsWith("File not found")) { - contentType = "text/plain"; - } else if (filePath.endsWith(".pdf")) { + if (filePath.endsWith(".pdf")) { contentType = "application/pdf"; } else if (filePath.endsWith(".png")) { contentType = "image/png"; @@ -92,7 +90,16 @@ export default async function readFile({ filePath }: { filePath: string }) { contentType = "image/jpeg"; } - return { file, contentType }; + if (!fs.existsSync(creationPath)) + return { + file: isRequestingAvatar ? "File not found." : fileNotFoundTemplate, + contentType: "text/html", + status: isRequestingAvatar ? 200 : 400, + }; + else { + const file = fs.readFileSync(creationPath); + return { file, contentType, status: 200 }; + } } } @@ -105,3 +112,21 @@ const streamToBuffer = (stream: any) => { stream.on("end", () => resolve(Buffer.concat(chunks))); }); }; + +const fileNotFoundTemplate = ` + + + + + File not found + + +

File not found

+

It is possible that the file you're looking for either doesn't exist or hasn't been created yet.

+

Some possible reasons are:

+
    +
  • You are trying to access a file too early, before it has been fully archived.
    • +
  • The file doesn't exist either because it encountered an error while being archived, or it simply doesn't exist.
    • +
+ + `; diff --git a/lib/client/avatarExists.ts b/lib/client/avatarExists.ts index 7490f3b..2e82fa6 100644 --- a/lib/client/avatarExists.ts +++ b/lib/client/avatarExists.ts @@ -1,4 +1,4 @@ export default async function avatarExists(fileUrl: string): Promise { const response = await fetch(fileUrl, { method: "HEAD" }); - return !(response.headers.get("content-type") === "text/plain"); + return !(response.headers.get("content-type") === "text/html"); } diff --git a/pages/api/archives/[...params].ts b/pages/api/archives/[...params].ts index 757f635..3e659a6 100644 --- a/pages/api/archives/[...params].ts +++ b/pages/api/archives/[...params].ts @@ -31,10 +31,10 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) { .status(401) .json({ response: "You don't have access to this collection." }); - const { file, contentType } = await readFile({ - filePath: `archives/${collectionId}/${linkId}`, - }); - res.setHeader("Content-Type", contentType).status(200); + const { file, contentType, status } = await readFile( + `archives/${collectionId}/${linkId}` + ); + res.setHeader("Content-Type", contentType).status(status as number); return res.send(file); } diff --git a/pages/api/avatar/[id].ts b/pages/api/avatar/[id].ts index ac4f1d3..a006dbf 100644 --- a/pages/api/avatar/[id].ts +++ b/pages/api/avatar/[id].ts @@ -13,7 +13,7 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) { if (!userId || !username) return res - .setHeader("Content-Type", "text/plain") + .setHeader("Content-Type", "text/html") .status(401) .send("You must be logged in."); else if (session?.user?.isSubscriber === false) @@ -24,7 +24,7 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) { if (!queryId) return res - .setHeader("Content-Type", "text/plain") + .setHeader("Content-Type", "text/html") .status(401) .send("Invalid parameters."); @@ -34,27 +34,27 @@ export default async function Index(req: NextApiRequest, res: NextApiResponse) { id: queryId, }, include: { - whitelistedUsers: true - } + whitelistedUsers: true, + }, }); - const whitelistedUsernames = targetUser?.whitelistedUsers.map(whitelistedUsername => whitelistedUsername.username); + const whitelistedUsernames = targetUser?.whitelistedUsers.map( + (whitelistedUsername) => whitelistedUsername.username + ); - if ( - targetUser?.isPrivate && - !whitelistedUsernames?.includes(username) - ) { + if (targetUser?.isPrivate && !whitelistedUsernames?.includes(username)) { return res - .setHeader("Content-Type", "text/plain") + .setHeader("Content-Type", "text/html") .send("This profile is private."); } } - const { file, contentType } = await readFile({ - filePath: `uploads/avatar/${queryId}.jpg`, - }); + const { file, contentType, status } = await readFile( + `uploads/avatar/${queryId}.jpg` + ); - res.setHeader("Content-Type", contentType); - - return res.send(file); + return res + .setHeader("Content-Type", contentType) + .status(status as number) + .send(file); } diff --git a/pages/api/data/index.ts b/pages/api/data/index.ts new file mode 100644 index 0000000..c01245f --- /dev/null +++ b/pages/api/data/index.ts @@ -0,0 +1,31 @@ +import type { NextApiRequest, NextApiResponse } from "next"; +import { getServerSession } from "next-auth/next"; +import { authOptions } from "@/pages/api/auth/[...nextauth]"; +import getData from "@/lib/api/controllers/data/getData"; +import postData from "@/lib/api/controllers/data/postData"; + +export default async function users(req: NextApiRequest, res: NextApiResponse) { + const session = await getServerSession(req, res, authOptions); + + if (!session?.user.id) { + return res.status(401).json({ response: "You must be logged in." }); + } else if (session?.user?.isSubscriber === false) + res.status(401).json({ + response: + "You are not a subscriber, feel free to reach out to us at support@linkwarden.app in case of any issues.", + }); + + if (req.method === "GET") { + const data = await getData(session.user.id); + if (data.status === 200) + return res + .setHeader("Content-Type", "application/json") + .setHeader("Content-Disposition", "attachment; filename=backup.json") + .status(data.status) + .json(data.response); + } else if (req.method === "POST") { + console.log(JSON.parse(req.body)); + const data = await postData(session.user.id, JSON.parse(req.body)); + return res.status(data.status).json({ response: data.response }); + } +} diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c1c04d8..eca457b 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -93,31 +93,32 @@ model Collection { } model UsersAndCollections { - user User @relation(fields: [userId], references: [id]) - userId Int + user User @relation(fields: [userId], references: [id]) + userId Int collection Collection @relation(fields: [collectionId], references: [id]) collectionId Int - canCreate Boolean - canUpdate Boolean - canDelete Boolean + canCreate Boolean + canUpdate Boolean + canDelete Boolean @@id([userId, collectionId]) } model Link { - id Int @id @default(autoincrement()) - name String - url String - description String @default("") + id Int @id @default(autoincrement()) + name String + url String + description String @default("") - pinnedBy User[] + pinnedBy User[] - collection Collection @relation(fields: [collectionId], references: [id]) - collectionId Int - tags Tag[] - createdAt DateTime @default(now()) + collection Collection @relation(fields: [collectionId], references: [id]) + collectionId Int + tags Tag[] + + createdAt DateTime @default(now()) } model Tag { diff --git a/types/enviornment.d.ts b/types/enviornment.d.ts index 047994c..1b2bb51 100644 --- a/types/enviornment.d.ts +++ b/types/enviornment.d.ts @@ -6,6 +6,7 @@ declare global { NEXTAUTH_URL: string; PAGINATION_TAKE_COUNT?: string; STORAGE_FOLDER?: string; + AUTOSCROLL_TIMEOUT?: string; SPACES_KEY?: string; SPACES_SECRET?: string; diff --git a/types/global.ts b/types/global.ts index 04b68e0..c4adcb5 100644 --- a/types/global.ts +++ b/types/global.ts @@ -36,7 +36,7 @@ export interface CollectionIncludingMembersAndLinkCount export interface AccountSettings extends User { profilePic: string; newPassword?: string; - whitelistedUsers: string[] + whitelistedUsers: string[]; } interface LinksIncludingTags extends Link { @@ -77,3 +77,11 @@ export type PublicLinkRequestQuery = { cursor?: number; collectionId: number; }; + +interface CollectionIncludingLinks extends Collection { + links: LinksIncludingTags[]; +} + +export interface Backup extends Omit { + collections: CollectionIncludingLinks[]; +}