From 5e6d46b6b94c249f1a1254fa91c3cdc205c46cfe Mon Sep 17 00:00:00 2001 From: daniel31x13 Date: Thu, 25 Jul 2024 13:43:55 -0400 Subject: [PATCH 1/9] bug fixed --- lib/api/archiveHandler.ts | 11 +- scripts/migration/indexArchives.js | 163 +++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 5 deletions(-) create mode 100644 scripts/migration/indexArchives.js diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.ts index c5e5f09..b158d97 100644 --- a/lib/api/archiveHandler.ts +++ b/lib/api/archiveHandler.ts @@ -86,17 +86,18 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) { image: user.archiveAsScreenshot && !link.image?.startsWith("archive") ? "pending" - : "unavailable", + : undefined, pdf: user.archiveAsPDF && !link.pdf?.startsWith("archive") ? "pending" - : "unavailable", + : undefined, + monolith: + user.archiveAsMonolith && !link.monolith?.startsWith("archive") + ? "pending" + : undefined, readable: !link.readable?.startsWith("archive") ? "pending" : undefined, - monolith: !link.monolith?.startsWith("archive") - ? "pending" - : undefined, preview: !link.readable?.startsWith("archive") ? "pending" : undefined, diff --git a/scripts/migration/indexArchives.js b/scripts/migration/indexArchives.js new file mode 100644 index 0000000..6f362da --- /dev/null +++ b/scripts/migration/indexArchives.js @@ -0,0 +1,163 @@ +// This is a script that looks for every link and checks if the preservations exist in the filesystem. +// If they do, it updates the link with the path to the preservation. +// If they don't, it passes. + +const { S3 } = require("@aws-sdk/client-s3"); +const { PrismaClient } = require("@prisma/client"); +const { existsSync } = require("fs"); +const util = require("util"); + +const prisma = new PrismaClient(); + +const STORAGE_FOLDER = process.env.STORAGE_FOLDER || "data"; + +const s3Client = + process.env.SPACES_ENDPOINT && + process.env.SPACES_REGION && + process.env.SPACES_KEY && + process.env.SPACES_SECRET + ? new S3({ + forcePathStyle: false, + endpoint: process.env.SPACES_ENDPOINT, + region: process.env.SPACES_REGION, + credentials: { + accessKeyId: process.env.SPACES_KEY, + secretAccessKey: process.env.SPACES_SECRET, + }, + }) + : undefined; + +async function checkFileExistence(path) { + if (s3Client) { + // One millisecond delay to avoid rate limiting + await new Promise((resolve) => setTimeout(resolve, 1)); + + const bucketParams = { + Bucket: process.env.SPACES_BUCKET_NAME, + Key: path, + }; + + try { + const headObjectAsync = util.promisify( + s3Client.headObject.bind(s3Client) + ); + + try { + await headObjectAsync(bucketParams); + return true; + } catch (err) { + return false; + } + } catch (err) { + console.log("Error:", err); + + return false; + } + } else { + try { + if (existsSync(STORAGE_FOLDER + "/" + path)) { + return true; + } else return false; + } catch (err) { + console.log(err); + } + } +} + +async function indexArchives() { + const links = await prisma.link.findMany({ + orderBy: { id: "asc" }, + }); + + // PDFs + for (let link of links) { + const path = `archives/${link.collectionId}/${link.id}.pdf`; + + const res = await checkFileExistence(path); + + if (res) { + await prisma.link.update({ + where: { id: link.id }, + data: { pdf: path }, + }); + console.log(`${link.id}`); + } else { + console.log(`${link.id}`); + } + } + + // Screenshots (PNGs) + for (let link of links) { + const path = `archives/${link.collectionId}/${link.id}.png`; + + const res = await checkFileExistence(path); + + if (res) { + await prisma.link.update({ + where: { id: link.id }, + data: { image: path }, + }); + console.log(`${link.id}`); + } else { + console.log(`${link.id}`); + } + } + + // Screenshots (JPEGs) + for (let link of links) { + const path = `archives/${link.collectionId}/${link.id}.jpeg`; + + const res = await checkFileExistence(path); + + if (res) { + await prisma.link.update({ + where: { id: link.id }, + data: { image: path }, + }); + console.log(`${link.id}`); + } else { + console.log(`${link.id}`); + } + } + + // Readability + for (let link of links) { + const path = `archives/${link.collectionId}/${link.id}_readability.json`; + + const res = await checkFileExistence(path); + + if (res) { + await prisma.link.update({ + where: { id: link.id }, + data: { readable: path }, + }); + console.log(`${link.id}`); + } else { + console.log(`${link.id}`); + } + } + + // Webpages + for (let link of links) { + const path = `archives/${link.collectionId}/${link.id}.html`; + + const res = await checkFileExistence(path); + + if (res) { + await prisma.link.update({ + where: { id: link.id }, + data: { monolith: path }, + }); + console.log(`${link.id}`); + } else { + console.log(`${link.id}`); + } + } + + await prisma.$disconnect(); +} + +indexArchives().catch((e) => { + console.error(e); + process.exit(1); +}); From 6cae2fb634d0d6cb347c961d49b6286a7b280526 Mon Sep 17 00:00:00 2001 From: daniel31x13 Date: Thu, 25 Jul 2024 13:45:44 -0400 Subject: [PATCH 2/9] update version number --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index db8cf5d..9a0fee3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "linkwarden", - "version": "v2.6.0", + "version": "v2.6.1", "main": "index.js", "repository": "https://github.com/linkwarden/linkwarden.git", "author": "Daniel31X13 ", From 9d8ae6970cf775057a9e0b1be89537787dec4132 Mon Sep 17 00:00:00 2001 From: daniel31x13 Date: Thu, 25 Jul 2024 13:57:33 -0400 Subject: [PATCH 3/9] minor fix --- scripts/migration/indexArchives.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/migration/indexArchives.js b/scripts/migration/indexArchives.js index 6f362da..9ef2809 100644 --- a/scripts/migration/indexArchives.js +++ b/scripts/migration/indexArchives.js @@ -66,6 +66,14 @@ async function checkFileExistence(path) { async function indexArchives() { const links = await prisma.link.findMany({ + select: { + id: true, + collectionId: true, + image: true, + pdf: true, + readable: true, + monolith: true, + }, orderBy: { id: "asc" }, }); From af7f0fb47c0ed476ac06681c4067d1520f6f14ef Mon Sep 17 00:00:00 2001 From: daniel31x13 Date: Thu, 25 Jul 2024 14:15:08 -0400 Subject: [PATCH 4/9] make script more efficient --- ...exArchives.js => pdfScreenshotIndexing.js} | 42 ++----------------- 1 file changed, 4 insertions(+), 38 deletions(-) rename scripts/migration/{indexArchives.js => pdfScreenshotIndexing.js} (76%) diff --git a/scripts/migration/indexArchives.js b/scripts/migration/pdfScreenshotIndexing.js similarity index 76% rename from scripts/migration/indexArchives.js rename to scripts/migration/pdfScreenshotIndexing.js index 9ef2809..a7659cf 100644 --- a/scripts/migration/indexArchives.js +++ b/scripts/migration/pdfScreenshotIndexing.js @@ -1,5 +1,5 @@ -// This is a script that looks for every link and checks if the preservations exist in the filesystem. -// If they do, it updates the link with the path to the preservation. +// This is a script that looks for every link and checks if the pdf/screenshot exist in the filesystem. +// If they do, it updates the link with the path in the db. // If they don't, it passes. const { S3 } = require("@aws-sdk/client-s3"); @@ -64,7 +64,7 @@ async function checkFileExistence(path) { } } -async function indexArchives() { +async function pdfScreenshotIndexing() { const links = await prisma.link.findMany({ select: { id: true, @@ -128,44 +128,10 @@ async function indexArchives() { } } - // Readability - for (let link of links) { - const path = `archives/${link.collectionId}/${link.id}_readability.json`; - - const res = await checkFileExistence(path); - - if (res) { - await prisma.link.update({ - where: { id: link.id }, - data: { readable: path }, - }); - console.log(`${link.id}`); - } else { - console.log(`${link.id}`); - } - } - - // Webpages - for (let link of links) { - const path = `archives/${link.collectionId}/${link.id}.html`; - - const res = await checkFileExistence(path); - - if (res) { - await prisma.link.update({ - where: { id: link.id }, - data: { monolith: path }, - }); - console.log(`${link.id}`); - } else { - console.log(`${link.id}`); - } - } - await prisma.$disconnect(); } -indexArchives().catch((e) => { +pdfScreenshotIndexing().catch((e) => { console.error(e); process.exit(1); }); From 7e96ba63dffa77e39b35c70ef063e6b7ba81a5dd Mon Sep 17 00:00:00 2001 From: daniel31x13 Date: Thu, 25 Jul 2024 14:21:39 -0400 Subject: [PATCH 5/9] minor improvement --- scripts/migration/pdfScreenshotIndexing.js | 23 +++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/scripts/migration/pdfScreenshotIndexing.js b/scripts/migration/pdfScreenshotIndexing.js index a7659cf..c688a13 100644 --- a/scripts/migration/pdfScreenshotIndexing.js +++ b/scripts/migration/pdfScreenshotIndexing.js @@ -65,6 +65,8 @@ async function checkFileExistence(path) { } async function pdfScreenshotIndexing() { + let counter = 0; + const links = await prisma.link.findMany({ select: { id: true, @@ -88,10 +90,11 @@ async function pdfScreenshotIndexing() { where: { id: link.id }, data: { pdf: path }, }); - console.log(`${link.id}`); - } else { - console.log(`${link.id}`); } + + console.log(counter, "id:", link.id, "PDF"); + + counter++; } // Screenshots (PNGs) @@ -105,10 +108,11 @@ async function pdfScreenshotIndexing() { where: { id: link.id }, data: { image: path }, }); - console.log(`${link.id}`); - } else { - console.log(`${link.id}`); } + + console.log(counter, "id:", link.id, "PNG"); + + counter++; } // Screenshots (JPEGs) @@ -122,10 +126,11 @@ async function pdfScreenshotIndexing() { where: { id: link.id }, data: { image: path }, }); - console.log(`${link.id}`); - } else { - console.log(`${link.id}`); } + + console.log(counter, "id:", link.id, "JPEG"); + + counter++; } await prisma.$disconnect(); From 6983e41576c4623667e1753cc4b1ae52b74df0d8 Mon Sep 17 00:00:00 2001 From: daniel31x13 Date: Thu, 25 Jul 2024 14:23:33 -0400 Subject: [PATCH 6/9] minor improvement --- scripts/migration/pdfScreenshotIndexing.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/migration/pdfScreenshotIndexing.js b/scripts/migration/pdfScreenshotIndexing.js index c688a13..422085b 100644 --- a/scripts/migration/pdfScreenshotIndexing.js +++ b/scripts/migration/pdfScreenshotIndexing.js @@ -65,8 +65,6 @@ async function checkFileExistence(path) { } async function pdfScreenshotIndexing() { - let counter = 0; - const links = await prisma.link.findMany({ select: { id: true, @@ -79,6 +77,8 @@ async function pdfScreenshotIndexing() { orderBy: { id: "asc" }, }); + let counter = 0; + // PDFs for (let link of links) { const path = `archives/${link.collectionId}/${link.id}.pdf`; @@ -92,11 +92,13 @@ async function pdfScreenshotIndexing() { }); } - console.log(counter, "id:", link.id, "PDF"); + console.log("count:", counter, "id:", link.id, "PDF"); counter++; } + counter = 0; + // Screenshots (PNGs) for (let link of links) { const path = `archives/${link.collectionId}/${link.id}.png`; @@ -110,11 +112,13 @@ async function pdfScreenshotIndexing() { }); } - console.log(counter, "id:", link.id, "PNG"); + console.log("count:", counter, "id:", link.id, "PNG"); counter++; } + counter = 0; + // Screenshots (JPEGs) for (let link of links) { const path = `archives/${link.collectionId}/${link.id}.jpeg`; @@ -128,7 +132,7 @@ async function pdfScreenshotIndexing() { }); } - console.log(counter, "id:", link.id, "JPEG"); + console.log("count:", counter, "id:", link.id, "JPEG"); counter++; } From 8e6f88d29fb46d8bf096cfb23857e61fa4c10abd Mon Sep 17 00:00:00 2001 From: daniel31x13 Date: Thu, 25 Jul 2024 23:43:26 -0400 Subject: [PATCH 7/9] merged the two migration scripts for v2.6.1 --- scripts/migration/descriptionToName.js | 70 ------------------- .../index.js} | 59 +++++++++++----- 2 files changed, 41 insertions(+), 88 deletions(-) delete mode 100644 scripts/migration/descriptionToName.js rename scripts/migration/{pdfScreenshotIndexing.js => v2.6.1/index.js} (68%) diff --git a/scripts/migration/descriptionToName.js b/scripts/migration/descriptionToName.js deleted file mode 100644 index 435e59f..0000000 --- a/scripts/migration/descriptionToName.js +++ /dev/null @@ -1,70 +0,0 @@ -// [Optional, but recommended] - -// We decided that the "name" field should be the auto-generated field instead of the "description" field, so we need to -// move the data from the "description" field to the "name" field for links that have an empty name. - -// This script is meant to be run only once. - -// Run the script with `node scripts/migration/descriptionToName.js` - -const { PrismaClient } = require("@prisma/client"); - -const prisma = new PrismaClient(); - -async function main() { - console.log("Starting..."); - - const count = await prisma.link.count({ - where: { - name: "", - description: { - not: "", - }, - }, - }); - - console.log( - `Applying the changes to ${count} ${ - count == 1 ? "link" : "links" - } in 10 seconds...` - ); - - await new Promise((resolve) => setTimeout(resolve, 10000)); - - console.log("Applying the changes..."); - - const links = await prisma.link.findMany({ - where: { - name: "", - description: { - not: "", - }, - }, - select: { - id: true, - description: true, - }, - }); - - for (const link of links) { - await prisma.link.update({ - where: { - id: link.id, - }, - data: { - name: link.description, - description: "", - }, - }); - } - - console.log("Done!"); -} - -main() - .catch((e) => { - throw e; - }) - .finally(async () => { - await prisma.$disconnect(); - }); diff --git a/scripts/migration/pdfScreenshotIndexing.js b/scripts/migration/v2.6.1/index.js similarity index 68% rename from scripts/migration/pdfScreenshotIndexing.js rename to scripts/migration/v2.6.1/index.js index 422085b..0c537e0 100644 --- a/scripts/migration/pdfScreenshotIndexing.js +++ b/scripts/migration/v2.6.1/index.js @@ -1,4 +1,12 @@ -// This is a script that looks for every link and checks if the pdf/screenshot exist in the filesystem. +// Run the script with `node scripts/migration/v2.6.1/index.js` +// Docker users can run the script with `docker exec -it CONTAINER_ID /bin/bash -c 'node scripts/migration/v2.6.1/index.js'` + +// There are two parts to this script: + +// Firstly we decided that the "name" field should be the auto-generated field instead of the "description" field, so we need to +// move the data from the "description" field to the "name" field for links that have an empty name. + +// Secondly it looks for every link and checks if the pdf/screenshot exist in the filesystem. // If they do, it updates the link with the path in the db. // If they don't, it passes. @@ -64,7 +72,34 @@ async function checkFileExistence(path) { } } -async function pdfScreenshotIndexing() { +async function main() { + console.log("Starting... Please do not interrupt the process."); + + const linksWithoutName = await prisma.link.findMany({ + where: { + name: "", + description: { + not: "", + }, + }, + select: { + id: true, + description: true, + }, + }); + + for (const link of linksWithoutName) { + await prisma.link.update({ + where: { + id: link.id, + }, + data: { + name: link.description, + description: "", + }, + }); + } + const links = await prisma.link.findMany({ select: { id: true, @@ -77,8 +112,6 @@ async function pdfScreenshotIndexing() { orderBy: { id: "asc" }, }); - let counter = 0; - // PDFs for (let link of links) { const path = `archives/${link.collectionId}/${link.id}.pdf`; @@ -92,13 +125,9 @@ async function pdfScreenshotIndexing() { }); } - console.log("count:", counter, "id:", link.id, "PDF"); - - counter++; + console.log("Indexing the PDF for link:", link.id); } - counter = 0; - // Screenshots (PNGs) for (let link of links) { const path = `archives/${link.collectionId}/${link.id}.png`; @@ -112,13 +141,9 @@ async function pdfScreenshotIndexing() { }); } - console.log("count:", counter, "id:", link.id, "PNG"); - - counter++; + console.log("Indexing the PNG for link:", link.id); } - counter = 0; - // Screenshots (JPEGs) for (let link of links) { const path = `archives/${link.collectionId}/${link.id}.jpeg`; @@ -132,15 +157,13 @@ async function pdfScreenshotIndexing() { }); } - console.log("count:", counter, "id:", link.id, "JPEG"); - - counter++; + console.log("Indexing the JPEG for link:", link.id); } await prisma.$disconnect(); } -pdfScreenshotIndexing().catch((e) => { +main().catch((e) => { console.error(e); process.exit(1); }); From 061e22d225ad7566e94f4033e818d5be75bd7977 Mon Sep 17 00:00:00 2001 From: daniel31x13 Date: Fri, 26 Jul 2024 11:54:13 -0400 Subject: [PATCH 8/9] bug fixed --- pages/api/v1/auth/[...nextauth].ts | 5 +---- pages/api/v1/logins/index.ts | 3 +-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/pages/api/v1/auth/[...nextauth].ts b/pages/api/v1/auth/[...nextauth].ts index dd38aa7..17bcaad 100644 --- a/pages/api/v1/auth/[...nextauth].ts +++ b/pages/api/v1/auth/[...nextauth].ts @@ -79,10 +79,7 @@ const STRIPE_SECRET_KEY = process.env.STRIPE_SECRET_KEY; const providers: Provider[] = []; -if ( - process.env.NEXT_PUBLIC_CREDENTIALS_ENABLED === "true" || - process.env.NEXT_PUBLIC_CREDENTIALS_ENABLED === undefined -) { +if (process.env.NEXT_PUBLIC_CREDENTIALS_ENABLED !== "false") { // undefined is for backwards compatibility providers.push( CredentialsProvider({ diff --git a/pages/api/v1/logins/index.ts b/pages/api/v1/logins/index.ts index 7e290af..8a98210 100644 --- a/pages/api/v1/logins/index.ts +++ b/pages/api/v1/logins/index.ts @@ -414,8 +414,7 @@ export function getLogins() { } return { credentialsEnabled: - process.env.NEXT_PUBLIC_CREDENTIALS_ENABLED === "true" || - process.env.NEXT_PUBLIC_CREDENTIALS_ENABLED === undefined + process.env.NEXT_PUBLIC_CREDENTIALS_ENABLED !== "false" ? "true" : "false", emailEnabled: From cd82083e09c3e673e890a49788b08010e3a45312 Mon Sep 17 00:00:00 2001 From: daniel31x13 Date: Fri, 26 Jul 2024 12:00:46 -0400 Subject: [PATCH 9/9] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9a0fee3..1c428cf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "linkwarden", - "version": "v2.6.1", + "version": "v2.6.2", "main": "index.js", "repository": "https://github.com/linkwarden/linkwarden.git", "author": "Daniel31X13 ",