From e7cb87a05d14bedf0b6990c6d0adc4dc4cc9b999 Mon Sep 17 00:00:00 2001 From: Ronny Engan Date: Thu, 8 Oct 2026 22:11:46 +0200 Subject: [PATCH 1/3] Removed test to avoid errors --- server/package.json | 1 - server/src/scripts/importMockBooks.ts | 21 --------------- server/test-data/books.json | 38 --------------------------- 3 files changed, 60 deletions(-) delete mode 100644 server/src/scripts/importMockBooks.ts delete mode 100644 server/test-data/books.json diff --git a/server/package.json b/server/package.json index 4fcbb98..f08aa5f 100644 --- a/server/package.json +++ b/server/package.json @@ -5,7 +5,6 @@ "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "dev": "tsx watch --env-file=.env src/index.ts", - "import:mock": "tsx --env-file=.env src/scripts/importMockBooks.ts", "build": "tsc", "start": "node dist/index.js", "format": "prettier --write .", diff --git a/server/src/scripts/importMockBooks.ts b/server/src/scripts/importMockBooks.ts deleted file mode 100644 index e0ce859..0000000 --- a/server/src/scripts/importMockBooks.ts +++ /dev/null @@ -1,21 +0,0 @@ -import fs from "node:fs"; -import { connectDatabase } from "../database/database.js"; -import { Book } from "../models/Book.js"; - -async function importMockBooks() { - await connectDatabase(); - - const file = fs.readFileSync("./test-data/books.json", "utf-8"); - const books = JSON.parse(file); - - // Clear existing books before importing mock data - await Book.deleteMany({}); - - await Book.insertMany(books); - - console.log(`Imported ${books.length} books`); - - process.exit(0); -} - -importMockBooks(); \ No newline at end of file diff --git a/server/test-data/books.json b/server/test-data/books.json deleted file mode 100644 index 531110f..0000000 --- a/server/test-data/books.json +++ /dev/null @@ -1,38 +0,0 @@ -[ - { - "googleBooksId": "mock-001", - "title": "The Hobbit", - "subtitle": "There and Back Again", - "authors": ["J. R. R. Tolkien"], - "publishedYear": 1937, - "genres": ["Fantasy", "Adventure"], - "description": "Bilbo Baggins is unexpectedly swept into an adventure.", - "coverURL": "https://example.com/hobbit.jpg", - "pageCount": 310, - "language": "en" - }, - { - "googleBooksId": "mock-002", - "title": "Dune", - "subtitle": "", - "authors": ["Frank Herbert"], - "publishedYear": 1965, - "genres": ["Science Fiction"], - "description": "A science-fiction novel set in a distant future.", - "coverURL": "https://example.com/dune.jpg", - "pageCount": 412, - "language": "en" - }, - { - "googleBooksId": "mock-003", - "title": "Good Omens", - "subtitle": "The Nice and Accurate Prophecies of Agnes Nutter, Witch", - "authors": ["Terry Pratchett", "Neil Gaiman"], - "publishedYear": 1990, - "genres": ["Fantasy", "Comedy"], - "description": "An angel and a demon attempt to prevent the apocalypse.", - "coverURL": "https://example.com/good-omens.jpg", - "pageCount": 288, - "language": "en" - } -] \ No newline at end of file From 4a556870cdc878d17c0595dd04b6e4527519456b Mon Sep 17 00:00:00 2001 From: Ronny Engan Date: Thu, 8 Oct 2026 22:27:53 +0200 Subject: [PATCH 2/3] Restore mock books for frontend development --- server/test-data/books.json | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 server/test-data/books.json diff --git a/server/test-data/books.json b/server/test-data/books.json new file mode 100644 index 0000000..531110f --- /dev/null +++ b/server/test-data/books.json @@ -0,0 +1,38 @@ +[ + { + "googleBooksId": "mock-001", + "title": "The Hobbit", + "subtitle": "There and Back Again", + "authors": ["J. R. R. Tolkien"], + "publishedYear": 1937, + "genres": ["Fantasy", "Adventure"], + "description": "Bilbo Baggins is unexpectedly swept into an adventure.", + "coverURL": "https://example.com/hobbit.jpg", + "pageCount": 310, + "language": "en" + }, + { + "googleBooksId": "mock-002", + "title": "Dune", + "subtitle": "", + "authors": ["Frank Herbert"], + "publishedYear": 1965, + "genres": ["Science Fiction"], + "description": "A science-fiction novel set in a distant future.", + "coverURL": "https://example.com/dune.jpg", + "pageCount": 412, + "language": "en" + }, + { + "googleBooksId": "mock-003", + "title": "Good Omens", + "subtitle": "The Nice and Accurate Prophecies of Agnes Nutter, Witch", + "authors": ["Terry Pratchett", "Neil Gaiman"], + "publishedYear": 1990, + "genres": ["Fantasy", "Comedy"], + "description": "An angel and a demon attempt to prevent the apocalypse.", + "coverURL": "https://example.com/good-omens.jpg", + "pageCount": 288, + "language": "en" + } +] \ No newline at end of file From bcec9a8b7a8064152dbd1a9d637a48edb67e03c4 Mon Sep 17 00:00:00 2001 From: Ronny Engan Date: Fri, 9 Oct 2026 17:16:11 +0200 Subject: [PATCH 3/3] feat: add Google Books API importer --- server/src/models/Book.ts | 5 ++ server/src/scripts/importBooks.ts | 93 +++++++++++++++++++++++++++ server/src/scripts/testBookImport.ts | 55 ++++++++++++++++ server/src/scripts/testGoogleBooks.ts | 29 +++++++++ server/src/utils/fetchGoogleBooks.ts | 45 +++++++++++++ server/src/utils/genreMapping.ts | 19 ++++++ server/src/utils/transformBook.ts | 19 ++++++ server/src/utils/validateBook.ts | 15 +++++ 8 files changed, 280 insertions(+) create mode 100644 server/src/scripts/importBooks.ts create mode 100644 server/src/scripts/testBookImport.ts create mode 100644 server/src/scripts/testGoogleBooks.ts create mode 100644 server/src/utils/fetchGoogleBooks.ts create mode 100644 server/src/utils/genreMapping.ts create mode 100644 server/src/utils/transformBook.ts create mode 100644 server/src/utils/validateBook.ts diff --git a/server/src/models/Book.ts b/server/src/models/Book.ts index 307b112..737dab0 100644 --- a/server/src/models/Book.ts +++ b/server/src/models/Book.ts @@ -27,6 +27,11 @@ const bookSchema = new mongoose.Schema({ default: null, }, + googleCategories: { + type: [String], + default: [], + }, + genres: { type: [String], default: [], diff --git a/server/src/scripts/importBooks.ts b/server/src/scripts/importBooks.ts new file mode 100644 index 0000000..78aff0e --- /dev/null +++ b/server/src/scripts/importBooks.ts @@ -0,0 +1,93 @@ + +import mongoose from "mongoose"; +import { connectDatabase } from "../database/database.js"; +import { Book } from "../models/Book.js"; +import { isValidBook } from "../utils/validateBook.js"; +import { transformBook } from "../utils/transformBook.js"; +import { fetchGoogleBooks } from "../utils/fetchGoogleBooks.js"; + +const SUBJECTS = [ + "fantasy", + "science_fiction", + "mystery", + "history", + "biography", +]; + +const BOOKS_PER_SUBJECT = 20; + +async function importBooks() { + let fetched = 0; + let invalid = 0; + let inserted = 0; + let updated = 0; + + try { + const apiKey = process.env.GOOGLE_BOOKS_API_KEY; + + if (!apiKey) { + throw new Error("Missing GOOGLE_BOOKS_API_KEY"); + } + + await connectDatabase(); + + for (const subject of SUBJECTS) { + console.log(`\nFetching books: ${subject}`); + + const url = new URL( + "https://www.googleapis.com/books/v1/volumes" + ); + + url.searchParams.set("q", subject.replaceAll("_", " ")); + url.searchParams.set("maxResults", String(BOOKS_PER_SUBJECT)); + url.searchParams.set("startIndex", "0"); + url.searchParams.set("key", apiKey); + + const data = await fetchGoogleBooks(url); + + console.log("Query:", url.searchParams.get("q")); + console.log("Total matching results:", data.totalItems); + console.log("Items returned:", data.items?.length ?? 0); + + const books = data.items ?? []; + + for (const book of books) { + fetched++; + + if (!isValidBook(book)) { + invalid++; + continue; + } + + const transformedBook = transformBook(book); + + const result = await Book.updateOne( + { googleBooksId: transformedBook.googleBooksId }, + { $set: transformedBook }, + { upsert: true } + ); + + if (result.upsertedCount > 0) { + inserted++; + } else if (result.matchedCount > 0) { + updated++; + } + } + + console.log(`Processed ${books.length} results`); + } + } catch (error) { + console.error("Import stopped:", error); + } finally { + console.log("\n--- Import Summary ---"); + console.log(`Fetched: ${fetched}`); + console.log(`Invalid/skipped: ${invalid}`); + console.log(`New books inserted: ${inserted}`); + console.log(`Existing books matched: ${updated}`); + console.log(`Total processed: ${fetched}`); + + await mongoose.disconnect(); + } +} + +importBooks(); diff --git a/server/src/scripts/testBookImport.ts b/server/src/scripts/testBookImport.ts new file mode 100644 index 0000000..7d2ee80 --- /dev/null +++ b/server/src/scripts/testBookImport.ts @@ -0,0 +1,55 @@ +import mongoose from "mongoose"; +import { connectDatabase } from "../database/database.js"; +import { Book } from "../models/Book.js"; +import { isValidBook } from "../utils/validateBook.js"; +import { transformBook } from "../utils/transformBook.js"; + +async function testImport() { + try { + await connectDatabase(); + + const apiKey = process.env.GOOGLE_BOOKS_API_KEY; + + if (!apiKey) { + throw new Error("Missing GOOGLE_BOOKS_API_KEY"); + } + + const url = new URL("https://www.googleapis.com/books/v1/volumes"); + + url.searchParams.set("q", "harry potter"); + url.searchParams.set("maxResults", "2"); + url.searchParams.set("key", apiKey); + + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Google API error: ${response.status}`); + } + + const data = await response.json(); + + for (const book of data.items ?? []) { + if (!isValidBook(book)) { + console.log(`Skipped: ${book.volumeInfo?.title}`); + continue; + } + + const transformedBook = transformBook(book); + + await Book.updateOne( + { googleBooksId: transformedBook.googleBooksId }, + { $set: transformedBook }, + { upsert: true } + ); + + console.log(`Saved: ${transformedBook.title}`); + break; // Only save one valid book for testing + } + } catch (error) { + console.error("Import failed:", error); + } finally { + await mongoose.disconnect(); + } +} + +testImport(); \ No newline at end of file diff --git a/server/src/scripts/testGoogleBooks.ts b/server/src/scripts/testGoogleBooks.ts new file mode 100644 index 0000000..878d596 --- /dev/null +++ b/server/src/scripts/testGoogleBooks.ts @@ -0,0 +1,29 @@ +import { isValidBook } from "../utils/validateBook.js"; +import { transformBook } from "../utils/transformBook.js"; +import { fetchGoogleBooks } from "../utils/fetchGoogleBooks.js"; + +const apiKey = process.env.GOOGLE_BOOKS_API_KEY; + +if (!apiKey) { + throw new Error("Missing GOOGLE_BOOKS_API_KEY in .env"); +} + +const url = new URL("https://www.googleapis.com/books/v1/volumes"); + +url.searchParams.set("q", "harry potter"); +url.searchParams.set("maxResults", "5"); +url.searchParams.set("startIndex", "5"); +url.searchParams.set("key", apiKey); + +const data = await fetchGoogleBooks(url); + +for (const book of data.items ?? []) { + if (!isValidBook(book)) { + console.log(`Skipped invalid book: ${book.volumeInfo?.title ?? "Unknown"}`); + continue; + } + + const transformedBook = transformBook(book); + + console.log(JSON.stringify(transformedBook, null, 2)); +} diff --git a/server/src/utils/fetchGoogleBooks.ts b/server/src/utils/fetchGoogleBooks.ts new file mode 100644 index 0000000..29405de --- /dev/null +++ b/server/src/utils/fetchGoogleBooks.ts @@ -0,0 +1,45 @@ +const MAX_RETRIES = 3; + +export async function fetchGoogleBooks(url: URL) { + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + const response = await fetch(url); + + if (response.ok) { + return await response.json(); + } + + const errorBody = await response.text(); + + // Retry temporary server errors and rate limits + const retryable = + response.status === 429 || + response.status === 500 || + response.status === 502 || + response.status === 503 || + response.status === 504; + + if (!retryable || attempt === MAX_RETRIES) { + throw new Error( + `Google Books API error ${response.status}: ${errorBody}` + ); + } + + // Stop if Google explicitly reports an exhausted daily quota + if ( + response.status === 429 && + errorBody.includes("defaultPerDayPerProject") + ) { + throw new Error("Google Books daily quota exhausted."); + } + + const delay = 2000 * 2 ** attempt; + + console.log( + `Google API returned ${response.status}. Retrying in ${delay / 1000}s...` + ); + + await new Promise((resolve) => setTimeout(resolve, delay)); + } + + throw new Error("Google Books request failed."); +} \ No newline at end of file diff --git a/server/src/utils/genreMapping.ts b/server/src/utils/genreMapping.ts new file mode 100644 index 0000000..60557fd --- /dev/null +++ b/server/src/utils/genreMapping.ts @@ -0,0 +1,19 @@ +export const GENRES = [ + "Fantasy", + "Science Fiction", + "Romance", + "Mystery & Crime", + "Horror", + "Adventure", + "Historical Fiction", + "Contemporary Fiction", + "Young Adult", + "Children's Books", + "Biography & Memoir", + "History", + "Science & Technology", + "Self-Help & Psychology", + "Other Nonfiction", +] as const; + +// to be completed later \ No newline at end of file diff --git a/server/src/utils/transformBook.ts b/server/src/utils/transformBook.ts new file mode 100644 index 0000000..086d958 --- /dev/null +++ b/server/src/utils/transformBook.ts @@ -0,0 +1,19 @@ +export function transformBook(book: any) { + const info = book.volumeInfo; + + return { + googleBooksId: book.id, + title: info.title, + subtitle: info.subtitle ?? "", + authors: info.authors ?? [], + publishedYear: info.publishedDate + ? Number(info.publishedDate.slice(0, 4)) || null + : null, + googleCategories: info.categories ?? [], + genres: info.categories ?? [], + description: info.description ?? "", + coverURL: info.imageLinks?.thumbnail ?? "", + pageCount: info.pageCount ?? null, + language: info.language ?? "", + }; +} \ No newline at end of file diff --git a/server/src/utils/validateBook.ts b/server/src/utils/validateBook.ts new file mode 100644 index 0000000..c534c62 --- /dev/null +++ b/server/src/utils/validateBook.ts @@ -0,0 +1,15 @@ +export function isValidBook(book: any): boolean { + const info = book.volumeInfo; + + return Boolean( + typeof book.id === "string" && + book.id.trim() && + typeof info?.title === "string" && + info.title.trim() && + Array.isArray(info.authors) && + info.authors.some( + (author: unknown) => + typeof author === "string" && author.trim().length > 0 + ) + ); +} \ No newline at end of file