diff --git a/server/package.json b/server/package.json index 4fcbb98..f08aa5f 100644 --- a/server/package.json +++ b/server/package.json @@ -5,7 +5,6 @@ "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "dev": "tsx watch --env-file=.env src/index.ts", - "import:mock": "tsx --env-file=.env src/scripts/importMockBooks.ts", "build": "tsc", "start": "node dist/index.js", "format": "prettier --write .", diff --git a/server/src/models/Book.ts b/server/src/models/Book.ts index 307b112..737dab0 100644 --- a/server/src/models/Book.ts +++ b/server/src/models/Book.ts @@ -27,6 +27,11 @@ const bookSchema = new mongoose.Schema({ default: null, }, + googleCategories: { + type: [String], + default: [], + }, + genres: { type: [String], default: [], diff --git a/server/src/scripts/importBooks.ts b/server/src/scripts/importBooks.ts new file mode 100644 index 0000000..78aff0e --- /dev/null +++ b/server/src/scripts/importBooks.ts @@ -0,0 +1,93 @@ + +import mongoose from "mongoose"; +import { connectDatabase } from "../database/database.js"; +import { Book } from "../models/Book.js"; +import { isValidBook } from "../utils/validateBook.js"; +import { transformBook } from "../utils/transformBook.js"; +import { fetchGoogleBooks } from "../utils/fetchGoogleBooks.js"; + +const SUBJECTS = [ + "fantasy", + "science_fiction", + "mystery", + "history", + "biography", +]; + +const BOOKS_PER_SUBJECT = 20; + +async function importBooks() { + let fetched = 0; + let invalid = 0; + let inserted = 0; + let updated = 0; + + try { + const apiKey = process.env.GOOGLE_BOOKS_API_KEY; + + if (!apiKey) { + throw new Error("Missing GOOGLE_BOOKS_API_KEY"); + } + + await connectDatabase(); + + for (const subject of SUBJECTS) { + console.log(`\nFetching books: ${subject}`); + + const url = new URL( + "https://www.googleapis.com/books/v1/volumes" + ); + + url.searchParams.set("q", subject.replaceAll("_", " ")); + url.searchParams.set("maxResults", String(BOOKS_PER_SUBJECT)); + url.searchParams.set("startIndex", "0"); + url.searchParams.set("key", apiKey); + + const data = await fetchGoogleBooks(url); + + console.log("Query:", url.searchParams.get("q")); + console.log("Total matching results:", data.totalItems); + console.log("Items returned:", data.items?.length ?? 0); + + const books = data.items ?? []; + + for (const book of books) { + fetched++; + + if (!isValidBook(book)) { + invalid++; + continue; + } + + const transformedBook = transformBook(book); + + const result = await Book.updateOne( + { googleBooksId: transformedBook.googleBooksId }, + { $set: transformedBook }, + { upsert: true } + ); + + if (result.upsertedCount > 0) { + inserted++; + } else if (result.matchedCount > 0) { + updated++; + } + } + + console.log(`Processed ${books.length} results`); + } + } catch (error) { + console.error("Import stopped:", error); + } finally { + console.log("\n--- Import Summary ---"); + console.log(`Fetched: ${fetched}`); + console.log(`Invalid/skipped: ${invalid}`); + console.log(`New books inserted: ${inserted}`); + console.log(`Existing books matched: ${updated}`); + console.log(`Total processed: ${fetched}`); + + await mongoose.disconnect(); + } +} + +importBooks(); diff --git a/server/src/scripts/importMockBooks.ts b/server/src/scripts/importMockBooks.ts deleted file mode 100644 index e0ce859..0000000 --- a/server/src/scripts/importMockBooks.ts +++ /dev/null @@ -1,21 +0,0 @@ -import fs from "node:fs"; -import { connectDatabase } from "../database/database.js"; -import { Book } from "../models/Book.js"; - -async function importMockBooks() { - await connectDatabase(); - - const file = fs.readFileSync("./test-data/books.json", "utf-8"); - const books = JSON.parse(file); - - // Clear existing books before importing mock data - await Book.deleteMany({}); - - await Book.insertMany(books); - - console.log(`Imported ${books.length} books`); - - process.exit(0); -} - -importMockBooks(); \ No newline at end of file diff --git a/server/src/scripts/testBookImport.ts b/server/src/scripts/testBookImport.ts new file mode 100644 index 0000000..7d2ee80 --- /dev/null +++ b/server/src/scripts/testBookImport.ts @@ -0,0 +1,55 @@ +import mongoose from "mongoose"; +import { connectDatabase } from "../database/database.js"; +import { Book } from "../models/Book.js"; +import { isValidBook } from "../utils/validateBook.js"; +import { transformBook } from "../utils/transformBook.js"; + +async function testImport() { + try { + await connectDatabase(); + + const apiKey = process.env.GOOGLE_BOOKS_API_KEY; + + if (!apiKey) { + throw new Error("Missing GOOGLE_BOOKS_API_KEY"); + } + + const url = new URL("https://www.googleapis.com/books/v1/volumes"); + + url.searchParams.set("q", "harry potter"); + url.searchParams.set("maxResults", "2"); + url.searchParams.set("key", apiKey); + + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Google API error: ${response.status}`); + } + + const data = await response.json(); + + for (const book of data.items ?? []) { + if (!isValidBook(book)) { + console.log(`Skipped: ${book.volumeInfo?.title}`); + continue; + } + + const transformedBook = transformBook(book); + + await Book.updateOne( + { googleBooksId: transformedBook.googleBooksId }, + { $set: transformedBook }, + { upsert: true } + ); + + console.log(`Saved: ${transformedBook.title}`); + break; // Only save one valid book for testing + } + } catch (error) { + console.error("Import failed:", error); + } finally { + await mongoose.disconnect(); + } +} + +testImport(); \ No newline at end of file diff --git a/server/src/scripts/testGoogleBooks.ts b/server/src/scripts/testGoogleBooks.ts new file mode 100644 index 0000000..878d596 --- /dev/null +++ b/server/src/scripts/testGoogleBooks.ts @@ -0,0 +1,29 @@ +import { isValidBook } from "../utils/validateBook.js"; +import { transformBook } from "../utils/transformBook.js"; +import { fetchGoogleBooks } from "../utils/fetchGoogleBooks.js"; + +const apiKey = process.env.GOOGLE_BOOKS_API_KEY; + +if (!apiKey) { + throw new Error("Missing GOOGLE_BOOKS_API_KEY in .env"); +} + +const url = new URL("https://www.googleapis.com/books/v1/volumes"); + +url.searchParams.set("q", "harry potter"); +url.searchParams.set("maxResults", "5"); +url.searchParams.set("startIndex", "5"); +url.searchParams.set("key", apiKey); + +const data = await fetchGoogleBooks(url); + +for (const book of data.items ?? []) { + if (!isValidBook(book)) { + console.log(`Skipped invalid book: ${book.volumeInfo?.title ?? "Unknown"}`); + continue; + } + + const transformedBook = transformBook(book); + + console.log(JSON.stringify(transformedBook, null, 2)); +} diff --git a/server/src/utils/fetchGoogleBooks.ts b/server/src/utils/fetchGoogleBooks.ts new file mode 100644 index 0000000..29405de --- /dev/null +++ b/server/src/utils/fetchGoogleBooks.ts @@ -0,0 +1,45 @@ +const MAX_RETRIES = 3; + +export async function fetchGoogleBooks(url: URL) { + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + const response = await fetch(url); + + if (response.ok) { + return await response.json(); + } + + const errorBody = await response.text(); + + // Retry temporary server errors and rate limits + const retryable = + response.status === 429 || + response.status === 500 || + response.status === 502 || + response.status === 503 || + response.status === 504; + + if (!retryable || attempt === MAX_RETRIES) { + throw new Error( + `Google Books API error ${response.status}: ${errorBody}` + ); + } + + // Stop if Google explicitly reports an exhausted daily quota + if ( + response.status === 429 && + errorBody.includes("defaultPerDayPerProject") + ) { + throw new Error("Google Books daily quota exhausted."); + } + + const delay = 2000 * 2 ** attempt; + + console.log( + `Google API returned ${response.status}. Retrying in ${delay / 1000}s...` + ); + + await new Promise((resolve) => setTimeout(resolve, delay)); + } + + throw new Error("Google Books request failed."); +} \ No newline at end of file diff --git a/server/src/utils/genreMapping.ts b/server/src/utils/genreMapping.ts new file mode 100644 index 0000000..60557fd --- /dev/null +++ b/server/src/utils/genreMapping.ts @@ -0,0 +1,19 @@ +export const GENRES = [ + "Fantasy", + "Science Fiction", + "Romance", + "Mystery & Crime", + "Horror", + "Adventure", + "Historical Fiction", + "Contemporary Fiction", + "Young Adult", + "Children's Books", + "Biography & Memoir", + "History", + "Science & Technology", + "Self-Help & Psychology", + "Other Nonfiction", +] as const; + +// to be completed later \ No newline at end of file diff --git a/server/src/utils/transformBook.ts b/server/src/utils/transformBook.ts new file mode 100644 index 0000000..086d958 --- /dev/null +++ b/server/src/utils/transformBook.ts @@ -0,0 +1,19 @@ +export function transformBook(book: any) { + const info = book.volumeInfo; + + return { + googleBooksId: book.id, + title: info.title, + subtitle: info.subtitle ?? "", + authors: info.authors ?? [], + publishedYear: info.publishedDate + ? Number(info.publishedDate.slice(0, 4)) || null + : null, + googleCategories: info.categories ?? [], + genres: info.categories ?? [], + description: info.description ?? "", + coverURL: info.imageLinks?.thumbnail ?? "", + pageCount: info.pageCount ?? null, + language: info.language ?? "", + }; +} \ No newline at end of file diff --git a/server/src/utils/validateBook.ts b/server/src/utils/validateBook.ts new file mode 100644 index 0000000..c534c62 --- /dev/null +++ b/server/src/utils/validateBook.ts @@ -0,0 +1,15 @@ +export function isValidBook(book: any): boolean { + const info = book.volumeInfo; + + return Boolean( + typeof book.id === "string" && + book.id.trim() && + typeof info?.title === "string" && + info.title.trim() && + Array.isArray(info.authors) && + info.authors.some( + (author: unknown) => + typeof author === "string" && author.trim().length > 0 + ) + ); +} \ No newline at end of file