diff --git a/server/src/config/importConfig.ts b/server/src/config/importConfig.ts new file mode 100644 index 0000000..4f38e35 --- /dev/null +++ b/server/src/config/importConfig.ts @@ -0,0 +1,29 @@ +//For editing what queriers to search for +export const IMPORT_CONFIG = { + // Google Books API settings + language: "en", + printType: "books", + queryType: "general", + + // Import limits + booksPerRequest: 20, + maxPagesPerSubject: 4, + maxRequestsPerRun: 20, + requestDelayMs: 1000, + + // Subjects to search for + subjects: [ + "fantasy", + "science fiction", + "mystery", + "history", + "biography", + "romance", + "horror", + "thriller", + "philosophy", + "psychology", + "technology", + "historical fiction", + ], +}; diff --git a/server/src/graphql/resolvers.ts b/server/src/graphql/resolvers.ts index a1acdef..28ddc64 100644 --- a/server/src/graphql/resolvers.ts +++ b/server/src/graphql/resolvers.ts @@ -8,7 +8,7 @@ import { listGenres, type BookRecord, type BookSort, -} from '../services/books.js'; +} from '../services/bookService.js'; import { addToShelf, getShelf, @@ -17,7 +17,7 @@ import { withEntry, withMyEntries, type ShelfSort, -} from '../services/shelf.js'; +} from '../services/shelfService.js'; import { USER_ID_HEADER, type Context } from './context.js'; /* diff --git a/server/src/models/ImportProgress.ts b/server/src/models/ImportProgress.ts new file mode 100644 index 0000000..6559eda --- /dev/null +++ b/server/src/models/ImportProgress.ts @@ -0,0 +1,36 @@ + +import mongoose from "mongoose"; + +const importProgressSchema = new mongoose.Schema( + { + queryKey: { + type: String, + required: true, + unique: true, + }, + + nextStartIndex: { + type: Number, + default: 0, + min: 0, + }, + + completed: { + type: Boolean, + default: false, + }, + + lastImportedAt: { + type: Date, + default: null, + }, + }, + { + timestamps: true, + } +); + +export const ImportProgress = mongoose.model( + "ImportProgress", + importProgressSchema +); diff --git a/server/src/scripts/importBooks.ts b/server/src/scripts/importBooks.ts index 78aff0e..1bb078c 100644 --- a/server/src/scripts/importBooks.ts +++ b/server/src/scripts/importBooks.ts @@ -1,93 +1,305 @@ - +//Main import function for bulk library import mongoose from "mongoose"; import { connectDatabase } from "../database/database.js"; import { Book } from "../models/Book.js"; +import { ImportProgress } from "../models/ImportProgress.js"; import { isValidBook } from "../utils/validateBook.js"; import { transformBook } from "../utils/transformBook.js"; -import { fetchGoogleBooks } from "../utils/fetchGoogleBooks.js"; +import { + fetchGoogleBooks, + GoogleBooksApiError, +} from "../utils/fetchGoogleBooks.js"; +import { IMPORT_CONFIG } from "../config/importConfig.js"; -const SUBJECTS = [ - "fantasy", - "science_fiction", - "mystery", - "history", - "biography", -]; +const { + subjects: SUBJECTS, + booksPerRequest: BOOKS_PER_REQUEST, + maxPagesPerSubject: MAX_PAGES_PER_SUBJECT, + maxRequestsPerRun: MAX_REQUESTS_PER_RUN, + requestDelayMs: REQUEST_DELAY_MS, +} = IMPORT_CONFIG; -const BOOKS_PER_SUBJECT = 20; +const sleep = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)); -async function importBooks() { - let fetched = 0; - let invalid = 0; - let inserted = 0; - let updated = 0; +class RequestBudgetExceeded extends Error { + constructor() { + super("Maximum API request limit reached"); + this.name = "RequestBudgetExceeded"; + } +} - try { - const apiKey = process.env.GOOGLE_BOOKS_API_KEY; +async function importBooks() { + let fetched = 0; + let invalid = 0; + let nonEnglish = 0; + let inserted = 0; + let updated = 0; + let unchanged = 0; + let apiRequests = 0; + let failedPages = 0; + let successfulPages = 0; + let apiErrors = 0; - if (!apiKey) { - throw new Error("Missing GOOGLE_BOOKS_API_KEY"); - } + const countRequest = () => { + if (apiRequests >= MAX_REQUESTS_PER_RUN) { + throw new RequestBudgetExceeded(); + } + apiRequests++; + }; - await connectDatabase(); + // Skip subjects that fail or have no results + // for the rest of this execution. + const skippedSubjects = new Set(); - for (const subject of SUBJECTS) { - console.log(`\nFetching books: ${subject}`); + try { + const apiKey = process.env.GOOGLE_BOOKS_API_KEY; - const url = new URL( - "https://www.googleapis.com/books/v1/volumes" - ); + if (!apiKey) { + throw new Error("Missing GOOGLE_BOOKS_API_KEY"); + } - url.searchParams.set("q", subject.replaceAll("_", " ")); - url.searchParams.set("maxResults", String(BOOKS_PER_SUBJECT)); - url.searchParams.set("startIndex", "0"); - url.searchParams.set("key", apiKey); + await connectDatabase(); - const data = await fetchGoogleBooks(url); + // Initialize progress for EVERY configured subject. + const searches = []; - console.log("Query:", url.searchParams.get("q")); - console.log("Total matching results:", data.totalItems); - console.log("Items returned:", data.items?.length ?? 0); + for (const subject of SUBJECTS) { + const query = + IMPORT_CONFIG.queryType === "subject" + ? `subject:${subject}` + : subject; - const books = data.items ?? []; + const queryKey = JSON.stringify({ + query, + language: IMPORT_CONFIG.language, + printType: IMPORT_CONFIG.printType, + }); - for (const book of books) { - fetched++; + const progress = await ImportProgress.findOneAndUpdate( + { queryKey }, + { + $setOnInsert: { + nextStartIndex: 0, + completed: false, + }, + }, + { upsert: true, returnDocument: "after" } + ); - if (!isValidBook(book)) { - invalid++; - continue; + searches.push({ subject, query, progress }); } - const transformedBook = transformBook(book); - - const result = await Book.updateOne( - { googleBooksId: transformedBook.googleBooksId }, - { $set: transformedBook }, - { upsert: true } + console.log( + `Initialized ${searches.length} search configurations.` ); - if (result.upsertedCount > 0) { - inserted++; - } else if (result.matchedCount > 0) { - updated++; + // Round-robin: one page per subject per round. + for ( + let round = 0; + round < MAX_PAGES_PER_SUBJECT; + round++ + ) { + console.log(`\n===== ROUND ${round + 1} =====`); + + let activeSubjects = 0; + + for (const search of searches) { + const { subject, query, progress } = search; + + if ( + progress.completed || + skippedSubjects.has(subject) + ) { + continue; + } + + activeSubjects++; + + if (apiRequests >= MAX_REQUESTS_PER_RUN) { + throw new RequestBudgetExceeded(); + } + + const startIndex = progress.nextStartIndex; + + const url = new URL( + "https://www.googleapis.com/books/v1/volumes" + ); + + url.searchParams.set("q", query); + url.searchParams.set( + "langRestrict", + IMPORT_CONFIG.language + ); + url.searchParams.set( + "printType", + IMPORT_CONFIG.printType + ); + url.searchParams.set( + "maxResults", + String(BOOKS_PER_REQUEST) + ); + url.searchParams.set( + "startIndex", + String(startIndex) + ); + url.searchParams.set("key", apiKey); + + console.log( + `\n${subject}: fetching from index ${startIndex}` + ); + + // Handle API failures separately from database errors. + let books: any[]; + + try { + const data = await fetchGoogleBooks( + url, + countRequest, + () => { + apiErrors++; + } + ); + + books = data.items ?? []; + } catch (error) { + if (error instanceof RequestBudgetExceeded) { + throw error; + } + + if ( + error instanceof Error && + error.message.includes( + "Google Books daily quota exhausted" + ) + ) { + throw error; + } + + + if (error instanceof GoogleBooksApiError) { + console.warn( + `[API WARNING] ${subject}: HTTP ${error.status} (${error.reason}).` + ); + console.warn( + `Skipping until next run. Progress remains at index ${startIndex}.` + ); + } else { + console.error( + `[API ERROR] ${subject}: ${error instanceof Error ? error.message : String(error) + }` + ); + } + + failedPages++; + skippedSubjects.add(subject); + continue; + + + failedPages++; + skippedSubjects.add(subject); + continue; + } + + console.log(`Received ${books.length} books`); + + if (books.length === 0) { + console.log( + `No results for ${subject}; retry in a future run.` + ); + + skippedSubjects.add(subject); + continue; + } + + // Database errors deliberately stop the entire import. + for (const book of books) { + fetched++; + + if ( + book.volumeInfo?.language !== + IMPORT_CONFIG.language + ) { + nonEnglish++; + continue; + } + + if (!isValidBook(book)) { + invalid++; + continue; + } + + const transformedBook = transformBook(book); + + const result = await Book.updateOne( + { + googleBooksId: + transformedBook.googleBooksId, + }, + { $set: transformedBook }, + { upsert: true } + ); + + if (result.upsertedCount > 0) { + inserted++; + } else if (result.modifiedCount > 0) { + updated++; + } else if (result.matchedCount > 0) { + unchanged++; + } + } + + // Save progress only after all book writes succeed. + progress.nextStartIndex = + startIndex + BOOKS_PER_REQUEST; + progress.lastImportedAt = new Date(); + + await progress.save(); + + successfulPages++; + + console.log( + `Saved ${subject} progress: next index ${progress.nextStartIndex}` + ); + + if (books.length < BOOKS_PER_REQUEST) { + console.log( + "Partial page received; will try next index in a later round." + ); + } + + await sleep(REQUEST_DELAY_MS); + } + + if (activeSubjects === 0) { + console.log("No active searches remaining."); + break; + } + } + } catch (error) { + if (error instanceof RequestBudgetExceeded) { + console.log( + "\nAPI request budget reached. Stopping normally." + ); + } else { + console.error("Import stopped:", error); + process.exitCode = 1; } - } + } finally { + console.log("\n--- Import Summary ---"); + console.log(`API attempts: ${apiRequests}`); + console.log(`Successful pages: ${successfulPages}`); + console.log(`Failed pages: ${failedPages}`); + console.log(`Fetched: ${fetched}`); + console.log(`Non-English skipped: ${nonEnglish}`); + console.log(`Invalid/skipped: ${invalid}`); + console.log(`New books inserted: ${inserted}`); + console.log(`Existing books modified: ${updated}`); + console.log(`Existing books unchanged: ${unchanged}`); + console.log(`API errors: ${apiErrors}`); - console.log(`Processed ${books.length} results`); + await mongoose.disconnect(); } - } catch (error) { - console.error("Import stopped:", error); - } finally { - console.log("\n--- Import Summary ---"); - console.log(`Fetched: ${fetched}`); - console.log(`Invalid/skipped: ${invalid}`); - console.log(`New books inserted: ${inserted}`); - console.log(`Existing books matched: ${updated}`); - console.log(`Total processed: ${fetched}`); - - await mongoose.disconnect(); - } } importBooks(); diff --git a/server/src/scripts/importSampleBooks.ts b/server/src/scripts/importSampleBooks.ts new file mode 100644 index 0000000..63cd238 --- /dev/null +++ b/server/src/scripts/importSampleBooks.ts @@ -0,0 +1,84 @@ +//Sample import for frontend development +import mongoose from "mongoose"; +import { connectDatabase } from "../database/database.js"; +import { Book } from "../models/Book.js"; +import { isValidBook } from "../utils/validateBook.js"; +import { transformBook } from "../utils/transformBook.js"; +import { fetchGoogleBooks } from "../utils/fetchGoogleBooks.js"; +import { IMPORT_CONFIG } from "../config/importConfig.js"; + +// Simple local development settings +const SAMPLE_SUBJECT = "fantasy"; +const SAMPLE_SIZE = 20; + +async function importSampleBooks() { + let inserted = 0; + let updated = 0; + let skipped = 0; + + try { + const apiKey = process.env.GOOGLE_BOOKS_API_KEY; + + if (!apiKey) { + throw new Error("Missing GOOGLE_BOOKS_API_KEY"); + } + + await connectDatabase(); + + const url = new URL( + "https://www.googleapis.com/books/v1/volumes" + ); + + const query = + IMPORT_CONFIG.queryType === "subject" + ? `subject:${SAMPLE_SUBJECT}` + : SAMPLE_SUBJECT; + + url.searchParams.set("q", query); + url.searchParams.set("langRestrict", IMPORT_CONFIG.language); + url.searchParams.set("printType", IMPORT_CONFIG.printType); + url.searchParams.set("maxResults", String(SAMPLE_SIZE)); + url.searchParams.set("startIndex", "0"); + url.searchParams.set("key", apiKey); + + const data = await fetchGoogleBooks(url); + const books = data.items ?? []; + + for (const book of books) { + if ( + book.volumeInfo?.language !== IMPORT_CONFIG.language || + !isValidBook(book) + ) { + skipped++; + continue; + } + + const transformedBook = transformBook(book); + + const result = await Book.updateOne( + { googleBooksId: transformedBook.googleBooksId }, + { $set: transformedBook }, + { upsert: true } + ); + + if (result.upsertedCount > 0) { + inserted++; + } else if (result.matchedCount > 0) { + updated++; + } + } + + console.log("\n--- Sample Import Summary ---"); + console.log(`Books returned: ${books.length}`); + console.log(`New books: ${inserted}`); + console.log(`Existing books matched: ${updated}`); + console.log(`Skipped: ${skipped}`); + } catch (error) { + console.error("Sample import failed:", error); + process.exitCode = 1; + } finally { + await mongoose.disconnect(); + } +} + +importSampleBooks(); diff --git a/server/src/scripts/testBookImport.ts b/server/src/scripts/testBookImport.ts deleted file mode 100644 index 7d2ee80..0000000 --- a/server/src/scripts/testBookImport.ts +++ /dev/null @@ -1,55 +0,0 @@ -import mongoose from "mongoose"; -import { connectDatabase } from "../database/database.js"; -import { Book } from "../models/Book.js"; -import { isValidBook } from "../utils/validateBook.js"; -import { transformBook } from "../utils/transformBook.js"; - -async function testImport() { - try { - await connectDatabase(); - - const apiKey = process.env.GOOGLE_BOOKS_API_KEY; - - if (!apiKey) { - throw new Error("Missing GOOGLE_BOOKS_API_KEY"); - } - - const url = new URL("https://www.googleapis.com/books/v1/volumes"); - - url.searchParams.set("q", "harry potter"); - url.searchParams.set("maxResults", "2"); - url.searchParams.set("key", apiKey); - - const response = await fetch(url); - - if (!response.ok) { - throw new Error(`Google API error: ${response.status}`); - } - - const data = await response.json(); - - for (const book of data.items ?? []) { - if (!isValidBook(book)) { - console.log(`Skipped: ${book.volumeInfo?.title}`); - continue; - } - - const transformedBook = transformBook(book); - - await Book.updateOne( - { googleBooksId: transformedBook.googleBooksId }, - { $set: transformedBook }, - { upsert: true } - ); - - console.log(`Saved: ${transformedBook.title}`); - break; // Only save one valid book for testing - } - } catch (error) { - console.error("Import failed:", error); - } finally { - await mongoose.disconnect(); - } -} - -testImport(); \ No newline at end of file diff --git a/server/src/scripts/testGoogleBooks.ts b/server/src/scripts/testGoogleBooks.ts deleted file mode 100644 index 878d596..0000000 --- a/server/src/scripts/testGoogleBooks.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { isValidBook } from "../utils/validateBook.js"; -import { transformBook } from "../utils/transformBook.js"; -import { fetchGoogleBooks } from "../utils/fetchGoogleBooks.js"; - -const apiKey = process.env.GOOGLE_BOOKS_API_KEY; - -if (!apiKey) { - throw new Error("Missing GOOGLE_BOOKS_API_KEY in .env"); -} - -const url = new URL("https://www.googleapis.com/books/v1/volumes"); - -url.searchParams.set("q", "harry potter"); -url.searchParams.set("maxResults", "5"); -url.searchParams.set("startIndex", "5"); -url.searchParams.set("key", apiKey); - -const data = await fetchGoogleBooks(url); - -for (const book of data.items ?? []) { - if (!isValidBook(book)) { - console.log(`Skipped invalid book: ${book.volumeInfo?.title ?? "Unknown"}`); - continue; - } - - const transformedBook = transformBook(book); - - console.log(JSON.stringify(transformedBook, null, 2)); -} diff --git a/server/src/services/books.ts b/server/src/services/bookService.ts similarity index 100% rename from server/src/services/books.ts rename to server/src/services/bookService.ts diff --git a/server/src/services/shelf.ts b/server/src/services/shelfService.ts similarity index 98% rename from server/src/services/shelf.ts rename to server/src/services/shelfService.ts index e636e27..ca711a4 100644 --- a/server/src/services/shelf.ts +++ b/server/src/services/shelfService.ts @@ -1,6 +1,6 @@ import type { InferSchemaType } from 'mongoose'; import { ShelfEntry } from '../models/ShelfEntry.js'; -import { findBooksByIds, type BookRecord } from './books.js'; +import { findBooksByIds, type BookRecord } from './bookService.js'; type ShelfEntryRecord = InferSchemaType; diff --git a/server/src/utils/fetchGoogleBooks.ts b/server/src/utils/fetchGoogleBooks.ts index 29405de..ef8d734 100644 --- a/server/src/utils/fetchGoogleBooks.ts +++ b/server/src/utils/fetchGoogleBooks.ts @@ -1,35 +1,62 @@ + const MAX_RETRIES = 3; -export async function fetchGoogleBooks(url: URL) { + +export class GoogleBooksApiError extends Error { + constructor( + public readonly status: number, + public readonly reason: string + ) { + super(`Google Books API returned HTTP ${status}`); + this.name = "GoogleBooksApiError"; + } +} + + +export async function fetchGoogleBooks( + url: URL, + onAttempt: () => void = () => { }, + onError: () => void = () => { } +) { for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { - const response = await fetch(url); + // Called before EVERY request, including retries. + // Throws if the import has exhausted its request budget. + onAttempt(); + + const response = await fetch(url, { + signal: AbortSignal.timeout(15000), + }); if (response.ok) { return await response.json(); } const errorBody = await response.text(); + onError(); + + // Stop if Google explicitly reports an exhausted daily quota. + if ( + (response.status === 429 || response.status === 403) && + errorBody.includes("defaultPerDayPerProject") + ) { + throw new Error("Google Books daily quota exhausted."); + } - // Retry temporary server errors and rate limits const retryable = response.status === 429 || - response.status === 500 || - response.status === 502 || - response.status === 503 || - response.status === 504; + [500, 502, 503, 504].includes(response.status); if (!retryable || attempt === MAX_RETRIES) { - throw new Error( - `Google Books API error ${response.status}: ${errorBody}` - ); - } + let reason = "unknown"; - // Stop if Google explicitly reports an exhausted daily quota - if ( - response.status === 429 && - errorBody.includes("defaultPerDayPerProject") - ) { - throw new Error("Google Books daily quota exhausted."); + try { + const parsed = JSON.parse(errorBody); + reason = parsed.error?.errors?.[0]?.reason ?? "unknown"; + } catch { + // Keep the fallback reason if Google returns non-JSON. + } + + throw new GoogleBooksApiError(response.status, reason); } const delay = 2000 * 2 ** attempt; @@ -42,4 +69,4 @@ export async function fetchGoogleBooks(url: URL) { } throw new Error("Google Books request failed."); -} \ No newline at end of file +}