From e8701a44962b6b1c5a9ea48b1193e8dc263f9d19 Mon Sep 17 00:00:00 2001 From: Robert Andreas Kyllo Date: Thu, 15 Oct 2026 20:50:53 +0300 Subject: [PATCH 1/2] feat: #7 add batched large catalog seed --- README.md | 42 ++++++++++++++++ docs/AI-USAGE.md | 1 + package.json | 5 +- server/package.json | 3 +- server/src/db/largeCatalog.test.ts | 34 +++++++++++++ server/src/db/largeCatalog.ts | 63 ++++++++++++++++++++++++ server/src/db/seedLarge.ts | 78 ++++++++++++++++++++++++++++++ 7 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 server/src/db/largeCatalog.test.ts create mode 100644 server/src/db/largeCatalog.ts create mode 100644 server/src/db/seedLarge.ts diff --git a/README.md b/README.md index 1fc3507..9944348 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,48 @@ All four must pass. CI runs the same thing. | `pnpm format` / `pnpm format:check` | Prettier | | `pnpm typecheck` | `tsc --noEmit` in every workspace | | `pnpm test` | Vitest, single run | +| `pnpm db:migrate` | Apply PostgreSQL migrations | +| `pnpm db:seed` | Reset to the small sample catalog | +| `pnpm db:seed:large` | Reset to a synthetic large catalog | + +## Database seed data + +Set `DATABASE_URL` in `server/.env`, then apply migrations before seeding: + +```bash +pnpm db:migrate +pnpm db:seed +``` + +`db:seed` replaces the catalog with 16 curated albums for normal development. +For scalability testing, `db:seed:large` generates 100,000 deterministic +synthetic albums in batches of 1,000. Set `ALBUM_COUNT` to change the size: + +```bash +ALBUM_COUNT=100000 pnpm db:seed:large +``` + +Both seed commands reset the catalog and delete associated reviews and crate +items. They are development/reset tools, never commands to run against user +data you intend to keep. + +### Scalability check + +The large seed is deterministic, so the same command can be used to repeat a +performance check. On 15 October 2026, using a local PostgreSQL 16 container +with 100,000 generated albums, it completed in **8.59 s**. Representative +`EXPLAIN (ANALYZE, BUFFERS)` results were: + +| Operation | Execution time | +| --------------------------------------------------------------------- | -------------: | +| Full-text search for an artist, ordered by relevance | 2.26 ms | +| Two-genre AND filter plus two decades, ordered by title | 22.65 ms | +| Complete `searchAlbums` GraphQL query with filters, facets and paging | 62.94 ms | + +The first two use the production search queries against all 100,000 rows; the +last timing is the HTTP response for the GraphQL resolver. All are below one +second on the local test database. Repeat this check on the VM after deploying +the backend, because its hardware and PostgreSQL version may differ. ## Project status diff --git a/docs/AI-USAGE.md b/docs/AI-USAGE.md index 565badd..90a2657 100644 --- a/docs/AI-USAGE.md +++ b/docs/AI-USAGE.md @@ -38,6 +38,7 @@ the part that matters most. | 2026-10-10 | evenkkl | Claude | Assisted with implementing the GraphQL Yoga server with `searchAlbums` for #6, and formulated an easy to understand PR description. Guided the reset of the prototype tables in the VM database (backup taken first). | | 2026-10-13 | evenkkl | Claude | Fixed the review feedback on #6: cursor values validated against the sort, `reviewCount` returns 0 until reviews exist. The group decided genre filters combine with AND; Claude made the filter and the genre counts match that. | | 2026-10-09 | robertky | Codex (ChatGPT) | AI was a good sparring partner when I drafted the initial #5 PostgreSQL migration. It gave me ideas on how to structure and improve the tables, relation tables, constraints, indexes, full-text search triggers and TypeScript migration runner. | +| 2026-10-15 | robertky | Codex (ChatGPT) | Assisted me when I designed and implemented #7's batched synthetic catalog generator, its tests and local performance verification. I reviewed the generated data, when AI suggested something before using it. | ## Notes diff --git a/package.json b/package.json index e23779d..8908434 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,10 @@ "format:check": "prettier --check .", "typecheck": "pnpm -r typecheck", "test": "pnpm -r test", - "test:watch": "pnpm --filter @t19/client test:watch" + "test:watch": "pnpm --filter @t19/client test:watch", + "db:migrate": "pnpm --filter @t19/server db:migrate", + "db:seed": "pnpm --filter @t19/server db:seed", + "db:seed:large": "pnpm --filter @t19/server db:seed:large" }, "devDependencies": { "@eslint/js": "^9.39.5", diff --git a/server/package.json b/server/package.json index 2526c92..f5081c6 100644 --- a/server/package.json +++ b/server/package.json @@ -11,7 +11,8 @@ "test": "vitest run", "db:generate": "drizzle-kit generate", "db:migrate": "tsx --env-file-if-exists=.env src/db/migrate.ts", - "db:seed": "tsx --env-file-if-exists=.env src/db/seed.ts" + "db:seed": "tsx --env-file-if-exists=.env src/db/seed.ts", + "db:seed:large": "tsx --env-file-if-exists=.env src/db/seedLarge.ts" }, "dependencies": { "dataloader": "^2.2.3", diff --git a/server/src/db/largeCatalog.test.ts b/server/src/db/largeCatalog.test.ts new file mode 100644 index 0000000..9081a93 --- /dev/null +++ b/server/src/db/largeCatalog.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest'; +import { + createLargeCatalogAlbum, + largeCatalogArtistNames, + largeCatalogGenreNames, + LARGE_SEED_DEFAULT_COUNT, + parseAlbumCount, +} from './largeCatalog.js'; + +describe('large catalog generator', () => { + it('uses 100,000 albums unless ALBUM_COUNT overrides it', () => { + expect(parseAlbumCount(undefined)).toBe(LARGE_SEED_DEFAULT_COUNT); + expect(parseAlbumCount('1000000')).toBe(1_000_000); + }); + + it.each(['0', '-1', '1.5', 'many'])('rejects an invalid album count: %s', (value) => { + expect(() => parseAlbumCount(value)).toThrow( + 'ALBUM_COUNT must be a positive whole number', + ); + }); + + it('produces deterministic albums with realistic search and filter variation', () => { + expect(createLargeCatalogAlbum(0)).toEqual({ + title: 'Synthetic Album 1', + releaseYear: 1950, + artistName: 'Synthetic Artist 1', + genreNames: ['Alternative'], + }); + expect(createLargeCatalogAlbum(2).genreNames).toHaveLength(2); + expect(createLargeCatalogAlbum(46).releaseYear).toBeNull(); + expect(largeCatalogArtistNames()).toHaveLength(5000); + expect(largeCatalogGenreNames()).toHaveLength(16); + }); +}); diff --git a/server/src/db/largeCatalog.ts b/server/src/db/largeCatalog.ts new file mode 100644 index 0000000..d55e1fe --- /dev/null +++ b/server/src/db/largeCatalog.ts @@ -0,0 +1,63 @@ +export const LARGE_SEED_BATCH_SIZE = 1000; +export const LARGE_SEED_DEFAULT_COUNT = 100_000; + +const ARTIST_COUNT = 5000; +const genreNames = [ + 'Alternative', + 'Ambient', + 'Classical', + 'Electronic', + 'Folk', + 'Hip hop', + 'Indie', + 'Jazz', + 'Metal', + 'Pop', + 'Punk', + 'R&B', + 'Rock', + 'Soul', + 'Techno', + 'World', +] as const; + +export type LargeCatalogAlbum = { + title: string; + releaseYear: number | null; + artistName: string; + genreNames: readonly string[]; +}; + +export function parseAlbumCount(value: string | undefined): number { + if (value === undefined) return LARGE_SEED_DEFAULT_COUNT; + + const count = Number(value); + if (!Number.isSafeInteger(count) || count < 1) { + throw new Error('ALBUM_COUNT must be a positive whole number'); + } + return count; +} + +export function largeCatalogArtistNames(): string[] { + return Array.from( + { length: ARTIST_COUNT }, + (_, index) => `Synthetic Artist ${index + 1}`, + ); +} + +export function largeCatalogGenreNames(): readonly string[] { + return genreNames; +} + +export function createLargeCatalogAlbum(index: number): LargeCatalogAlbum { + const ordinal = index + 1; + const primaryGenre = genreNames[index % genreNames.length]!; + const secondaryGenre = genreNames[(index + 3) % genreNames.length]!; + + return { + title: `Synthetic Album ${ordinal}`, + releaseYear: ordinal % 47 === 0 ? null : 1950 + (index % 76), + artistName: `Synthetic Artist ${(index % ARTIST_COUNT) + 1}`, + genreNames: ordinal % 3 === 0 ? [primaryGenre, secondaryGenre] : [primaryGenre], + }; +} diff --git a/server/src/db/seedLarge.ts b/server/src/db/seedLarge.ts new file mode 100644 index 0000000..601f8d1 --- /dev/null +++ b/server/src/db/seedLarge.ts @@ -0,0 +1,78 @@ +import { sql } from 'drizzle-orm'; +import { db, pool } from './client.js'; +import { + createLargeCatalogAlbum, + largeCatalogArtistNames, + largeCatalogGenreNames, + LARGE_SEED_BATCH_SIZE, + parseAlbumCount, +} from './largeCatalog.js'; +import { album, albumArtist, albumGenre, artist, genre } from './schema.js'; + +const albumCount = parseAlbumCount(process.env.ALBUM_COUNT); + +try { + const { artistIds, genreIds } = await db.transaction(async (tx) => { + await tx.execute(sql`TRUNCATE album, artist, genre RESTART IDENTITY CASCADE`); + + const artistRows = await tx + .insert(artist) + .values(largeCatalogArtistNames().map((name) => ({ name }))) + .returning(); + const genreRows = await tx + .insert(genre) + .values(largeCatalogGenreNames().map((name) => ({ name }))) + .returning(); + + return { + artistIds: new Map(artistRows.map((row) => [row.name, row.id])), + genreIds: new Map(genreRows.map((row) => [row.name, row.id])), + }; + }); + + for (let offset = 0; offset < albumCount; offset += LARGE_SEED_BATCH_SIZE) { + const count = Math.min(LARGE_SEED_BATCH_SIZE, albumCount - offset); + const catalog = Array.from({ length: count }, (_, index) => + createLargeCatalogAlbum(offset + index), + ); + + await db.transaction(async (tx) => { + const insertedAlbums = await tx + .insert(album) + .values(catalog.map(({ title, releaseYear }) => ({ title, releaseYear }))) + .returning({ id: album.id }); + + const albumArtists = catalog.map((entry, index) => { + const albumId = insertedAlbums[index]?.id; + const artistId = artistIds.get(entry.artistName); + if (albumId === undefined || artistId === undefined) { + throw new Error('Large catalog seed lost an album or artist id'); + } + return { albumId, artistId }; + }); + const albumGenres = catalog.flatMap((entry, index) => { + const albumId = insertedAlbums[index]?.id; + if (albumId === undefined) throw new Error('Large catalog seed lost an album id'); + return entry.genreNames.map((name) => { + const genreId = genreIds.get(name); + if (genreId === undefined) throw new Error(`Unknown generated genre: ${name}`); + return { albumId, genreId }; + }); + }); + + await tx.insert(albumArtist).values(albumArtists); + await tx.insert(albumGenre).values(albumGenres); + }); + + const inserted = Math.min(offset + count, albumCount); + if (inserted === albumCount || inserted % 10_000 === 0) { + console.log( + `Seeded ${inserted.toLocaleString()} of ${albumCount.toLocaleString()} albums.`, + ); + } + } +} finally { + await pool.end(); +} + +console.log(`Large seed complete: ${albumCount.toLocaleString()} synthetic albums.`); From f64136beac48cd84bd984798793e2af008491313 Mon Sep 17 00:00:00 2001 From: Robert Andreas Kyllo Date: Fri, 16 Oct 2026 18:10:14 +0300 Subject: [PATCH 2/2] docs: fix benchmark command line continuations --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 414de8a..7b17ea2 100644 --- a/README.md +++ b/README.md @@ -104,9 +104,9 @@ the API, for example: ```bash psql "$DATABASE_URL" -f server/scripts/perf-check.sql -curl -s -o /dev/null -w 'total %{time_total}s\\n' \\ - -H 'content-type: application/json' \\ - --data @server/scripts/search-albums-perf.json \\ +curl -s -o /dev/null -w 'total %{time_total}s\n' \ + -H 'content-type: application/json' \ + --data @server/scripts/search-albums-perf.json \ http://localhost:3001/graphql ```