diff --git a/README.md b/README.md index aff8274..7b17ea2 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,58 @@ All four must pass. CI runs the same thing. | `pnpm --filter @t19/server dev` | GraphQL API on `:3001/graphql` | | `pnpm --filter @t19/server db:migrate` | Apply the Drizzle migrations | | `pnpm --filter @t19/server db:seed` | Load the sample catalog | +| `pnpm db:seed:large` | Reset to a synthetic large catalog | + +## Database seed data + +`db:seed` replaces the catalog with 16 curated albums for normal development. +For scalability testing, `db:seed:large` generates 100,000 deterministic +synthetic albums in batches of 1,000. Set `ALBUM_COUNT` to change the size: + +```bash +ALBUM_COUNT=100000 pnpm db:seed:large +``` + +PowerShell: + +```powershell +$env:ALBUM_COUNT=100000; pnpm db:seed:large +``` + +Both seed commands reset the catalog and delete associated reviews and crate +items. They are development/reset tools, never commands to run against user +data you intend to keep. Each batch commits separately; if large seeding is +interrupted, rerun the command to reset and regenerate the catalog. + +### Scalability check + +The large seed is deterministic, so the same commands can repeat a performance +check. Run the committed [SQL checks](server/scripts/perf-check.sql), then send +the committed [GraphQL request body](server/scripts/search-albums-perf.json) to +the API, for example: + +```bash +psql "$DATABASE_URL" -f server/scripts/perf-check.sql +curl -s -o /dev/null -w 'total %{time_total}s\n' \ + -H 'content-type: application/json' \ + --data @server/scripts/search-albums-perf.json \ + http://localhost:3001/graphql +``` + +On 16 October 2026, using a local PostgreSQL 16 container with 100,000 +generated albums, seeding completed in **8.42 s**. Representative timings were: + +| Operation | Execution time | +| ------------------------------------------------------------------ | -------------: | +| Full-text search for an artist, ordered by relevance | 2.90 ms | +| Broad full-text search for `album`, ordered by relevance | 41.36 ms | +| Two-genre AND filter plus two decades, ordered by title | 19.57 ms | +| Complete broad `searchAlbums` GraphQL query with facets and paging | 34.65 ms | + +The first three use the production search queries against all 100,000 rows; +the last timing is the HTTP response for the GraphQL resolver. Repeat this +check on the VM after deploying the backend, because its hardware and +PostgreSQL version may differ. ## Project status diff --git a/docs/AI-USAGE.md b/docs/AI-USAGE.md index 88b7c18..97f3658 100644 --- a/docs/AI-USAGE.md +++ b/docs/AI-USAGE.md @@ -51,6 +51,7 @@ the part that matters most. | 2026-10-15 | rachelks | Codex (ChatGPT) | Assisted with reviewing M2 documentation for #48, identifying outdated setup instructions and missing frontend information. | | 2026-10-15 | erikhfj | opencode | Documentation pass for #48: this log and the README brought up to date with what has actually landed | | 2026-10-15 | erikhfj | opencode | Folded #46 into the README, and added `db:migrate` to the deploy on #45 after #46 pointed out it never ran: a deploy that lands migration files without applying them leaves the schema behind the code. Also reconciled `DATABASE_URL`, which the systemd unit reads from the checkout root and the server package scripts read from `server/.env` | +| 2026-10-15 | robertky | Codex (ChatGPT) | Assisted me when I designed and implemented #7's batched synthetic catalog generator, its tests and local performance verification. I reviewed the generated data, when AI suggested something before using it. | | 2026-10-16 | rachelks | Codex (ChatGPT) | Assisted with updating README and VM setup documentation for #48, clarifying implemented functionality and local development instructions. | | 2026-10-16 | evenkkl | Claude | Assisted with reviewing PRs #50 and #51, checking the changes against the code and identifying areas for improvement. | diff --git a/package.json b/package.json index e23779d..8908434 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,10 @@ "format:check": "prettier --check .", "typecheck": "pnpm -r typecheck", "test": "pnpm -r test", - "test:watch": "pnpm --filter @t19/client test:watch" + "test:watch": "pnpm --filter @t19/client test:watch", + "db:migrate": "pnpm --filter @t19/server db:migrate", + "db:seed": "pnpm --filter @t19/server db:seed", + "db:seed:large": "pnpm --filter @t19/server db:seed:large" }, "devDependencies": { "@eslint/js": "^9.39.5", diff --git a/server/package.json b/server/package.json index 2526c92..f5081c6 100644 --- a/server/package.json +++ b/server/package.json @@ -11,7 +11,8 @@ "test": "vitest run", "db:generate": "drizzle-kit generate", "db:migrate": "tsx --env-file-if-exists=.env src/db/migrate.ts", - "db:seed": "tsx --env-file-if-exists=.env src/db/seed.ts" + "db:seed": "tsx --env-file-if-exists=.env src/db/seed.ts", + "db:seed:large": "tsx --env-file-if-exists=.env src/db/seedLarge.ts" }, "dependencies": { "dataloader": "^2.2.3", diff --git a/server/scripts/perf-check.sql b/server/scripts/perf-check.sql new file mode 100644 index 0000000..bd1cde0 --- /dev/null +++ b/server/scripts/perf-check.sql @@ -0,0 +1,38 @@ +-- Run after `ALBUM_COUNT=100000 pnpm db:seed:large`. +-- Execute with: psql "$DATABASE_URL" -f server/scripts/perf-check.sql +-- The generated genres are inserted in the listed order: Alternative is 1 and +-- Electronic is 4. The title and decade query mirrors the production filters. + +-- A selective full-text lookup, ordered exactly as RELEVANCE in searchAlbums. +EXPLAIN (ANALYZE, BUFFERS) +SELECT album.id +FROM album +WHERE album.search_vector @@ websearch_to_tsquery('simple', 'Synthetic Artist 1') +ORDER BY ts_rank(album.search_vector, websearch_to_tsquery('simple', 'Synthetic Artist 1')) DESC, + album.id ASC +LIMIT 20; + +-- A deliberately broad full-text lookup: every generated title matches "album". +EXPLAIN (ANALYZE, BUFFERS) +SELECT album.id +FROM album +WHERE album.search_vector @@ websearch_to_tsquery('simple', 'album') +ORDER BY ts_rank(album.search_vector, websearch_to_tsquery('simple', 'album')) DESC, + album.id ASC +LIMIT 20; + +-- Two genres combine with AND; the two selected decades combine with OR. +EXPLAIN (ANALYZE, BUFFERS) +SELECT album.id +FROM album +WHERE EXISTS ( + SELECT 1 FROM album_genre + WHERE album_genre.album_id = album.id AND album_genre.genre_id = 1 +) + AND EXISTS ( + SELECT 1 FROM album_genre + WHERE album_genre.album_id = album.id AND album_genre.genre_id = 4 + ) + AND (album.release_year / 10) * 10 IN (1990, 2000) +ORDER BY lower(album.title) ASC, album.id ASC +LIMIT 20; diff --git a/server/scripts/search-albums-perf.json b/server/scripts/search-albums-perf.json new file mode 100644 index 0000000..e95ba4a --- /dev/null +++ b/server/scripts/search-albums-perf.json @@ -0,0 +1,3 @@ +{ + "query": "query BroadSearch { searchAlbums(input: { query: \"album\", sort: RELEVANCE }, first: 20) { totalCount edges { cursor node { id title releaseYear artists { id name } genres { id name } reviewCount } } pageInfo { hasNextPage endCursor } genreFacets { genre { id name } count } decadeFacets { decade count } } }" +} diff --git a/server/src/db/largeCatalog.test.ts b/server/src/db/largeCatalog.test.ts new file mode 100644 index 0000000..9081a93 --- /dev/null +++ b/server/src/db/largeCatalog.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest'; +import { + createLargeCatalogAlbum, + largeCatalogArtistNames, + largeCatalogGenreNames, + LARGE_SEED_DEFAULT_COUNT, + parseAlbumCount, +} from './largeCatalog.js'; + +describe('large catalog generator', () => { + it('uses 100,000 albums unless ALBUM_COUNT overrides it', () => { + expect(parseAlbumCount(undefined)).toBe(LARGE_SEED_DEFAULT_COUNT); + expect(parseAlbumCount('1000000')).toBe(1_000_000); + }); + + it.each(['0', '-1', '1.5', 'many'])('rejects an invalid album count: %s', (value) => { + expect(() => parseAlbumCount(value)).toThrow( + 'ALBUM_COUNT must be a positive whole number', + ); + }); + + it('produces deterministic albums with realistic search and filter variation', () => { + expect(createLargeCatalogAlbum(0)).toEqual({ + title: 'Synthetic Album 1', + releaseYear: 1950, + artistName: 'Synthetic Artist 1', + genreNames: ['Alternative'], + }); + expect(createLargeCatalogAlbum(2).genreNames).toHaveLength(2); + expect(createLargeCatalogAlbum(46).releaseYear).toBeNull(); + expect(largeCatalogArtistNames()).toHaveLength(5000); + expect(largeCatalogGenreNames()).toHaveLength(16); + }); +}); diff --git a/server/src/db/largeCatalog.ts b/server/src/db/largeCatalog.ts new file mode 100644 index 0000000..d55e1fe --- /dev/null +++ b/server/src/db/largeCatalog.ts @@ -0,0 +1,63 @@ +export const LARGE_SEED_BATCH_SIZE = 1000; +export const LARGE_SEED_DEFAULT_COUNT = 100_000; + +const ARTIST_COUNT = 5000; +const genreNames = [ + 'Alternative', + 'Ambient', + 'Classical', + 'Electronic', + 'Folk', + 'Hip hop', + 'Indie', + 'Jazz', + 'Metal', + 'Pop', + 'Punk', + 'R&B', + 'Rock', + 'Soul', + 'Techno', + 'World', +] as const; + +export type LargeCatalogAlbum = { + title: string; + releaseYear: number | null; + artistName: string; + genreNames: readonly string[]; +}; + +export function parseAlbumCount(value: string | undefined): number { + if (value === undefined) return LARGE_SEED_DEFAULT_COUNT; + + const count = Number(value); + if (!Number.isSafeInteger(count) || count < 1) { + throw new Error('ALBUM_COUNT must be a positive whole number'); + } + return count; +} + +export function largeCatalogArtistNames(): string[] { + return Array.from( + { length: ARTIST_COUNT }, + (_, index) => `Synthetic Artist ${index + 1}`, + ); +} + +export function largeCatalogGenreNames(): readonly string[] { + return genreNames; +} + +export function createLargeCatalogAlbum(index: number): LargeCatalogAlbum { + const ordinal = index + 1; + const primaryGenre = genreNames[index % genreNames.length]!; + const secondaryGenre = genreNames[(index + 3) % genreNames.length]!; + + return { + title: `Synthetic Album ${ordinal}`, + releaseYear: ordinal % 47 === 0 ? null : 1950 + (index % 76), + artistName: `Synthetic Artist ${(index % ARTIST_COUNT) + 1}`, + genreNames: ordinal % 3 === 0 ? [primaryGenre, secondaryGenre] : [primaryGenre], + }; +} diff --git a/server/src/db/seedLarge.ts b/server/src/db/seedLarge.ts new file mode 100644 index 0000000..fe0f495 --- /dev/null +++ b/server/src/db/seedLarge.ts @@ -0,0 +1,79 @@ +import { sql } from 'drizzle-orm'; +import { db, pool } from './client.js'; +import { + createLargeCatalogAlbum, + largeCatalogArtistNames, + largeCatalogGenreNames, + LARGE_SEED_BATCH_SIZE, + parseAlbumCount, +} from './largeCatalog.js'; +import { album, albumArtist, albumGenre, artist, genre } from './schema.js'; + +const albumCount = parseAlbumCount(process.env.ALBUM_COUNT); + +try { + const { artistIds, genreIds } = await db.transaction(async (tx) => { + await tx.execute(sql`TRUNCATE album, artist, genre RESTART IDENTITY CASCADE`); + + const artistRows = await tx + .insert(artist) + .values(largeCatalogArtistNames().map((name) => ({ name }))) + .returning(); + const genreRows = await tx + .insert(genre) + .values(largeCatalogGenreNames().map((name) => ({ name }))) + .returning(); + + return { + artistIds: new Map(artistRows.map((row) => [row.name, row.id])), + genreIds: new Map(genreRows.map((row) => [row.name, row.id])), + }; + }); + + for (let offset = 0; offset < albumCount; offset += LARGE_SEED_BATCH_SIZE) { + const count = Math.min(LARGE_SEED_BATCH_SIZE, albumCount - offset); + const catalog = Array.from({ length: count }, (_, index) => + createLargeCatalogAlbum(offset + index), + ); + + await db.transaction(async (tx) => { + const insertedAlbums = await tx + .insert(album) + .values(catalog.map(({ title, releaseYear }) => ({ title, releaseYear }))) + .returning({ id: album.id, title: album.title }); + const albumIds = new Map(insertedAlbums.map(({ id, title }) => [title, id])); + + const albumArtists = catalog.map((entry) => { + const albumId = albumIds.get(entry.title); + const artistId = artistIds.get(entry.artistName); + if (albumId === undefined || artistId === undefined) { + throw new Error('Large catalog seed lost an album or artist id'); + } + return { albumId, artistId }; + }); + const albumGenres = catalog.flatMap((entry) => { + const albumId = albumIds.get(entry.title); + if (albumId === undefined) throw new Error('Large catalog seed lost an album id'); + return entry.genreNames.map((name) => { + const genreId = genreIds.get(name); + if (genreId === undefined) throw new Error(`Unknown generated genre: ${name}`); + return { albumId, genreId }; + }); + }); + + await tx.insert(albumArtist).values(albumArtists); + await tx.insert(albumGenre).values(albumGenres); + }); + + const inserted = Math.min(offset + count, albumCount); + if (inserted === albumCount || inserted % 10_000 === 0) { + console.log( + `Seeded ${inserted.toLocaleString()} of ${albumCount.toLocaleString()} albums.`, + ); + } + } +} finally { + await pool.end(); +} + +console.log(`Large seed complete: ${albumCount.toLocaleString()} synthetic albums.`);