From fb757b73edc46d221f031eff7127dd1789e1f090 Mon Sep 17 00:00:00 2001 From: Ronja Date: Sat, 17 Oct 2026 09:51:08 +0200 Subject: [PATCH] feat: add a script for synthetic test data and measure the search npm run db:synthetic -- add [--count N] adds fake items with SYN- ids in groups of their own (100 000 by default), and -- remove deletes them again. The normal seed never runs it and leaves the rows alone. The README has the search times with and without indexes, and what EXPLAIN shows. --- README.md | 66 ++++++++-- backend/package.json | 1 + backend/src/seed/synthetic.ts | 122 ++++++++++++++++++ backend/src/seed/syntheticData.test.ts | 78 ++++++++++++ backend/src/seed/syntheticData.ts | 164 +++++++++++++++++++++++++ docs/AI_USAGE.md | 14 +++ package.json | 1 + 7 files changed, 439 insertions(+), 7 deletions(-) create mode 100644 backend/src/seed/synthetic.ts create mode 100644 backend/src/seed/syntheticData.test.ts create mode 100644 backend/src/seed/syntheticData.ts diff --git a/README.md b/README.md index 7ada51f..01831d1 100644 --- a/README.md +++ b/README.md @@ -230,13 +230,14 @@ npm run db:seed `npm run db:migrate` creates the tables in your database by applying the migrations in `backend/prisma/migrations/`. Run it again whenever someone has added a new migration. -| Command | What it does | -| --------------------- | ------------------------------------------------------------------------------ | -| `npm run db:generate` | Generates the Prisma client from `backend/prisma/schema.prisma` | -| `npm run db:migrate` | Creates a migration from changes in the schema and applies it to your database | -| `npm run db:deploy` | Applies existing migrations without creating new ones. Used on the server | -| `npm run db:seed` | Loads the catalog from `data/` into the database. See "Seeding" below | -| `npm run db:studio` | Opens Prisma Studio in the browser, to look at the data in the database | +| Command | What it does | +| ----------------------------- | ----------------------------------------------------------------------------------------------- | +| `npm run db:generate` | Generates the Prisma client from `backend/prisma/schema.prisma` | +| `npm run db:migrate` | Creates a migration from changes in the schema and applies it to your database | +| `npm run db:deploy` | Applies existing migrations without creating new ones. Used on the server | +| `npm run db:seed` | Loads the catalog from `data/` into the database. See "Seeding" below | +| `npm run db:synthetic -- add` | Adds 100 000 fake items for scalability tests. `remove` deletes them. See "Synthetic test data" | +| `npm run db:studio` | Opens Prisma Studio in the browser, to look at the data in the database | ### 5. Check the connection @@ -316,6 +317,57 @@ The tests in `backend/src/catalog/` run with `npm test`. The search tests use yo **Without `DATABASE_URL` in `backend/.env`, the database tests are skipped.** Vitest then reports them as skipped and the run still passes, so a passing run without a database does not mean the search was tested. Check that the output says 0 skipped. +### Synthetic test data and measurements + +The real catalog has about 560 items, which is too small to see whether the search scales. `npm run db:synthetic` adds fake items for that: + +```bash +npm run db:synthetic -- add # 100 000 items (use --count for another number) +npm run db:synthetic -- remove # deletes all synthetic data again +``` + +The items copy descriptions from the real catalog, so the words are spread like in real data. They are spread over the four apparatus (vault items on VT, elements on the others), about 5 % have a name, and the elements have a difficulty value from 0.1 to 1.0. Every id starts with `SYN-` and every item is in a synthetic group of its own (`SYN-UB-1001` and so on), so the real catalog is never mixed up with them. The same input gives the same rows. `npm run db:seed` never adds synthetic data, and it leaves the `SYN-` rows alone. `add` stops if synthetic data already exists, and runs in one transaction, so it either adds everything or nothing. `remove` deletes only rows with a `SYN-` id. Use a local database, never the one on the virtual machine. + +**Measured on 2026-10-17** with 100 562 catalog entries and 1 415 215 words (562 entries are real), PostgreSQL 18 in a Docker container on a laptop, after `ANALYZE`. The time is the median of 7 calls to `searchCatalog` from Node, including Prisma, with 20 items per page and without `totalCount`. "Without indexes" means that all indexes on `catalog_entries` and `catalog_words` except the primary keys were dropped (and created again afterwards). + +| Query | With indexes (ms) | Without indexes (ms) | +| --------------------------------- | ----------------: | -------------------: | +| browse, sort CODE | 1.6 | 20.8 | +| browse, sort NAME | 1.2 | 20.2 | +| browse, sort DIFFICULTY (UB) | 1.1 | 12.6 | +| filter apparatus UB | 1.4 | 12.5 | +| filter difficulty 0.1/0.2 (BB) | 1.1 | 12.6 | +| search "a" (one letter) | 1.6 | 48.2 | +| search "t" (one letter) | 1.4 | 47.7 | +| search "forward" (common word) | 1.3 | 48.0 | +| search "kip" + UB | 10.1 | 35.1 | +| search "forward turn" (two words) | 1.8 | 52.1 | +| search "syn5003" (rare word) | 1.1 | 29.6 | +| search "1001.5" (code) | 4.6 | 48.7 | + +Pages far into a result (100 items per page; the cursor for page N comes from following the pages before it): + +| Query | Page | With indexes (ms) | Without indexes (ms) | +| ----------------- | ---: | ----------------: | -------------------: | +| browse, sort CODE | 1 | 1.5 | 18.9 | +| browse, sort CODE | 101 | 2.2 | 39.2 | +| browse, sort CODE | 501 | 1.9 | 34.5 | +| search "a" | 1 | 1.8 | 49.8 | +| search "a" | 101 | 4.0 | 75.7 | +| search "a" | 501 | 1.5 | 31.1 | + +`EXPLAIN (ANALYZE, BUFFERS)` on the queries Prisma sends shows that the indexes are used: + +- **Page 501 of the list sorted by code:** an index scan on `catalog_entries_code_sort_id_idx` reads 99 rows and takes 0.11 ms. A page far into the list costs the same as the first page, because the cursor is a condition on the index and nothing is skipped (no `OFFSET`). +- **A one-letter search ("a"):** an index scan on `catalog_entries_code_sort_id_idx` walks the entries in order and checks each one against `catalog_words` through its primary key (63 entries checked for 20 results). It stops as soon as the page is full, so the search does not read all 1.4 million words. +- **Without the indexes** the same queries read the whole table: 12–52 ms for a page, and 30–76 ms for a page far into a result, and the time grows with the number of items. + +The tables take 63 MB (`catalog_entries`) and 222 MB (`catalog_words`) with their indexes, 185 MB of it in indexes. That is the price of the speed, at this size. Adding the synthetic data takes about 70 seconds for 100 000 items. + +**Reading the numbers.** These are one machine and one run, so use them to compare, not as promises. The slowest case with indexes is a search for a common word together with a filter ("kip" on uneven bars, 10 ms), where PostgreSQL has to check many words. A one-letter search that is only a trailing "s" ("s") finds nothing on purpose, because the trailing "s" is removed from a search word. + +Run `ANALYZE` after adding or removing many rows, or PostgreSQL plans from old statistics (`npm run db:synthetic` does not run it). To repeat the measurement, call `searchCatalog` from a small script and use `EXPLAIN (ANALYZE, BUFFERS)` on the queries Prisma logs (`log: [{ emit: 'event', level: 'query' }]`). The application code never uses raw SQL, but `EXPLAIN` is typed by hand in `psql`. + ## Project structure | Folder | Contents | diff --git a/backend/package.json b/backend/package.json index dd0d4ac..db3930c 100644 --- a/backend/package.json +++ b/backend/package.json @@ -9,6 +9,7 @@ "prisma:migrate": "prisma migrate dev", "prisma:deploy": "prisma migrate deploy", "prisma:seed": "prisma db seed", + "synthetic": "tsx src/seed/synthetic.ts", "prisma:studio": "prisma studio" }, "engines": { diff --git a/backend/src/seed/synthetic.ts b/backend/src/seed/synthetic.ts new file mode 100644 index 0000000..79e72c9 --- /dev/null +++ b/backend/src/seed/synthetic.ts @@ -0,0 +1,122 @@ +// Adds or removes synthetic catalog rows for scalability tests (issue #25). +// +// npm run db:synthetic -- add [--count 100000] +// npm run db:synthetic -- remove +// +// The rows have ids that start with "SYN-" and groups of their own. The normal +// seed never runs this script and never touches these rows. `add` writes +// everything in one transaction, and refuses to run when synthetic rows +// already exist (run `remove` first). `remove` deletes only rows with a +// "SYN-" id, so the real catalog and user data are left alone. + +import { fileURLToPath } from 'node:url' +import { parseArgs } from 'node:util' +import { createPrismaClient } from '../db.js' +import type { Prisma } from '../generated/prisma/client.js' +import { loadCatalogData } from './catalogData.js' +import { SYNTHETIC_PREFIX, generateSynthetic } from './syntheticData.js' + +const DEFAULT_COUNT = 100_000 +const DEFAULT_DATA_DIR = fileURLToPath( + new URL('../../../data/', import.meta.url), +) +const IS_SYNTHETIC = { id: { startsWith: SYNTHETIC_PREFIX } } +/** Rows per createMany. Postgres allows at most 65535 parameters per query. */ +const BATCH = 5000 + +async function insertInBatches( + rows: T[], + insert: (batch: T[]) => Promise, +) { + for (let i = 0; i < rows.length; i += BATCH) + await insert(rows.slice(i, i + BATCH)) +} + +export async function addSynthetic(count: number, dataDir = DEFAULT_DATA_DIR) { + const rows = generateSynthetic(loadCatalogData(dataDir), count) + const prisma = createPrismaClient() + try { + const existing = await prisma.catalogEntry.count({ where: IS_SYNTHETIC }) + if (existing > 0) + throw new Error( + `${existing} synthetic entries already exist. Run "npm run db:synthetic -- remove" first.`, + ) + const started = Date.now() + await prisma.$transaction( + async (tx) => { + await tx.elementGroup.createMany({ data: rows.groups }) + await insertInBatches(rows.elements, (data) => + tx.element.createMany({ data }), + ) + await insertInBatches(rows.vaults, (data) => + tx.vault.createMany({ data }), + ) + await insertInBatches(rows.entries, (data) => + tx.catalogEntry.createMany({ + data: data as Prisma.CatalogEntryCreateManyInput[], + }), + ) + await insertInBatches(rows.words, (data) => + tx.catalogWord.createMany({ data }), + ) + }, + { timeout: 30 * 60_000, maxWait: 10_000 }, + ) + console.log( + `Added ${rows.entries.length} synthetic items (${rows.elements.length} elements, ${rows.vaults.length} vaults) in ${rows.groups.length} groups, with ${rows.words.length} search words, in ${((Date.now() - started) / 1000).toFixed(1)} s.`, + ) + console.log( + 'Run "npm run db:synthetic -- remove" to delete them. Run ANALYZE (see the README) before measuring.', + ) + } finally { + await prisma.$disconnect() + } +} + +export async function removeSynthetic() { + const prisma = createPrismaClient() + try { + const result = await prisma.$transaction( + async (tx) => { + // The words are deleted with their entries (cascade). + const entries = await tx.catalogEntry.deleteMany({ + where: IS_SYNTHETIC, + }) + const elements = await tx.element.deleteMany({ where: IS_SYNTHETIC }) + const vaults = await tx.vault.deleteMany({ where: IS_SYNTHETIC }) + const groups = await tx.elementGroup.deleteMany({ + where: IS_SYNTHETIC, + }) + return { entries, elements, vaults, groups } + }, + { timeout: 30 * 60_000, maxWait: 10_000 }, + ) + console.log( + `Removed ${result.entries.count} entries, ${result.elements.count} elements, ${result.vaults.count} vaults and ${result.groups.count} groups.`, + ) + } finally { + await prisma.$disconnect() + } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const { values, positionals } = parseArgs({ + options: { count: { type: 'string' }, data: { type: 'string' } }, + allowPositionals: true, + }) + const command = positionals[0] + const run = + command === 'add' + ? addSynthetic(Number(values.count ?? DEFAULT_COUNT), values.data) + : command === 'remove' + ? removeSynthetic() + : Promise.reject( + new Error( + 'Usage: npm run db:synthetic -- add [--count 100000] or npm run db:synthetic -- remove', + ), + ) + run.catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error) + process.exit(1) + }) +} diff --git a/backend/src/seed/syntheticData.test.ts b/backend/src/seed/syntheticData.test.ts new file mode 100644 index 0000000..2add6e9 --- /dev/null +++ b/backend/src/seed/syntheticData.test.ts @@ -0,0 +1,78 @@ +// Tests for the synthetic data generator. No database is needed. + +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' +import { loadCatalogData } from './catalogData.js' +import { SYNTHETIC_PREFIX, generateSynthetic } from './syntheticData.js' + +const data = loadCatalogData( + fileURLToPath(new URL('../../../data/', import.meta.url)), +) + +describe('generateSynthetic', () => { + const rows = generateSynthetic(data, 5000) + + it('makes the requested number of items, spread over the apparatus', () => { + expect(rows.entries).toHaveLength(5000) + expect(rows.elements.length + rows.vaults.length).toBe(5000) + expect(rows.vaults.length).toBe(1250) // one in four is on vault + expect(new Set(rows.entries.map((e) => e.apparatusId))).toEqual( + new Set(['VT', 'UB', 'BB', 'FX']), + ) + }) + + it('starts every id with SYN- and gives the items groups of their own', () => { + const ids = [ + ...rows.groups.map((g) => g.id), + ...rows.elements.map((e) => e.id), + ...rows.vaults.map((v) => v.id), + ...rows.entries.map((e) => e.id), + ...rows.entries.map((e) => e.elementGroupId), + ] + expect(ids.every((id) => id.startsWith(SYNTHETIC_PREFIX))).toBe(true) + const realGroups = new Set(data.groups.map((g) => g.id)) + expect(rows.groups.some((g) => realGroups.has(g.id))).toBe(false) + }) + + it('has unique ids, and group numbers that do not clash with real groups', () => { + expect(new Set(rows.entries.map((e) => e.id)).size).toBe(5000) + expect(new Set(rows.groups.map((g) => g.id)).size).toBe(rows.groups.length) + const realNumbers = new Set( + data.groups.map((g) => `${g.apparatusId}-${g.number}`), + ) + for (const g of rows.groups) + expect(realNumbers.has(`${g.apparatusId}-${g.number}`)).toBe(false) + }) + + it('has valid values: 0.1-1.0 for elements, the real D-value for vaults', () => { + for (const e of rows.elements) { + expect(Number(e.value)).toBeGreaterThanOrEqual(0.1) + expect(Number(e.value)).toBeLessThanOrEqual(1) + } + const realVaultValues = new Set( + data.items.filter((i) => i.kind === 'vault').map((i) => i.value), + ) + for (const v of rows.vaults) + expect(realVaultValues.has(v.dValue)).toBe(true) + }) + + it('gives every entry search words, all pointing at an existing entry', () => { + const entryIds = new Set(rows.entries.map((e) => e.id)) + expect(rows.words.every((w) => entryIds.has(w.entryId))).toBe(true) + const withWords = new Set(rows.words.map((w) => w.entryId)) + expect(withWords.size).toBe(5000) + const keys = rows.words.map((w) => `${w.entryId}|${w.word}`) + expect(new Set(keys).size).toBe(keys.length) // the primary key (entry, word) + }) + + it('gives the same rows every time', () => { + expect(generateSynthetic(data, 100).entries).toEqual( + generateSynthetic(data, 100).entries, + ) + }) + + it('rejects a count that is not a positive whole number', () => { + expect(() => generateSynthetic(data, 0)).toThrow() + expect(() => generateSynthetic(data, 1.5)).toThrow() + }) +}) diff --git a/backend/src/seed/syntheticData.ts b/backend/src/seed/syntheticData.ts new file mode 100644 index 0000000..7a690fa --- /dev/null +++ b/backend/src/seed/syntheticData.ts @@ -0,0 +1,164 @@ +// Generates synthetic catalog rows for scalability tests (issue #25). Nothing +// here touches the database: synthetic.ts writes the rows. The rows look like +// the real catalog (descriptions are copied from real items, so the words are +// spread the same way), but every id starts with "SYN-" and every group is a +// group of its own, so the real catalog is never mixed up with them. + +import { searchWords } from '../catalog/words.js' +import { type CatalogData, codeSort } from './catalogData.js' + +export const SYNTHETIC_PREFIX = 'SYN-' + +/** Items per synthetic group. Keeps the sequence number in a code below 10000. */ +const GROUP_SIZE = 1000 +/** First group number. The real groups are numbered 1, 2, 3, ... */ +const FIRST_GROUP_NUMBER = 1001 +const APPARATUS_ORDER = ['VT', 'UB', 'BB', 'FX'] as const +/** Share of the items that are named after a gymnast. */ +const NAMED_SHARE = 0.05 + +export type SyntheticRows = { + groups: { + id: string + apparatusId: string + number: number + name: string + }[] + elements: { + id: string + elementGroupId: string + code: string + value: string + description: string + name: string | null + image: string + }[] + vaults: { + id: string + elementGroupId: string + code: string + dValue: string + description: string + name: string | null + image: string + }[] + entries: { + id: string + type: 'ELEMENT' | 'VAULT' + code: string + name: string | null + description: string + difficulty: string + apparatusId: string + elementGroupId: string + hasName: boolean + nameSort: string + codeSort: string + }[] + words: { entryId: string; word: string }[] +} + +/** A small, fast random number generator, so the same count gives the same rows. */ +function mulberry32(seed: number): () => number { + let a = seed + return () => { + a = (a + 0x6d2b79f5) | 0 + let t = Math.imul(a ^ (a >>> 15), 1 | a) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +/** + * `count` synthetic items spread over the four apparatus (vault on VT, the + * others elements). The real catalog in `data` is only used as a source of + * descriptions and vault values. + */ +export function generateSynthetic( + data: CatalogData, + count: number, +): SyntheticRows { + if (!Number.isInteger(count) || count < 1) + throw new Error(`The count must be a whole number of at least 1: ${count}`) + + const apparatusOfGroup = new Map( + data.groups.map((g) => [g.id, g.apparatusId]), + ) + const templates = new Map() + for (const item of data.items) { + const apparatusId = apparatusOfGroup.get(item.elementGroupId)! + templates.set(apparatusId, [...(templates.get(apparatusId) ?? []), item]) + } + for (const apparatusId of APPARATUS_ORDER) + if (!templates.get(apparatusId)?.length) + throw new Error(`The catalog has no items on ${apparatusId} to copy`) + + const random = mulberry32(25) + const rows: SyntheticRows = { + groups: [], + elements: [], + vaults: [], + entries: [], + words: [], + } + const groupIds = new Set() + + for (let i = 0; i < count; i++) { + const apparatusId = APPARATUS_ORDER[i % APPARATUS_ORDER.length]! + const indexOnApparatus = Math.floor(i / APPARATUS_ORDER.length) + const number = + FIRST_GROUP_NUMBER + Math.floor(indexOnApparatus / GROUP_SIZE) + const code = `${number}.${(indexOnApparatus % GROUP_SIZE) + 1}` + const groupId = `${SYNTHETIC_PREFIX}${apparatusId}-${number}` + const id = `${SYNTHETIC_PREFIX}${apparatusId}-${code}-a` + + if (!groupIds.has(groupId)) { + groupIds.add(groupId) + rows.groups.push({ + id: groupId, + apparatusId, + number, + name: `Synthetic group ${number}`, + }) + } + + const options = templates.get(apparatusId)! + const template = options[Math.floor(random() * options.length)]! + const name = random() < NAMED_SHARE ? `Syn${i}` : null + const isVault = apparatusId === 'VT' + const difficulty = isVault + ? template.value + : ((1 + Math.floor(random() * 10)) / 10).toFixed(1) // 0.1 ... 1.0 + + const common = { + id, + elementGroupId: groupId, + code, + description: template.description, + name, + image: 'synthetic.png', + } + if (isVault) rows.vaults.push({ ...common, dValue: difficulty }) + else rows.elements.push({ ...common, value: difficulty }) + + rows.entries.push({ + id, + type: isVault ? 'VAULT' : 'ELEMENT', + code, + name, + description: template.description, + difficulty, + apparatusId, + elementGroupId: groupId, + hasName: name !== null, + nameSort: name?.toLowerCase() ?? '', + codeSort: codeSort(code), + }) + + const words = new Set() + for (const text of [code, name ?? '', template.description]) + searchWords(text).forEach((w) => words.add(w)) + words.forEach((word) => rows.words.push({ entryId: id, word })) + } + return rows +} diff --git a/docs/AI_USAGE.md b/docs/AI_USAGE.md index 49e8d28..5bc470b 100644 --- a/docs/AI_USAGE.md +++ b/docs/AI_USAGE.md @@ -184,3 +184,17 @@ keyset condition into separate queries. They also moved the variants hint to its own issue. Claude made the changes, checked the query plans with `EXPLAIN ANALYZE`, added tests for each point and updated the docs. + +## Synthetic test data (`backend/src/seed/synthetic.ts`) + +- **Tool:** Claude Code +- **What the team did:** Wrote issue #25: a separate script that adds 50 000 to + 100 000 fake elements with `SYN-` ids in groups of their own, one command that + removes them, and measurements with and without indexes in the README. +- **What Claude did:** Wrote the generator (descriptions are copied from the real + catalog, so the words are spread like in real data), the `add` and `remove` + commands, and tests for the generator. Ran them on a throwaway PostgreSQL 18 + database with 100 000 items: checked that `remove` leaves the real catalog and + that the normal seed leaves the `SYN-` rows alone, measured the search with + and without indexes, read `EXPLAIN (ANALYZE, BUFFERS)` for three queries and + wrote the results in the README. diff --git a/package.json b/package.json index 3999596..3516842 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "db:migrate": "npm run prisma:migrate -w backend", "db:deploy": "npm run prisma:deploy -w backend", "db:seed": "npm run prisma:seed -w backend", + "db:synthetic": "npm run synthetic -w backend --", "db:studio": "npm run prisma:studio -w backend" }, "dependencies": {