diff --git a/backend/src/catalog/search.test.ts b/backend/src/catalog/search.test.ts index 71adfbb..1cb75a0 100644 --- a/backend/src/catalog/search.test.ts +++ b/backend/src/catalog/search.test.ts @@ -12,6 +12,7 @@ import { BadUserInputError, type CatalogSearchArgs, listElementGroups, + MAX_TOTAL_COUNT, searchCatalog, searchTerms, } from './search.js' @@ -502,4 +503,54 @@ describe.runIf(hasDatabase)('searchCatalog on the seeded catalog', () => { const page = await searchCatalog(prisma, { first: 1 }) expect(page.totalCount).toBeUndefined() }) + + describe('the count stops at the most that is counted', () => { + const count = async (maxTotalCount?: number, search?: string) => + ( + await searchCatalog(prisma, { + first: 1, + search, + withTotalCount: true, + maxTotalCount, + }) + ).totalCount + + it('counts up to 1000 as it is, and the catalog is smaller than that', async () => { + expect(MAX_TOTAL_COUNT).toBe(1000) + expect(totalEntries).toBeLessThanOrEqual(MAX_TOTAL_COUNT) + expect(await count()).toBe(totalEntries) + }) + + it('stops at one more than the most, so a count above it means "more than that"', async () => { + expect(totalEntries).toBeGreaterThan(10) + expect(await count(10)).toBe(11) + expect(await count(1)).toBe(2) + }) + + it('is not capped when the most is as big as the number of matches, or bigger', async () => { + expect(await count(totalEntries)).toBe(totalEntries) + expect(await count(totalEntries + 500)).toBe(totalEntries) + // The most that is one less than the number of matches: one more than it is the + // number of matches, which is above the most. + expect(await count(totalEntries - 1)).toBe(totalEntries) + }) + + it('caps a search too, and a search with fewer matches than the most is counted as it is', async () => { + const all = await searchCatalog(prisma, { + first: 100, + search: 'handspring', + withTotalCount: true, + }) + expect(all.totalCount).toBeGreaterThan(5) + + expect(await count(5, 'handspring')).toBe(6) + expect(await count(all.totalCount! + 10, 'handspring')).toBe( + all.totalCount, + ) + }) + + it('counts nothing for text with no searchable words, also with a cap', async () => { + expect(await count(10, '%')).toBe(0) + }) + }) }) diff --git a/backend/src/catalog/search.ts b/backend/src/catalog/search.ts index 6115d75..3b806fe 100644 --- a/backend/src/catalog/search.ts +++ b/backend/src/catalog/search.ts @@ -36,19 +36,33 @@ export type CatalogSearchArgs = { /** Page size, 1–100. */ first?: number | null after?: string | null - /** Count all matches. Only set it when the client asks for totalCount. */ + /** Count the matches. Only set it when the client asks for totalCount. */ withTotalCount?: boolean + /** + * The most matches that are counted, MAX_TOTAL_COUNT if it is left out. The count + * stops at one more than this, so a count above it means "more than this many". + */ + maxTotalCount?: number } export type CatalogSearchResult = { edges: { cursor: string; node: CatalogEntry }[] pageInfo: { hasNextPage: boolean; endCursor: string | null } - /** Only set when withTotalCount was true. */ + /** + * Only set when withTotalCount was true. At most maxTotalCount + 1: a value above + * maxTotalCount means that there are more matches than that. + */ totalCount?: number } const DIRECTIONS: readonly SortDirection[] = ['ASC', 'DESC'] const MAX_PAGE_SIZE = 100 +/** + * How many matches `totalCount` counts. A search that has more matches gets one more + * than this (1001), which the interface shows as "1000+". The database stops counting + * there, so the cost of the count does not grow with the catalog. + */ +export const MAX_TOTAL_COUNT = 1000 /** Limits for the search text. Far more than anyone types, and each word adds a condition to the query. */ export const MAX_SEARCH_LENGTH = 80 export const MAX_SEARCH_WORDS = 8 @@ -237,9 +251,15 @@ export async function searchCatalog( endCursor: edges.at(-1)?.cursor ?? null, }, } - // Counting reads every match, so it only runs when the client asks for it. - if (args.withTotalCount) - result.totalCount = await prisma.catalogEntry.count({ where }) + // Counting reads the matches, so it only runs when the client asks for it, and it + // stops after one more than the most that is counted (take). + if (args.withTotalCount) { + const max = args.maxTotalCount ?? MAX_TOTAL_COUNT + result.totalCount = await prisma.catalogEntry.count({ + where, + take: max + 1, + }) + } return result } diff --git a/backend/src/schema.graphql b/backend/src/schema.graphql index 6dfa941..d3180e6 100644 --- a/backend/src/schema.graphql +++ b/backend/src/schema.graphql @@ -205,7 +205,8 @@ type CatalogConnection { """ Number of matches in the whole result set, not only this page. Counting is expensive on a large result, so the server only counts when the client asks - for this field. + for this field, and it stops at 1001: a value above 1000 means that there are + more than 1000 matches (the interface shows "1000+"). """ totalCount: Int! } diff --git a/docs/AI_USAGE.md b/docs/AI_USAGE.md index 57ddbf7..184e762 100644 --- a/docs/AI_USAGE.md +++ b/docs/AI_USAGE.md @@ -460,3 +460,9 @@ page one at a time (the drawing, the warning, the disabled buttons, the not-found page, "Try again", "Trying again…", keeping the current element and the variant link) and checked that a test failed each time. + +## The count of the search stops at 1001 (`backend/src/catalog/search.ts`) + +- **Tool:** Claude Code +- **What the team did:** Decided that the page says "1000+" when there are more than 1000 results, and that the server must not count more than that. +- **What Claude did:** Made the database stop counting at 1001 (`count` with `take`), kept the schema the same (a `totalCount` above 1000 means more than 1000), wrote the tests (the count stops at one more than the most, it is not capped below that, and a search is capped too) and broke the cap to check that they fail without it. diff --git a/docs/API.md b/docs/API.md index e4bc931..7b61a51 100644 --- a/docs/API.md +++ b/docs/API.md @@ -158,7 +158,7 @@ The requirement is that the solution works with any amount of data. The catalog - **Filter and sort need indexes.** `schema.prisma` declares an index for each sort field combined with `id` (`codeSort`, `difficulty`, and `hasName` with `nameSort`), and for the filter columns (apparatus, group). Without them the database sorts the whole catalog for every request. The name sort puts named items first in both directions (`hasName` descending), so ascending and descending each have their own index. The apparatus filter is often combined with a sort, so there are also indexes on apparatus with `difficulty` and with `codeSort`. The difficulty sort always has an apparatus filter, so it uses the index on apparatus and `difficulty`. The difficulty filter without an apparatus uses the index on `difficulty`. Without them, a filtered page walks past the rows of the other apparatus first, which is slow in one direction (for example vault sorted by ascending difficulty). - **Sort by name needs a rule for items without a name.** Most items have no name. A cursor does not accept `nulls` placement, so the sort uses a plain `hasName` field and a `nameSort` field that is never `null` (see Search without raw SQL). - **The text search must not scan the table.** A search for `%word%` cannot use an ordinary index. The search uses a word table with a B-tree index and a prefix match, which finds the rows that have a word starting with the search term without reading every row. A search that is correct but reads every row works on 500 rows and not on 5 million. -- **`totalCount` is only calculated when the client asks for it.** Counting a large result takes time: about 30 ms for the whole catalog and about 100 ms for a search in a test with 225 000 entries, and the cost grows with the catalog. The resolver for the field runs only if `totalCount` is in the query, so a client that scrolls without showing a total does not pay for it. The client should therefore ask for `totalCount` only with the first page of a new search or filter, and not with the pages that are loaded while scrolling. Together with the debounce on the search field, this keeps the count from running on every keystroke. If the count is still too slow, it can be capped (for example "1000+"), which needs a change in the schema. +- **`totalCount` is only calculated when the client asks for it.** Counting a large result takes time: about 30 ms for the whole catalog and about 100 ms for a search in a test with 225 000 entries, and the cost grows with the catalog. The resolver for the field runs only if `totalCount` is in the query, so a client that scrolls without showing a total does not pay for it. The client should therefore ask for `totalCount` only with the first page of a new search or filter, and not with the pages that are loaded while scrolling. Together with the debounce on the search field, this keeps the count from running on every keystroke. **The count is capped:** it stops at 1001 (`MAX_TOTAL_COUNT` is 1000, and the database counts one more), so the cost does not grow with the catalog. A `totalCount` above 1000 means that there are more than 1000 matches, and the interface shows "1000+". The schema is the same (`Int!`), only the meaning of a value above 1000 is new. - **Elements and vaults are different tables, but one list.** A query cannot combine two tables into one sorted and paged result. The data model keeps them apart because they work differently, so the list is read from `CatalogEntry`, a read-only table with the shared columns that the seed script fills. It never gets out of date because the catalog never changes between seed runs. - **Variants are few.** A variant group has two or three members, so `variants` does not need paging.