From c13f4d5211b01e54e70f2ccf6d627d6f5d9c475d Mon Sep 17 00:00:00 2001 From: "Andreas S. Steffensen" Date: Tue, 13 Oct 2026 21:31:05 +0200 Subject: [PATCH 1/2] feat: prepare Geoapify venue data and opening hours --- .env.example | 3 + .gitignore | 1 + README.md | 3 + docs/geoapify-data-import.md | 265 +++++++++++++ server/README.md | 4 + server/package-lock.json | 49 ++- server/package.json | 4 + server/src/data/geoapify.ts | 393 +++++++++++++++++++ server/src/data/openingHours.ts | 111 ++++++ server/src/data/prepareVenues.ts | 190 ++++++++++ server/src/scripts/prepareVenues.ts | 102 +++++ server/src/scripts/previewGeoapify.ts | 98 +++++ server/tests/geoapify.test.ts | 518 ++++++++++++++++++++++++++ server/tests/openingHours.test.ts | 143 +++++++ server/tests/prepareVenues.test.ts | 192 ++++++++++ 15 files changed, 2075 insertions(+), 1 deletion(-) create mode 100644 docs/geoapify-data-import.md create mode 100644 server/src/data/geoapify.ts create mode 100644 server/src/data/openingHours.ts create mode 100644 server/src/data/prepareVenues.ts create mode 100644 server/src/scripts/prepareVenues.ts create mode 100644 server/src/scripts/previewGeoapify.ts create mode 100644 server/tests/geoapify.test.ts create mode 100644 server/tests/openingHours.test.ts create mode 100644 server/tests/prepareVenues.test.ts diff --git a/.env.example b/.env.example index ca2e148..ae3aa35 100644 --- a/.env.example +++ b/.env.example @@ -5,3 +5,6 @@ DATABASE_URL=postgresql://tapfinder_app:REPLACE_WITH_PASSWORD@localhost:5432/tap PORT=4000 # Keep the API local until a reverse proxy (for example, Nginx) is configured. HOST=127.0.0.1 + +# Only needed for the Geoapify data preview/import, not for starting GraphQL. +GEOAPIFY_API_KEY=REPLACE_WITH_GEOAPIFY_KEY diff --git a/.gitignore b/.gitignore index 3ca4e9c..73b5d7c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ node_modules/ dist/ build/ coverage/ +server/imports/ # Local environment and secrets .env diff --git a/README.md b/README.md index a588ca1..eabc214 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,9 @@ Se [server/README.md](server/README.md) for npm-oppsett, VM-konfigurasjon, lokal testing, databasebruker og migrasjoner. PostgreSQL og den planlagte backendtjenesten kjører direkte på VM-en; Docker brukes ikke. +Se [Geoapify data import](docs/geoapify-data-import.md) for å hente et lite utvalg +ekte utesteder, undersøke tilgjengelige felt og forberede databaseimporten. + ## Frontend-deploy Etter at en Pull Request er godkjent og merget til `main`, kjører GitHub diff --git a/docs/geoapify-data-import.md b/docs/geoapify-data-import.md new file mode 100644 index 0000000..9487c14 --- /dev/null +++ b/docs/geoapify-data-import.md @@ -0,0 +1,265 @@ +# Geoapify data import + +The intended flow is **Geoapify → import script → PostgreSQL on the team VM → +our GraphQL API → frontend**. Importing is a separate operation; browsing the +application should query our database rather than call Geoapify for every visit. + +## What is available now + +- PostgreSQL connection code and migration tooling exist in `server/`. +- The implemented GraphQL API currently exposes only `health` and `databaseStatus`. +- The agreed Venue V1 contract is a draft; Venue SQL tables and resolvers are not + implemented in the current branch. +- The frontend still reads mock places. +- `data:preview` fetches a sample and saves original source records locally. It + does not write to PostgreSQL or expose venues through GraphQL. + +## First step: inspect real source data + +From the repository root: + +```powershell +npm --prefix server ci +Copy-Item .env.example server/.env +``` + +Only copy the example if `server/.env` does not already exist. Obtain a key from +[Geoapify MyProjects](https://myprojects.geoapify.com/) and replace +`GEOAPIFY_API_KEY=REPLACE_WITH_GEOAPIFY_KEY` in `server/.env`. A database URL is +not needed for this step. Never put the real key in the example file. + +```powershell +npm --prefix server run data:preview -- --limit 20 +``` + +This requests one page of bars, pubs and nightclubs within 5 km of +`longitude=10.3951, latitude=63.4305` in central Trondheim. It then requests +`details` for each returned place to obtain available opening hours, website and +source identifiers. The default run makes at most 21 HTTP requests; it is a +sample, not a complete city catalog. Requests stop on an error, and a JSON file +is written only after the entire page succeeds. + +The output in `server/imports/` includes the retrieval time, search parameters, +source identifiers, original place/detail records and a preview of available +Venue fields. The directory is ignored by Git. No API key is included in the +export. The console lists missing required location fields; this is a coverage +report, not full Venue validation. + +Additional commands: + +```powershell +# Fetch the next sample page, with the same search area. +npm --prefix server run data:preview -- --limit 20 --offset 20 + +# Fetch only the Places page, without per-place detail requests. +npm --prefix server run data:preview -- --limit 20 --no-details + +# Show usage without a key or database connection. +npm --prefix server run data:preview -- --help +``` + +### Step 1 toward a Trondheim dataset: check the area boundary + +```powershell +npm --prefix server run data:preview -- --trondheim --limit 20 +``` + +`--trondheim` first resolves `Trondheim, Norway` with the Geocoding API, +restricted to Norwegian city results. It rejects a result without a Trondheim +name, Norwegian country code, city result type or place ID. That place ID is +then used as the Places API boundary filter instead of the 5 km radius. + +The console prints the resolved area, and the export saves `search.filter` and +`search.area`, including the selected geocoding record. This run still requests +only one sample page; it does not fetch every venue in Trondheim. It makes at +most 22 HTTP requests with the default limit and details enabled. Verify the +resolved area's coverage before expanding retrieval. A source boundary or +category search cannot guarantee that every real venue is mapped. + +Keep the sample limit at 20 while checking the area. Database writes are separate. + +### Step 2: collect the remaining query pages + +```powershell +npm --prefix server run data:preview -- --trondheim --all-pages --limit 50 +``` + +This starts a fresh collection at offset 0, resolves the area once and requests +consecutive pages of 50 records. It keeps the category/area query unchanged and +stops on a short or empty page. It fetches details for unique places, skips +duplicates by OSM type/ID (or Geoapify ID when OSM identity is missing) and saves +one combined JSON file. Existing exports are retained. + +The default cap is 10 pages, or at most 500 source records with this command. +`--max-pages` accepts 1–20 and requires `--all-pages`. Do not combine +`--all-pages` with a nonzero `--offset`. The console reports progress per page, +and `pagination` in the export records page counts, duplicate counts and whether +a short/empty page ended retrieval. If the cap is reached, `pagination.capped` +is true, the output is explicitly marked partial and the command exits with +code 2. Increase the cap only if needed; a partial export must not be treated as +the finished query. + +This collects the API's results for the selected categories and area. It cannot +guarantee that every real venue is mapped or that the provider returns a complete +catalog. A later request failure stops the run without writing a new export. + +### Step 3: prepare the Venue fields locally + +```powershell +npm --prefix server run data:prepare +``` + +This selects the newest completed Trondheim export with all pages and details +enabled. Sample pages, capped exports and details-free exports are skipped. +It validates required fields, coordinate bounds, source identities and website +URLs, then writes a separate `venue-drafts-*.json` file. It makes no API calls +and does not connect to PostgreSQL. The original export remains unchanged. + +Each draft contains a `venue` object with the agreed field names except `id`, +which the database must assign. Unknown price level, rating, description and +image values remain null. `reviewCount: 0` is an initial app counter, not a claim +about external reviews; an eventual importer must preserve existing user data. +Recognized source categories become factual tags such as `Internettilgang` or +`Rullestoltilgjengelig`; generic or unknown categories do not invent tags. + +Opening hours are intentionally still an intermediate step. `openingHoursRaw` +retains the source text and `openingHoursStatus` is `pending` or `missing`. +The draft's `openingHours` list is empty until conversion is implemented. The +file is marked `readyForDatabaseImport: false`; it is not yet a complete API +Venue or the final database import payload. + +### Step 4: convert opening hours + +```powershell +npm --prefix server run data:prepare -- --opening-hours +``` + +This uses the pinned `opening_hours` library to parse the source's OSM syntax +offline. Clear, week-stable schedules become `OpeningHours` entries. Missing +source hours remain an empty list with status `missing`. Date-dependent schedules, +uncertain closing times, parser warnings and annotations remain an empty list +with status `needs-review`, a reason and the original text. We do not invent +closing times or silently repair source rules. The console lists review cases. + +The result is a new `venue-hours-drafts-*.json` file. Other fields and the source +identity retain the same format. Original exports and earlier drafts are retained. +`readyForDatabaseImport` remains false until database compatibility and review +handling are settled. + +Intervals represent **calendar days in venue-local time (Europe/Oslo)**: + +- Monday `20:00–02:30` becomes Monday `20:00–24:00` plus Tuesday `00:00–02:30`. +- A day can have multiple entries, preserving breaks and morning/night intervals. +- A completely closed day has `closed: true`, `opens: null`, `closes: null`. +- Unknown hours use `[]`; they do not become seven closed days. +- `24:00` means the end of the named day; intervals have `opens < closes`. + +The existing GraphQL list shape permits multiple entries per weekday. Database +validation and frontend opening-status logic must use the same calendar-day +convention and allow these repeated weekdays. A unique `(venue, day)` constraint +would prevent multiple intervals and must not be assumed for this representation. + +The parser evaluates a fixed January reference week after checking that the +expression is week-stable. Only clock strings are exported; no reference dates +are stored as actual opening dates. Holiday exceptions and other variable rules +require an additional representation or an explicit unknown-hours fallback. + +## Mapping to the agreed Venue contract + +| Venue field | Source / handling | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | Our own persistent database ID, assigned on first import. | +| `name` | Geoapify `name`; report missing names. | +| `type` | Initially map `catering.bar`, `catering.pub`, `adult.nightclub` to `Bar`, `Pub`, `Nattklubb`. Expand after inspecting the source. | +| `city` | Geoapify `city`; do not infer it from a search radius. | +| `area` | Available `suburb` or `district`; report missing values. | +| `address` | Available street and house number; report a missing street. | +| `latitude`, `longitude` | Geoapify `lat`, `lon`; validate coordinate ranges. | +| `priceLevel` | Not provided by the documented API contract; leave unknown until curated or sourced elsewhere. | +| `rating` | Not provided by the documented API contract; leave unknown until app reviews or another source exist. | +| `reviewCount` | Count our app's reviews; initially zero when a venue has no app reviews. Do not manufacture external review counts. | +| `description` | Available `description`, otherwise null. | +| `openingHours` | Preserve source `opening_hours` for later parsing. Missing hours mean unknown, represented by an empty list in V1; never fabricate closed days. | +| `tags` | Preserve source categories in the preview; choose user-facing tags during normalization. | +| `imageUrl` | Null until an image source has been selected; raw media references remain in the saved response. | +| `websiteUrl` | Available `website`, otherwise null. | + +Geoapify opening hours use OpenStreetMap syntax. Each V1 `OpeningHours` entry +stores one interval; the list can represent multiple intervals on the same day +if consumers support repeated weekdays. Source expressions can also contain +holiday exceptions or open-ended times, which V1 does not represent completely. +Use a suitable parser during normalization and retain the original expression. +Unsupported expressions must remain unknown rather than be misrepresented. + +If required fields such as `area` or `address` are missing, curate them from a +verified source or keep the record out of the public Venue list. Do not satisfy +GraphQL non-null fields with invented values. The preview makes these gaps visible +before database import. + +## Next step: import into PostgreSQL + +1. Reuse any Venue migration/backend branch already in progress. The GraphQL + draft alone does not create a SQL table. Store the 16 public fields plus + internal source identity, original opening-hours expression and refresh time. +2. Use our own persistent Venue ID and a unique external source identity. + Prefer the pair `osm_type` + `osm_id` when present; retain the current + Geoapify place ID for further API requests. A Geoapify ID can change for the + same OSM object. Flag records without reliable identity for review. +3. Normalize the inspected source records and validate required fields. An + importer should insert new venues and update existing source fields using + `ON CONFLICT ... DO UPDATE`, preserving internal IDs and user-generated data. + Reimporting the same source should not create duplicates or reset reviews. +4. Apply the reviewed migration, then import a small sample in a transaction. + Compare row counts and source identities, then rerun the sample to check + duplicate handling. A partial search must never delete absent venues. +5. Implement Venue resolvers against PostgreSQL and replace the frontend's mock + API. Add backend search, filters, sorting and pagination before expanding + the catalog. Schedule refreshes after the manual workflow is verified. + +When importing from a local computer, keep this SSH tunnel open: + +```powershell +ssh -N -L 5433:127.0.0.1:5432 YOUR_VM_USER@it2810-07.idi.ntnu.no +``` + +Set `DATABASE_URL` in `server/.env` to the existing VM database, using local +`127.0.0.1:5433`. On the VM itself use `127.0.0.1:5432`. Ask the database owner +for the actual database/user and credentials through the group's normal private +channel. The documented defaults are database `tapfinder`, user `tapfinder_app`; +their existence and access have not been verified by this preview tool. + +## Tests and attribution + +```powershell +npm --prefix server run test:data +npm --prefix server run build +``` + +The preview unit tests use Node's test runner and synthetic API responses; they +need neither a key nor PostgreSQL. They cover missing values, coordinate bounds, +OSM identity preservation, details lookup, pagination inputs, empty results and +API errors, including Trondheim boundary selection and rejection of unexpected +city lookup results. Pagination tests cover fixed search parameters, duplicate +source identities, short/empty pages, page caps and failures on later pages. +Live data coverage and database behavior require separate checks. +Preparation tests cover required and optional fields, source identity, +duplicate rejection, tag mapping, coordinates and website normalization. +Opening-hours tests cover weekday mapping, midnight and Sunday carryover, +multiple intervals, 24/7, closed versus unknown days and unsupported expressions. +The existing frontend uses Vitest; application/API testing remains a later step. + +Retain source attribution with imported records. Display OpenStreetMap +attribution when showing its data, and Geoapify attribution as required by the +API plan. Follow the provider's current terms when publishing the catalog. + +This preview implementation and its unit tests were prepared with Codex. The +Trondheim boundary lookup was checked against Geoapify with one live Places +result on 2026-10-13. Database writes have not been performed. + +Primary documentation: + +- [Places API](https://apidocs.geoapify.com/docs/places/index.html) +- [Place Details API](https://apidocs.geoapify.com/docs/place-details/) +- [Source identifiers and optional details](https://apidocs.geoapify.com/how-to/place-discovery/place-details-by-id-or-coordinates) +- [Storage and attribution](https://www.geoapify.com/places-api/) +- [Opening-hours parser](https://github.com/opening-hours/opening_hours.js) diff --git a/server/README.md b/server/README.md index 5b9ba5f..52102ba 100644 --- a/server/README.md +++ b/server/README.md @@ -5,6 +5,10 @@ Målet er at både backend og PostgreSQL kjører direkte på prosjektets VM, ute Docker. Backendoppsettet er foreløpig et skjelett med helsesjekker; Tapfinders domene-API og tabeller kommer når gruppa har bestemt datamodellen. +For source-data preview and the next database-import steps, see +[Geoapify data import](../docs/geoapify-data-import.md). The preview requires only +`GEOAPIFY_API_KEY` and does not connect to PostgreSQL. + ## Krav og npm-oppsett - Node.js og npm installert på utviklermaskinen og VM-en. diff --git a/server/package-lock.json b/server/package-lock.json index 1173b26..b2ab0f8 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -9,6 +9,7 @@ "@apollo/server": "^5.0.0", "dotenv": "^16.6.1", "graphql": "^16.11.0", + "opening_hours": "3.15.0", "pg": "^8.16.3" }, "devDependencies": { @@ -1489,6 +1490,34 @@ "url": "https://opencollective.com/express" } }, + "node_modules/i18next": { + "version": "26.4.4", + "resolved": "https://registry.npmjs.org/i18next/-/i18next-26.4.4.tgz", + "integrity": "sha512-O+rqmQUi54OwAgnZ4SAwO6aRrbXZt36VZXWt6zvq0FpC3vJbH46oQRs6u4Uwm1vf8ZI7UqOXQiJhQVTor7vRiQ==", + "funding": [ + { + "type": "individual", + "url": "https://www.locize.com/i18next" + }, + { + "type": "individual", + "url": "https://www.i18next.com/how-to/faq#i18next-is-awesome.-how-can-i-support-the-project" + }, + { + "type": "individual", + "url": "https://www.locize.com" + } + ], + "license": "MIT", + "peerDependencies": { + "typescript": "^5 || ^6 || ^7" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/iconv-lite": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", @@ -1756,6 +1785,19 @@ "node": ">= 0.8" } }, + "node_modules/opening_hours": { + "version": "3.15.0", + "resolved": "https://registry.npmjs.org/opening_hours/-/opening_hours-3.15.0.tgz", + "integrity": "sha512-gSPN1YSLCcS4H4YL3Pc/0QRgoafuIhasQ44bM0U+62TABoEyIEeP9AuAIGSGHd9AWycq1eBwxbNfgkiCpx8bCQ==", + "license": "LGPL-3.0-only", + "dependencies": { + "i18next": "^26.3.6", + "suncalc": "^2.0.1" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -2152,6 +2194,11 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/suncalc": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/suncalc/-/suncalc-2.1.1.tgz", + "integrity": "sha512-ufMOPzYRk5aomEs3g+qeJGdCPZWC3TL94E0HQagoSusFnNXCvfuewWBroY3lxRHqA7IA8UGvgg7dcrSSDJsuEg==" + }, "node_modules/to-buffer": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.2.2.tgz", @@ -2249,7 +2296,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/server/package.json b/server/package.json index 04a6f92..7a170ac 100644 --- a/server/package.json +++ b/server/package.json @@ -6,6 +6,9 @@ "dev": "tsx watch src/index.ts", "build": "tsc -p tsconfig.json", "start": "node dist/index.js", + "data:preview": "tsx src/scripts/previewGeoapify.ts", + "data:prepare": "tsx src/scripts/prepareVenues.ts", + "test:data": "tsx --test tests/geoapify.test.ts tests/prepareVenues.test.ts tests/openingHours.test.ts", "migrate:create": "node-pg-migrate create", "migrate:up": "node-pg-migrate up", "migrate:down": "node-pg-migrate down" @@ -14,6 +17,7 @@ "@apollo/server": "^5.0.0", "dotenv": "^16.6.1", "graphql": "^16.11.0", + "opening_hours": "3.15.0", "pg": "^8.16.3" }, "devDependencies": { diff --git a/server/src/data/geoapify.ts b/server/src/data/geoapify.ts new file mode 100644 index 0000000..d67dcc0 --- /dev/null +++ b/server/src/data/geoapify.ts @@ -0,0 +1,393 @@ +export const previewSearch = { + categories: ['catering.bar', 'catering.pub', 'adult.nightclub'], + longitude: 10.3951, + latitude: 63.4305, + radius: 5000, +} + +export interface PreviewOptions { + limit: number + offset: number + details: boolean + trondheim?: boolean + allPages?: boolean + maxPages?: number +} + +export interface PreviewPagination { + pagesFetched: number + recordsFetched: number + duplicatesSkipped: number + exhausted: boolean + capped: boolean +} + +export interface ResolvedPreviewSearch { + categories: string[] + filter: + | { type: 'circle'; lon: number; lat: number; radius: number } + | { type: 'place'; id: string } + area?: { + name: string + formatted: string + countryCode: string + resultType: string + raw: Record + } +} + +export interface GeoapifyFeature { + type: 'Feature' + properties: Record + [key: string]: unknown +} + +export interface PreviewPlace { + source: { + geoapifyPlaceId: string + osmType: string | null + osmId: string | null + } + availableFields: { + name: string | null + type: string | null + city: string | null + area: string | null + address: string | null + latitude: number | null + longitude: number | null + description: string | null + websiteUrl: string | null + openingHoursRaw: string | null + categories: string[] + } + missingRequiredFields: string[] + raw: { place: GeoapifyFeature; details: GeoapifyFeature | null } +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function string(value: unknown): string | null { + return typeof value === 'string' && value.trim() ? value.trim() : null +} + +function coordinate(value: unknown, max: number): number | null { + return typeof value === 'number' && + Number.isFinite(value) && + Math.abs(value) <= max + ? value + : null +} + +export function parsePreviewOptions(args: string[]): PreviewOptions { + const options: PreviewOptions = { limit: 20, offset: 0, details: true } + for (let i = 0; i < args.length; i += 1) { + const flag = args[i] + if (flag === '--all-pages') { + options.allPages = true + continue + } + if (flag === '--trondheim') { + options.trondheim = true + continue + } + if (flag === '--no-details') { + options.details = false + continue + } + if (flag !== '--limit' && flag !== '--offset' && flag !== '--max-pages') { + throw new Error(`Unknown option: ${flag}. Use --help for usage.`) + } + const input = args[++i] + if (!input || !/^\d+$/.test(input)) { + throw new Error(`${flag} must be followed by a whole number.`) + } + const value = Number(input) + const validRange = + flag === '--limit' + ? value >= 1 && value <= 100 + : flag === '--max-pages' + ? value >= 1 && value <= 20 + : value >= 0 + if (!Number.isSafeInteger(value) || !validRange) { + throw new Error( + `${flag} is out of range (${flag === '--limit' ? '1–100' : flag === '--max-pages' ? '1–20' : '0 or higher'}).`, + ) + } + options[ + flag === '--limit' + ? 'limit' + : flag === '--max-pages' + ? 'maxPages' + : 'offset' + ] = value + } + if (options.maxPages !== undefined && !options.allPages) { + throw new Error('--max-pages requires --all-pages.') + } + if (options.allPages && options.offset !== 0) { + throw new Error( + '--all-pages starts at offset 0. Omit --offset to collect the whole query.', + ) + } + return options +} + +export function parseFeatures(payload: unknown): GeoapifyFeature[] { + if ( + !record(payload) || + payload.type !== 'FeatureCollection' || + !Array.isArray(payload.features) + ) { + throw new Error('Geoapify returned an invalid FeatureCollection.') + } + return payload.features.map((feature: unknown) => { + if ( + !record(feature) || + feature.type !== 'Feature' || + !record(feature.properties) + ) { + throw new Error('Geoapify returned a feature without valid properties.') + } + return feature as GeoapifyFeature + }) +} + +export function previewPlace( + place: GeoapifyFeature, + details: GeoapifyFeature | null, +): PreviewPlace { + const properties = { ...place.properties, ...details?.properties } + const placeId = string(properties.place_id) + if (!placeId) throw new Error('A Geoapify place is missing its place_id.') + + const datasource = record(properties.datasource) ? properties.datasource : {} + const sourceRaw = record(datasource.raw) ? datasource.raw : {} + const osmType = ['n', 'w', 'r'].includes(String(sourceRaw.osm_type)) + ? String(sourceRaw.osm_type) + : null + const osmId = + typeof sourceRaw.osm_id === 'number' && + Number.isSafeInteger(sourceRaw.osm_id) && + sourceRaw.osm_id > 0 + ? String(sourceRaw.osm_id) + : typeof sourceRaw.osm_id === 'string' && + /^[1-9]\d*$/.test(sourceRaw.osm_id) + ? sourceRaw.osm_id + : null + const categories = Array.isArray(properties.categories) + ? properties.categories.filter( + (category): category is string => typeof category === 'string', + ) + : [] + const type = categories.includes('adult.nightclub') + ? 'Nattklubb' + : categories.includes('catering.pub') + ? 'Pub' + : categories.includes('catering.bar') + ? 'Bar' + : null + const street = string(properties.street) + const houseNumber = string(properties.housenumber) + const availableFields = { + name: string(properties.name), + type, + city: string(properties.city), + area: string(properties.suburb) ?? string(properties.district), + address: street ? [street, houseNumber].filter(Boolean).join(' ') : null, + latitude: coordinate(properties.lat, 90), + longitude: coordinate(properties.lon, 180), + description: string(properties.description), + websiteUrl: string(properties.website), + openingHoursRaw: string(properties.opening_hours), + categories, + } + const required = [ + 'name', + 'type', + 'city', + 'area', + 'address', + 'latitude', + 'longitude', + ] as const + return { + source: { geoapifyPlaceId: placeId, osmType, osmId }, + availableFields, + missingRequiredFields: required.filter( + (field) => availableFields[field] === null, + ), + raw: { place, details }, + } +} + +export async function fetchPreview( + apiKey: string, + options: PreviewOptions, + request: typeof fetch = fetch, + onProgress?: (message: string) => void, +): Promise<{ + search: ResolvedPreviewSearch + places: PreviewPlace[] + pagination: PreviewPagination +}> { + if (!apiKey.trim() || apiKey.startsWith('REPLACE_')) { + throw new Error('Set GEOAPIFY_API_KEY in server/.env before fetching data.') + } + + async function getJson(url: URL, init?: RequestInit): Promise { + let response: Response + try { + response = await request(url, { + ...init, + signal: AbortSignal.timeout(20_000), + }) + } catch { + // Request errors can contain a URL with the API key. Do not log them. + throw new Error( + 'Geoapify request failed or timed out. Check your connection and API key.', + ) + } + if (!response.ok) { + throw new Error( + `Geoapify returned HTTP ${response.status}. Check your API key and quota.`, + ) + } + try { + return await response.json() + } catch { + throw new Error('Geoapify returned invalid JSON.') + } + } + + async function getFeatures(url: URL, init?: RequestInit) { + return parseFeatures(await getJson(url, init)) + } + + const search: ResolvedPreviewSearch = { + categories: previewSearch.categories, + filter: { + type: 'circle', + lon: previewSearch.longitude, + lat: previewSearch.latitude, + radius: previewSearch.radius, + }, + } + if (options.trondheim) { + const url = new URL('https://api.geoapify.com/v1/geocode/search') + url.search = new URLSearchParams({ + text: 'Trondheim, Norway', + type: 'city', + filter: 'countrycode:no', + format: 'json', + lang: 'nb', + limit: '1', + apiKey, + }).toString() + const payload = await getJson(url) + const area = + record(payload) && Array.isArray(payload.results) + ? payload.results[0] + : undefined + if ( + !record(area) || + area.country_code !== 'no' || + area.result_type !== 'city' || + ![string(area.city), string(area.name)].some( + (name) => name?.toLowerCase() === 'trondheim', + ) || + !string(area.place_id) + ) { + throw new Error( + 'Geoapify did not resolve Trondheim, Norway to a valid city area. No places were fetched.', + ) + } + search.filter = { type: 'place', id: string(area.place_id)! } + search.area = { + name: 'Trondheim', + formatted: string(area.formatted) ?? 'Trondheim, Norway', + countryCode: 'no', + resultType: 'city', + raw: area, + } + } + + const results: PreviewPlace[] = [] + const pagination: PreviewPagination = { + pagesFetched: 0, + recordsFetched: 0, + duplicatesSkipped: 0, + exhausted: false, + capped: false, + } + const seenSources = new Set() + const seenPlaceIds = new Set() + const sourceKey = (place: PreviewPlace) => + place.source.osmType && place.source.osmId + ? `osm:${place.source.osmType}:${place.source.osmId}` + : `geoapify:${place.source.geoapifyPlaceId}` + const maxPages = options.allPages ? (options.maxPages ?? 10) : 1 + for (let page = 0; page < maxPages; page += 1) { + const offset = options.offset + page * options.limit + onProgress?.(`Fetching page ${page + 1} (offset ${offset})...`) + const places = await getFeatures( + new URL('https://api.geoapify.com/v2/places'), + { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify({ + categories: search.categories, + filter: search.filter, + limit: options.limit, + offset, + lang: 'nb', + }), + }, + ) + pagination.pagesFetched += 1 + pagination.recordsFetched += places.length + for (const place of places) { + const candidate = previewPlace(place, null) + const placeId = candidate.source.geoapifyPlaceId + if (seenSources.has(sourceKey(candidate)) || seenPlaceIds.has(placeId)) { + pagination.duplicatesSkipped += 1 + continue + } + let details: GeoapifyFeature | null = null + if (options.details) { + const url = new URL('https://api.geoapify.com/v2/place-details') + url.search = new URLSearchParams({ + id: placeId, + features: 'details', + lang: 'nb', + apiKey, + }).toString() + details = + (await getFeatures(url)).find( + (feature) => feature.properties.feature_type === 'details', + ) ?? null + } + const result = previewPlace(place, details) + const duplicate = + seenSources.has(sourceKey(result)) || + seenPlaceIds.has(result.source.geoapifyPlaceId) + seenSources.add(sourceKey(candidate)) + seenSources.add(sourceKey(result)) + seenPlaceIds.add(placeId) + seenPlaceIds.add(result.source.geoapifyPlaceId) + if (duplicate) pagination.duplicatesSkipped += 1 + else results.push(result) + } + onProgress?.( + `Page ${page + 1} done: ${places.length} records, ${results.length} unique places collected.`, + ) + if (places.length < options.limit) { + pagination.exhausted = true + break + } + } + pagination.capped = Boolean(options.allPages && !pagination.exhausted) + return { search, places: results, pagination } +} diff --git a/server/src/data/openingHours.ts b/server/src/data/openingHours.ts new file mode 100644 index 0000000..b1924d7 --- /dev/null +++ b/server/src/data/openingHours.ts @@ -0,0 +1,111 @@ +import OpeningHoursParser from 'opening_hours' + +export const weekdays = [ + 'MONDAY', + 'TUESDAY', + 'WEDNESDAY', + 'THURSDAY', + 'FRIDAY', + 'SATURDAY', + 'SUNDAY', +] as const +export type Weekday = (typeof weekdays)[number] +export interface OpeningHours { + day: Weekday + opens: string | null + closes: string | null + closed: boolean +} + +export interface OpeningHoursResult { + hours: OpeningHours[] + status: 'missing' | 'normalized' | 'needs-review' + reviewReason: + | 'variable-schedule' + | 'parser-warning' + | 'uncertain-interval' + | 'annotated-interval' + | 'invalid-expression' + | null + warnings: string[] +} + +// Output is venue-local wall-clock time. The fixed January week avoids DST +// changes; only week-stable expressions can become a recurring weekday list. +export function normalizeOpeningHours( + raw: string | null, + location = { latitude: 63.4305, longitude: 10.3951 }, +): OpeningHoursResult { + if (!raw?.trim()) + return { hours: [], status: 'missing', reviewReason: null, warnings: [] } + const needsReview = ( + reviewReason: OpeningHoursResult['reviewReason'], + warnings: string[] = [], + ): OpeningHoursResult => ({ + hours: [], + status: 'needs-review', + reviewReason, + warnings, + }) + try { + const parser = new OpeningHoursParser( + raw.trim(), + { + lat: location.latitude, + lon: location.longitude, + address: { country_code: 'no', state: 'Trøndelag' }, + }, + { + mode: undefined, + tag_key: 'opening_hours', + map_value: false, + warnings_severity: undefined, + locale: 'en', + }, + ) + const warnings = parser.getWarnings() + if (!parser.isWeekStable()) + return needsReview('variable-schedule', warnings) + if (warnings.length) return needsReview('parser-warning', warnings) + const from = new Date(2024, 0, 8) + const to = new Date(2024, 0, 15) + const intervals = parser.getOpenIntervals(from, to) + if (intervals.some((interval) => interval[2])) + return needsReview('uncertain-interval') + if (intervals.some((interval) => interval[3])) + return needsReview('annotated-interval') + const clock = (timestamp: number) => { + const date = new Date(timestamp) + return `${String(date.getHours()).padStart(2, '0')}:${String(date.getMinutes()).padStart(2, '0')}` + } + const hours = weekdays.flatMap((day, index): OpeningHours[] => { + const start = new Date(2024, 0, 8 + index).getTime() + const end = new Date(2024, 0, 9 + index).getTime() + const ranges = intervals + .map(([opens, closes]): [number, number] => [ + Math.max(start, opens.getTime()), + Math.min(end, closes.getTime()), + ]) + .filter(([opens, closes]) => opens < closes) + .sort((a, b) => a[0] - b[0]) + const merged: { opens: number; closes: number }[] = [] + for (const [opens, closes] of ranges) { + const previous = merged.at(-1) + if (previous && opens <= previous.closes) + previous.closes = Math.max(previous.closes, closes) + else merged.push({ opens, closes }) + } + if (!merged.length) + return [{ day, opens: null, closes: null, closed: true }] + return merged.map(({ opens, closes }) => ({ + day, + opens: clock(opens), + closes: closes === end ? '24:00' : clock(closes), + closed: false, + })) + }) + return { hours, status: 'normalized', reviewReason: null, warnings: [] } + } catch { + return needsReview('invalid-expression') + } +} diff --git a/server/src/data/prepareVenues.ts b/server/src/data/prepareVenues.ts new file mode 100644 index 0000000..d061cf4 --- /dev/null +++ b/server/src/data/prepareVenues.ts @@ -0,0 +1,190 @@ +import { + normalizeOpeningHours, + type OpeningHours, + type OpeningHoursResult, +} from './openingHours.js' + +type JsonRecord = Record + +// Intermediate data only: database IDs and opening-hours conversion come later. +export interface VenueDraft { + sourceKey: string + source: { osmType: string; osmId: string; geoapifyPlaceId: string } + venue: { + name: string + type: string + city: string + area: string + address: string + latitude: number + longitude: number + priceLevel: null + rating: null + reviewCount: 0 + description: string | null + openingHours: OpeningHours[] + tags: string[] + imageUrl: null + websiteUrl: string | null + } + openingHoursRaw: string | null + openingHoursStatus: 'pending' | OpeningHoursResult['status'] + openingHoursReviewReason: OpeningHoursResult['reviewReason'] + openingHoursWarnings: string[] +} + +function record(value: unknown): value is JsonRecord { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function text(value: unknown): string | null { + return typeof value === 'string' && value.trim() ? value.trim() : null +} + +export function isCompletedTrondheimExport( + value: unknown, +): value is JsonRecord { + if (!record(value) || !record(value.search) || !record(value.pagination)) + return false + return ( + value.version === 1 && + value.provider === 'Geoapify' && + value.search.trondheim === true && + value.search.allPages === true && + value.search.details === true && + value.search.offset === 0 && + value.pagination.exhausted === true && + value.pagination.capped === false && + Array.isArray(value.places) + ) +} + +export function prepareVenueDrafts( + payload: unknown, + options: { openingHours?: boolean } = {}, +): VenueDraft[] { + if (!isCompletedTrondheimExport(payload)) { + throw new Error( + 'Use a completed Trondheim export with --all-pages and details enabled.', + ) + } + const places = payload.places as unknown[] + if (places.length === 0) + throw new Error('The selected export has no places to prepare.') + const seenSources = new Set() + return places.map((place, index): VenueDraft => { + if ( + !record(place) || + !record(place.availableFields) || + !record(place.source) + ) { + throw new Error(`Record ${index + 1} has invalid place/source data.`) + } + const fields = place.availableFields + const requiredText = (field: string) => { + const value = text(fields[field]) + if (!value) throw new Error(`Record ${index + 1} is missing ${field}.`) + return value + } + const coordinate = (field: string, bound: number) => { + const value = fields[field] + if ( + typeof value !== 'number' || + !Number.isFinite(value) || + Math.abs(value) > bound + ) { + throw new Error(`Record ${index + 1} has invalid ${field}.`) + } + return value + } + const osmType = text(place.source.osmType) + const osmId = text(place.source.osmId) + const geoapifyPlaceId = text(place.source.geoapifyPlaceId) + if ( + !osmType || + !['n', 'w', 'r'].includes(osmType) || + !osmId || + !/^[1-9]\d*$/.test(osmId) || + !geoapifyPlaceId + ) { + throw new Error( + `Record ${index + 1} is missing reliable source identity.`, + ) + } + const sourceKey = `openstreetmap:${osmType}:${osmId}` + if (seenSources.has(sourceKey)) + throw new Error(`Duplicate source identity at record ${index + 1}.`) + seenSources.add(sourceKey) + const city = requiredText('city') + if (city.toLowerCase() !== 'trondheim') + throw new Error(`Record ${index + 1} is outside the expected city.`) + const type = requiredText('type') + if (!['Bar', 'Pub', 'Nattklubb'].includes(type)) + throw new Error(`Record ${index + 1} has an unsupported venue type.`) + let websiteUrl = text(fields.websiteUrl) + if (websiteUrl) { + try { + const website = new URL(websiteUrl) + if ( + !['http:', 'https:'].includes(website.protocol) || + website.username || + website.password + ) + throw new Error() + websiteUrl = website.href + } catch { + throw new Error(`Record ${index + 1} has an invalid website URL.`) + } + } + const categoryTags: Record = { + internet_access: 'Internettilgang', + vegan: 'Veganske alternativer', + vegetarian: 'Vegetariske alternativer', + 'wheelchair.yes': 'Rullestoltilgjengelig', + 'wheelchair.limited': 'Begrenset rullestoltilgang', + 'production.brewery': 'Bryggeri', + } + const categories = Array.isArray(fields.categories) ? fields.categories : [] + const tags = [ + ...new Set( + categories.flatMap((category) => + typeof category === 'string' && Object.hasOwn(categoryTags, category) + ? [categoryTags[category]] + : [], + ), + ), + ] + const openingHoursRaw = text(fields.openingHoursRaw) + const latitude = coordinate('latitude', 90) + const longitude = coordinate('longitude', 180) + const normalizedHours = options.openingHours + ? normalizeOpeningHours(openingHoursRaw, { latitude, longitude }) + : null + return { + sourceKey, + source: { osmType, osmId, geoapifyPlaceId }, + venue: { + name: requiredText('name'), + type, + city, + area: requiredText('area'), + address: requiredText('address'), + latitude, + longitude, + priceLevel: null, + rating: null, + reviewCount: 0, + description: text(fields.description), + openingHours: normalizedHours?.hours ?? [], + tags, + imageUrl: null, + websiteUrl, + }, + openingHoursRaw, + openingHoursStatus: + normalizedHours?.status ?? (openingHoursRaw ? 'pending' : 'missing'), + openingHoursReviewReason: normalizedHours?.reviewReason ?? null, + openingHoursWarnings: normalizedHours?.warnings ?? [], + } + }) +} diff --git a/server/src/scripts/prepareVenues.ts b/server/src/scripts/prepareVenues.ts new file mode 100644 index 0000000..89d111a --- /dev/null +++ b/server/src/scripts/prepareVenues.ts @@ -0,0 +1,102 @@ +import { randomUUID } from 'node:crypto' +import { readFile, readdir, stat, writeFile } from 'node:fs/promises' +import { basename, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { + isCompletedTrondheimExport, + prepareVenueDrafts, +} from '../data/prepareVenues.js' + +async function main() { + const args = process.argv.slice(2) + if (args.includes('--help')) { + console.log( + 'Usage: npm --prefix server run data:prepare -- [--opening-hours]\nPrepares the newest completed Trondheim export in server/imports/.\n--opening-hours converts unambiguous weekly schedules and flags other expressions for review.\nWrites Venue drafts locally; no API calls or database writes.', + ) + return + } + if (args.length && !(args.length === 1 && args[0] === '--opening-hours')) + throw new Error('Unknown option. Use --help for usage.') + const openingHours = args.includes('--opening-hours') + const directory = fileURLToPath(new URL('../../imports/', import.meta.url)) + const candidates = await Promise.all( + (await readdir(directory)) + .filter((name) => /^geoapify-\d.*\.json$/.test(name)) + .map(async (name) => ({ + path: join(directory, name), + modified: (await stat(join(directory, name))).mtimeMs, + })), + ) + candidates.sort((a, b) => b.modified - a.modified) + for (const candidate of candidates) { + let payload: unknown + try { + payload = JSON.parse(await readFile(candidate.path, 'utf8')) + } catch { + console.warn( + `Skipping unreadable source export: ${basename(candidate.path)}`, + ) + continue + } + if (!isCompletedTrondheimExport(payload)) continue + const venues = prepareVenueDrafts(payload, { openingHours }) + const preparedAt = new Date().toISOString() + const output = join( + directory, + `venue-${openingHours ? 'hours-' : ''}drafts-${preparedAt.replaceAll(':', '-')}-${randomUUID()}.json`, + ) + await writeFile( + output, + `${JSON.stringify( + { + version: 1, + stage: openingHours ? 'venue-hours-draft' : 'venue-draft', + readyForDatabaseImport: false, + provider: 'Geoapify', + attribution: payload.attribution, + sourceFile: basename(candidate.path), + sourceFetchedAt: payload.fetchedAt, + search: payload.search, + preparedAt, + venues, + }, + null, + 2, + )}\n`, + { flag: 'wx' }, + ) + console.log(`Source: ${basename(candidate.path)}`) + console.log(`Prepared ${venues.length} Venue drafts to ${output}`) + console.log( + `Websites: ${venues.filter((item) => item.venue.websiteUrl).length}/${venues.length}.`, + ) + console.log( + `Opening hours: normalized ${venues.filter((item) => item.openingHoursStatus === 'normalized').length}; needs review ${venues.filter((item) => item.openingHoursStatus === 'needs-review').length}; missing ${venues.filter((item) => item.openingHoursStatus === 'missing').length}; pending ${venues.filter((item) => item.openingHoursStatus === 'pending').length}.`, + ) + const review = venues.filter( + (item) => item.openingHoursStatus === 'needs-review', + ) + if (review.length) + console.table( + review.map((item) => ({ + name: item.venue.name, + reason: item.openingHoursReviewReason, + source: item.openingHoursRaw, + })), + ) + console.log( + 'Database IDs and schema compatibility remain pending; flagged opening hours require review. Review counts are initial app counters, not external review counts. No API calls or database writes were made.', + ) + return + } + throw new Error( + 'No completed Trondheim export found. Run data:preview with --trondheim --all-pages and details enabled first.', + ) +} + +main().catch((error: unknown) => { + console.error( + error instanceof Error ? error.message : 'Venue preparation failed.', + ) + process.exitCode = 1 +}) diff --git a/server/src/scripts/previewGeoapify.ts b/server/src/scripts/previewGeoapify.ts new file mode 100644 index 0000000..9e6fa4a --- /dev/null +++ b/server/src/scripts/previewGeoapify.ts @@ -0,0 +1,98 @@ +import { randomUUID } from 'node:crypto' +import { mkdir, writeFile } from 'node:fs/promises' +import { fileURLToPath } from 'node:url' +import { config } from 'dotenv' +import { fetchPreview, parsePreviewOptions } from '../data/geoapify.js' + +const serverRoot = new URL('../../', import.meta.url) +config({ path: fileURLToPath(new URL('.env', serverRoot)) }) + +async function main() { + const args = process.argv.slice(2) + if (args.includes('--help')) { + console.log(`Usage: npm --prefix server run data:preview -- [--trondheim] [--all-pages] [--limit 20] [--offset 0] [--no-details] [--max-pages 10] +Fetch one page of bars, pubs and nightclubs within 5 km of central Trondheim. +--trondheim resolves Trondheim, Norway and searches inside its Geoapify area boundary. +Default: 20 places plus one Place Details request per place; limit range: 1–100. +--trondheim adds one city lookup request; the selected area is saved in the export. +--all-pages fetches consecutive pages from offset 0, stopping on a short or empty page. +Default cap: 10 pages; --max-pages accepts 1–20. A capped export is marked partial. +Saves source data under server/imports/ and prints missing required Venue fields. +No database connection is used. --no-details fetches only the Places page.`) + return + } + const options = parsePreviewOptions(args) + const { search, places, pagination } = await fetchPreview( + process.env.GEOAPIFY_API_KEY ?? '', + options, + fetch, + (message) => console.log(message), + ) + const fetchedAt = new Date().toISOString() + const directory = new URL('imports/', serverRoot) + await mkdir(directory, { recursive: true }) + const output = new URL( + `geoapify-${fetchedAt.replaceAll(':', '-')}-${randomUUID()}.json`, + directory, + ) + await writeFile( + output, + `${JSON.stringify( + { + version: 1, + provider: 'Geoapify', + attribution: 'Powered by Geoapify | © OpenStreetMap contributors', + fetchedAt, + search: { ...search, ...options }, + pagination, + places, + }, + null, + 2, + )}\n`, + { flag: 'wx' }, + ) + + console.log( + search.area + ? `Search area: ${search.area.formatted} (Geoapify city boundary)` + : 'Search area: within 5 km of central Trondheim', + ) + console.table( + places.map((place) => ({ + name: place.availableFields.name ?? '(missing name)', + type: place.availableFields.type, + city: place.availableFields.city, + missingRequiredFields: place.missingRequiredFields.join(', ') || '(none)', + openingHours: place.availableFields.openingHoursRaw + ? 'available' + : 'unknown', + website: place.availableFields.websiteUrl ? 'available' : 'unknown', + })), + ) + console.log( + `Saved ${places.length} source records to ${fileURLToPath(output)}`, + ) + console.log( + `Pages fetched: ${pagination.pagesFetched}. Duplicate source records skipped: ${pagination.duplicatesSkipped}.`, + ) + if (pagination.capped) { + console.error( + 'Page cap reached: this export is partial. Increase --max-pages before treating retrieval as finished.', + ) + process.exitCode = 2 + } else if (options.allPages) { + console.log( + 'Pagination finished for this category/area query. This does not guarantee every real venue is mapped. No database writes were made.', + ) + } else { + console.log( + 'This is a sample page, not a complete Trondheim catalog or a database import.', + ) + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : 'Data preview failed.') + process.exitCode = 1 +}) diff --git a/server/tests/geoapify.test.ts b/server/tests/geoapify.test.ts new file mode 100644 index 0000000..2b43371 --- /dev/null +++ b/server/tests/geoapify.test.ts @@ -0,0 +1,518 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + fetchPreview, + parseFeatures, + parsePreviewOptions, + previewPlace, + type GeoapifyFeature, +} from '../src/data/geoapify.js' + +// Synthetic API fixtures only; these are not application seed data. +function feature(properties: Record): GeoapifyFeature { + return { type: 'Feature', properties } +} + +function collection(features: GeoapifyFeature[]) { + return { type: 'FeatureCollection', features } +} + +const place = feature({ + place_id: 'search-id', + name: 'Test pub', + categories: ['catering', 'catering.pub'], + city: 'Trondheim', + street: 'Test street', + housenumber: '4', + lat: 63.43, + lon: 10.39, + datasource: { raw: { osm_type: 'n', osm_id: 123 } }, +}) + +test('preview retains source identity, original records and unknown required fields', () => { + const result = previewPlace(place, null) + assert.equal(result.source.osmType, 'n') + assert.equal(result.source.osmId, '123') + assert.equal(result.availableFields.latitude, 63.43) + assert.equal(result.availableFields.longitude, 10.39) + assert.equal(result.availableFields.type, 'Pub') + assert.equal(result.availableFields.address, 'Test street 4') + assert.deepEqual(result.missingRequiredFields, ['area']) + assert.equal(result.availableFields.openingHoursRaw, null) + assert.equal(result.raw.place, place) +}) + +test('details can update the Geoapify identifier while retaining the same OSM object', () => { + const details = feature({ + place_id: 'details-id', + feature_type: 'details', + suburb: 'Midtbyen', + website: 'https://example.com', + opening_hours: 'Mo-Fr 16:00-01:00; Sa-Su off', + datasource: { raw: { osm_type: 'n', osm_id: 123 } }, + }) + const result = previewPlace(place, details) + assert.equal(result.source.geoapifyPlaceId, 'details-id') + assert.equal(result.source.osmId, '123') + assert.equal(result.availableFields.websiteUrl, 'https://example.com') + assert.equal( + result.availableFields.openingHoursRaw, + 'Mo-Fr 16:00-01:00; Sa-Su off', + ) + assert.deepEqual(result.missingRequiredFields, []) + assert.equal(result.raw.details, details) +}) + +test('missing values, invalid coordinates and missing source identifiers stay unknown', () => { + const result = previewPlace( + feature({ place_id: 'id', lat: 91, lon: Number.NaN }), + null, + ) + assert.deepEqual(result.missingRequiredFields, [ + 'name', + 'type', + 'city', + 'area', + 'address', + 'latitude', + 'longitude', + ]) + assert.equal(result.source.osmId, null) + assert.equal(result.source.osmType, null) +}) + +test('invalid API envelopes and missing place identifiers fail clearly', () => { + assert.throws(() => parseFeatures({ features: [] }), /FeatureCollection/) + assert.throws( + () => parseFeatures(collection([{} as GeoapifyFeature])), + /properties/, + ) + assert.throws(() => previewPlace(feature({}), null), /place_id/) + assert.deepEqual(parseFeatures(collection([])), []) +}) + +test('CLI rejects malformed limits and offsets before any API call', () => { + assert.deepEqual( + parsePreviewOptions(['--limit', '5', '--offset', '20', '--no-details']), + { limit: 5, offset: 20, details: false }, + ) + for (const args of [ + ['--limit', '0'], + ['--limit', '101'], + ['--offset', '-1'], + ['--limit', '1.5'], + ['--offset'], + ['--unknown'], + ]) { + assert.throws(() => parsePreviewOptions(args)) + } +}) + +test('CLI can request the Trondheim area without changing the sample size', () => { + assert.deepEqual(parsePreviewOptions(['--trondheim']), { + limit: 20, + offset: 0, + details: true, + trondheim: true, + }) +}) + +const trondheimArea = { + name: 'Trondheim', + city: 'Trondheim', + country_code: 'no', + result_type: 'city', + place_id: 'trondheim-area-id', + formatted: 'Trondheim, Trøndelag, Norge', +} + +test('Trondheim mode searches the resolved boundary and saves the chosen area', async () => { + const requests: { url: URL; init: RequestInit | undefined }[] = [] + const request: typeof fetch = async (input, init) => { + const url = new URL(String(input)) + requests.push({ url, init }) + return new Response( + JSON.stringify( + requests.length === 1 + ? { results: [trondheimArea] } + : collection([place]), + ), + ) + } + const result = await fetchPreview( + 'test-key', + { + limit: 20, + offset: 0, + details: false, + trondheim: true, + }, + request, + ) + assert.equal(requests.length, 2) + assert.equal(requests[0].url.searchParams.get('text'), 'Trondheim, Norway') + assert.equal(requests[0].url.searchParams.get('filter'), 'countrycode:no') + const placesQuery = JSON.parse(String(requests[1].init?.body)) + assert.deepEqual(placesQuery.filter, { + type: 'place', + id: 'trondheim-area-id', + }) + assert.deepEqual(result.search.filter, placesQuery.filter) + assert.equal(result.search.area?.formatted, 'Trondheim, Trøndelag, Norge') + assert.equal(result.places[0].availableFields.name, 'Test pub') + assert.ok(!JSON.stringify(result).includes('test-key')) +}) + +test('unexpected city lookup results stop before any place search', async () => { + for (const payload of [ + {}, + { results: [] }, + { results: [null] }, + { results: [{ ...trondheimArea, country_code: 'us' }] }, + { results: [{ ...trondheimArea, result_type: 'street' }] }, + { results: [{ ...trondheimArea, name: 'Oslo', city: 'Oslo' }] }, + { results: [{ ...trondheimArea, place_id: '' }] }, + ]) { + let calls = 0 + const request: typeof fetch = async () => { + calls += 1 + return new Response(JSON.stringify(payload)) + } + await assert.rejects( + fetchPreview( + 'test-key', + { + limit: 20, + offset: 0, + details: false, + trondheim: true, + }, + request, + ), + /valid city area/, + ) + assert.equal(calls, 1) + } +}) + +test('city lookup failures do not expose the credential-bearing request URL', async () => { + await assert.rejects( + fetchPreview( + 'test-key', + { + limit: 20, + offset: 0, + details: false, + trondheim: true, + }, + async (url) => { + throw new Error(String(url)) + }, + ), + (error: unknown) => { + assert.ok(error instanceof Error) + assert.match(error.message, /request failed/) + assert.ok(!error.message.includes('test-key')) + return true + }, + ) +}) + +test('fetch obtains details by ID and selects the details feature rather than the first feature', async () => { + const requests: { url: URL; init: RequestInit | undefined }[] = [] + const request: typeof fetch = async (input, init) => { + const url = new URL(String(input)) + requests.push({ url, init }) + return new Response( + JSON.stringify( + requests.length === 1 + ? collection([place]) + : collection([ + feature({ feature_type: 'building', name: 'Wrong building' }), + feature({ + feature_type: 'details', + name: 'Detailed pub', + website: 'https://example.com', + }), + ]), + ), + { status: 200 }, + ) + } + const { places: results } = await fetchPreview( + 'test-key', + { limit: 5, offset: 20, details: true }, + request, + ) + assert.equal(requests.length, 2) + const searchBody = JSON.parse(String(requests[0].init?.body)) + assert.equal(searchBody.filter.lon, 10.3951) + assert.equal(searchBody.filter.lat, 63.4305) + assert.equal(searchBody.limit, 5) + assert.equal(searchBody.offset, 20) + assert.equal(requests[1].url.searchParams.get('id'), 'search-id') + assert.equal(results[0].availableFields.name, 'Detailed pub') + assert.equal(results[0].availableFields.websiteUrl, 'https://example.com') + assert.ok(!JSON.stringify(results).includes('test-key')) +}) + +test('no-details mode and empty details responses retain available Places data', async () => { + let calls = 0 + const request: typeof fetch = async () => { + calls += 1 + return new Response( + JSON.stringify(collection(calls % 2 === 1 ? [place] : [])), + ) + } + const withDetails = await fetchPreview( + 'test-key', + { limit: 20, offset: 0, details: true }, + request, + ) + assert.equal(withDetails.places[0].raw.details, null) + assert.equal(withDetails.places[0].availableFields.name, 'Test pub') + const withoutDetails = await fetchPreview( + 'test-key', + { limit: 20, offset: 0, details: false }, + request, + ) + assert.equal(calls, 3) + assert.equal(withoutDetails.places[0].raw.details, null) +}) + +test('an empty Places response is a successful empty sample', async () => { + const request: typeof fetch = async () => + new Response(JSON.stringify(collection([]))) + assert.deepEqual( + ( + await fetchPreview( + 'test-key', + { limit: 20, offset: 0, details: true }, + request, + ) + ).places, + [], + ) +}) + +test('CLI accepts bounded pagination and rejects ambiguous or invalid page settings', () => { + assert.deepEqual( + parsePreviewOptions([ + '--trondheim', + '--all-pages', + '--limit', + '50', + '--max-pages', + '5', + ]), + { + limit: 50, + offset: 0, + details: true, + trondheim: true, + allPages: true, + maxPages: 5, + }, + ) + for (const args of [ + ['--all-pages', '--max-pages', '0'], + ['--all-pages', '--max-pages', '21'], + ['--max-pages', '5'], + ['--all-pages', '--offset', '20'], + ]) + assert.throws(() => parsePreviewOptions(args)) +}) + +test('pagination keeps the search fixed and deduplicates by OSM identity across pages', async () => { + const other = feature({ + ...place.properties, + place_id: 'other-id', + name: 'Other pub', + datasource: { raw: { osm_type: 'w', osm_id: 456 } }, + }) + const repeated = feature({ + ...place.properties, + place_id: 'changed-geoapify-id', + }) + const queries: Record[] = [] + const request: typeof fetch = async (input, init) => { + if (new URL(String(input)).pathname === '/v1/geocode/search') { + return new Response(JSON.stringify({ results: [trondheimArea] })) + } + queries.push(JSON.parse(String(init?.body))) + return new Response( + JSON.stringify( + collection(queries.length === 1 ? [place, other] : [repeated]), + ), + ) + } + const result = await fetchPreview( + 'test-key', + { + limit: 2, + offset: 0, + details: false, + trondheim: true, + allPages: true, + }, + request, + ) + assert.deepEqual( + queries.map((query) => query.offset), + [0, 2], + ) + assert.deepEqual(queries[0].filter, queries[1].filter) + assert.deepEqual(queries[0].categories, queries[1].categories) + assert.equal(result.places.length, 2) + assert.deepEqual(result.pagination, { + pagesFetched: 2, + recordsFetched: 3, + duplicatesSkipped: 1, + exhausted: true, + capped: false, + }) +}) + +test('hitting the page cap reports a partial collection and stops even when pages repeat', async () => { + let calls = 0 + const request: typeof fetch = async () => { + calls += 1 + return new Response(JSON.stringify(collection([place, place]))) + } + const result = await fetchPreview( + 'test-key', + { + limit: 2, + offset: 0, + details: false, + allPages: true, + maxPages: 2, + }, + request, + ) + assert.equal(calls, 2) + assert.equal(result.places.length, 1) + assert.equal(result.pagination.capped, true) + assert.equal(result.pagination.exhausted, false) + assert.equal(result.pagination.duplicatesSkipped, 3) +}) + +test('an empty page after a full page finishes retrieval without losing collected places', async () => { + let calls = 0 + const request: typeof fetch = async () => { + calls += 1 + return new Response(JSON.stringify(collection(calls === 1 ? [place] : []))) + } + const result = await fetchPreview( + 'test-key', + { + limit: 1, + offset: 0, + details: false, + allPages: true, + }, + request, + ) + assert.equal(calls, 2) + assert.equal(result.places.length, 1) + assert.equal(result.pagination.exhausted, true) + assert.equal(result.pagination.capped, false) +}) + +test('duplicate source records do not trigger repeated details requests', async () => { + let pages = 0 + let detailRequests = 0 + const repeated = feature({ + ...place.properties, + place_id: 'different-request-id', + }) + const request: typeof fetch = async (input) => { + if (new URL(String(input)).pathname === '/v2/place-details') { + detailRequests += 1 + return new Response( + JSON.stringify( + collection([ + feature({ + ...place.properties, + feature_type: 'details', + place_id: 'latest-id', + }), + ]), + ), + ) + } + pages += 1 + return new Response( + JSON.stringify(collection(pages === 1 ? [place, repeated] : [])), + ) + } + const result = await fetchPreview( + 'test-key', + { + limit: 2, + offset: 0, + details: true, + allPages: true, + }, + request, + ) + assert.equal(pages, 2) + assert.equal(detailRequests, 1) + assert.equal(result.places.length, 1) + assert.equal(result.places[0].source.geoapifyPlaceId, 'latest-id') + assert.equal(result.pagination.duplicatesSkipped, 1) +}) + +test('a later API failure is reported instead of returning a successful partial collection', async () => { + let calls = 0 + const request: typeof fetch = async () => { + calls += 1 + return calls === 1 + ? new Response(JSON.stringify(collection([place]))) + : new Response('quota error', { status: 429 }) + } + await assert.rejects( + fetchPreview( + 'test-key', + { + limit: 1, + offset: 0, + details: false, + allPages: true, + }, + request, + ), + /HTTP 429/, + ) + assert.equal(calls, 2) +}) + +test('API errors do not expose keys, response bodies or request URLs', async () => { + const options = { limit: 20, offset: 0, details: true } + await assert.rejects( + fetchPreview('test-key', options, async () => { + throw new Error('https://example.com?apiKey=test-key') + }), + /request failed/, + ) + await assert.rejects( + fetchPreview( + 'test-key', + options, + async () => new Response('test-key', { status: 429 }), + ), + /HTTP 429/, + ) + await assert.rejects( + fetchPreview('test-key', options, async () => new Response('test-key')), + /invalid JSON/, + ) + for (const key of ['', 'REPLACE_WITH_GEOAPIFY_KEY']) { + await assert.rejects( + fetchPreview(key, options, async () => { + assert.fail('Must not request without a key') + }), + /GEOAPIFY_API_KEY/, + ) + } +}) diff --git a/server/tests/openingHours.test.ts b/server/tests/openingHours.test.ts new file mode 100644 index 0000000..1fdd485 --- /dev/null +++ b/server/tests/openingHours.test.ts @@ -0,0 +1,143 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { normalizeOpeningHours, weekdays } from '../src/data/openingHours.js' + +test('missing hours remain unknown, with no fabricated closed days', () => { + for (const value of [null, '', ' ']) { + assert.deepEqual(normalizeOpeningHours(value), { + hours: [], + status: 'missing', + reviewReason: null, + warnings: [], + }) + } +}) + +test('a clear weekly schedule produces the agreed weekday names and closed-day entries', () => { + const result = normalizeOpeningHours('Mo-Fr 12:00-18:00; Sa 14:00-22:00') + assert.equal(result.status, 'normalized') + assert.deepEqual( + result.hours.map((entry) => entry.day), + [...weekdays], + ) + assert.deepEqual(result.hours[0], { + day: 'MONDAY', + opens: '12:00', + closes: '18:00', + closed: false, + }) + assert.deepEqual(result.hours[5], { + day: 'SATURDAY', + opens: '14:00', + closes: '22:00', + closed: false, + }) + assert.deepEqual(result.hours[6], { + day: 'SUNDAY', + opens: null, + closes: null, + closed: true, + }) +}) + +test('overnight hours split at midnight onto the correct calendar days', () => { + const result = normalizeOpeningHours('Mo 20:00-02:30') + assert.equal(result.status, 'normalized') + assert.deepEqual(result.hours.slice(0, 2), [ + { day: 'MONDAY', opens: '20:00', closes: '24:00', closed: false }, + { day: 'TUESDAY', opens: '00:00', closes: '02:30', closed: false }, + ]) + assert.equal(result.hours[2].closed, true) +}) + +test('Sunday overnight hours wrap into Monday without losing the spillover', () => { + const result = normalizeOpeningHours('Su 22:00-02:00') + assert.equal(result.status, 'normalized') + assert.deepEqual(result.hours[0], { + day: 'MONDAY', + opens: '00:00', + closes: '02:00', + closed: false, + }) + assert.deepEqual(result.hours.at(-1), { + day: 'SUNDAY', + opens: '22:00', + closes: '24:00', + closed: false, + }) +}) + +test('multiple intervals on a day are preserved without filling the break', () => { + const result = normalizeOpeningHours('Mo 10:00-12:00,14:00-18:00') + assert.equal(result.status, 'normalized') + assert.deepEqual( + result.hours.filter((entry) => entry.day === 'MONDAY'), + [ + { day: 'MONDAY', opens: '10:00', closes: '12:00', closed: false }, + { day: 'MONDAY', opens: '14:00', closes: '18:00', closed: false }, + ], + ) +}) + +test('24/7 produces seven full-day intervals', () => { + const result = normalizeOpeningHours('24/7') + assert.equal(result.status, 'normalized') + assert.deepEqual( + result.hours, + weekdays.map((day) => ({ + day, + opens: '00:00', + closes: '24:00', + closed: false, + })), + ) +}) + +test('explicitly closed schedules differ from missing hours', () => { + const result = normalizeOpeningHours('off') + assert.equal(result.status, 'normalized') + assert.deepEqual( + result.hours, + weekdays.map((day) => ({ day, opens: null, closes: null, closed: true })), + ) +}) + +test('holiday and seasonal rules are flagged rather than flattened into a misleading week', () => { + for (const value of [ + 'Mo-Fr 12:00-18:00; PH off', + 'Jan-Mar Mo-Fr 12:00-18:00', + ]) { + const result = normalizeOpeningHours(value) + assert.equal(result.status, 'needs-review') + assert.equal(result.reviewReason, 'variable-schedule') + assert.deepEqual(result.hours, []) + } +}) + +test('open-ended times do not gain invented closing times', () => { + const result = normalizeOpeningHours('Mo-Fr 12:00-00:00; Sa-Su 12:00+') + assert.equal(result.status, 'needs-review') + assert.equal(result.reviewReason, 'uncertain-interval') + assert.deepEqual(result.hours, []) +}) + +test('warnings about overlapping overnight day rules require review', () => { + const result = normalizeOpeningHours( + 'Mo-Th 15:00-23:00; Fr 15:00-01:00; Sa 14:00-01:00', + ) + assert.equal(result.status, 'needs-review') + assert.equal(result.reviewReason, 'parser-warning') + assert.ok(result.warnings.length > 0) + assert.deepEqual(result.hours, []) +}) + +test('invalid free text and annotated hours do not become unqualified opening intervals', () => { + const invalid = normalizeOpeningHours('weekend until 01.00') + assert.equal(invalid.reviewReason, 'invalid-expression') + assert.deepEqual(invalid.hours, []) + const annotated = normalizeOpeningHours( + 'Mo 10:00-12:00 open "appointment only"', + ) + assert.equal(annotated.status, 'needs-review') + assert.deepEqual(annotated.hours, []) +}) diff --git a/server/tests/prepareVenues.test.ts b/server/tests/prepareVenues.test.ts new file mode 100644 index 0000000..25b7c37 --- /dev/null +++ b/server/tests/prepareVenues.test.ts @@ -0,0 +1,192 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + isCompletedTrondheimExport, + prepareVenueDrafts, +} from '../src/data/prepareVenues.js' + +function place() { + return { + source: { osmType: 'n', osmId: '123', geoapifyPlaceId: 'test-id' }, + availableFields: { + name: 'Test pub', + type: 'Pub', + city: 'Trondheim', + area: 'Midtbyen', + address: 'Test street 4', + latitude: 63.43, + longitude: 10.39, + description: null as string | null, + websiteUrl: 'https://example.com' as string | null, + openingHoursRaw: 'Mo-Sa 20:00-02:30' as string | null, + categories: ['catering', 'catering.pub'], + }, + } +} + +function sourceExport(places = [place()]) { + return { + version: 1, + provider: 'Geoapify', + search: { trondheim: true, allPages: true, details: true, offset: 0 }, + pagination: { exhausted: true, capped: false }, + places, + } +} + +test('preparation preserves known fields and separates pending hours from Venue fields', () => { + const source = sourceExport() + const before = JSON.stringify(source) + const draft = prepareVenueDrafts(source)[0] + assert.equal(draft.sourceKey, 'openstreetmap:n:123') + assert.equal(draft.venue.name, 'Test pub') + assert.equal(draft.venue.latitude, 63.43) + assert.equal(draft.venue.longitude, 10.39) + assert.equal(draft.venue.priceLevel, null) + assert.equal(draft.venue.rating, null) + assert.equal(draft.venue.description, null) + assert.equal(draft.venue.imageUrl, null) + assert.equal(draft.venue.reviewCount, 0) + assert.equal(draft.venue.websiteUrl, 'https://example.com/') + assert.deepEqual(draft.venue.openingHours, []) + assert.equal(draft.openingHoursRaw, 'Mo-Sa 20:00-02:30') + assert.equal(draft.openingHoursStatus, 'pending') + assert.ok(!Object.hasOwn(draft.venue, 'id')) + assert.equal(JSON.stringify(source), before) +}) + +test('missing optional data stays unknown instead of becoming fabricated hours or tags', () => { + const item = place() + item.availableFields.websiteUrl = null + item.availableFields.openingHoursRaw = null + const draft = prepareVenueDrafts(sourceExport([item]))[0] + assert.equal(draft.venue.websiteUrl, null) + assert.equal(draft.openingHoursRaw, null) + assert.equal(draft.openingHoursStatus, 'missing') + assert.deepEqual(draft.venue.openingHours, []) + assert.deepEqual(draft.venue.tags, []) +}) + +test('the opening-hours option populates Venue hours while retaining source text and status', () => { + const item = place() + item.availableFields.openingHoursRaw = 'Mo 20:00-02:30' + const result = prepareVenueDrafts(sourceExport([item]), { + openingHours: true, + })[0] + assert.equal(result.openingHoursStatus, 'normalized') + assert.equal(result.openingHoursRaw, 'Mo 20:00-02:30') + assert.equal(result.openingHoursReviewReason, null) + assert.deepEqual(result.venue.openingHours.slice(0, 2), [ + { day: 'MONDAY', opens: '20:00', closes: '24:00', closed: false }, + { day: 'TUESDAY', opens: '00:00', closes: '02:30', closed: false }, + ]) +}) + +test('tags come only from recognized source categories and are deduplicated', () => { + const item = place() + item.availableFields.categories = [ + 'catering.pub', + 'wheelchair', + 'wheelchair.yes', + 'internet_access', + 'internet_access', + 'unknown.category', + ] + assert.deepEqual(prepareVenueDrafts(sourceExport([item]))[0].venue.tags, [ + 'Rullestoltilgjengelig', + 'Internettilgang', + ]) +}) + +test('samples and incomplete or details-free exports cannot become the selected catalog', () => { + for (const source of [ + {}, + sourceExport([]), + { + ...sourceExport(), + search: { ...sourceExport().search, allPages: false }, + }, + { ...sourceExport(), search: { ...sourceExport().search, details: false } }, + { ...sourceExport(), search: { ...sourceExport().search, offset: 20 } }, + { ...sourceExport(), pagination: { exhausted: false, capped: true } }, + ]) + assert.throws(() => prepareVenueDrafts(source)) + assert.equal(isCompletedTrondheimExport(sourceExport()), true) +}) + +test('missing required fields, wrong cities and unsupported types fail instead of being filled in', () => { + for (const field of ['name', 'area', 'address']) { + const item = place() + Object.assign(item.availableFields, { [field]: ' ' }) + assert.throws( + () => prepareVenueDrafts(sourceExport([item])), + new RegExp(field), + ) + } + const wrongCity = place() + wrongCity.availableFields.city = 'Oslo' + assert.throws( + () => prepareVenueDrafts(sourceExport([wrongCity])), + /expected city/, + ) + const wrongType = place() + wrongType.availableFields.type = 'Unknown' + assert.throws( + () => prepareVenueDrafts(sourceExport([wrongType])), + /venue type/, + ) +}) + +test('out-of-range and non-finite coordinates fail validation', () => { + for (const [field, value] of [ + ['latitude', 91], + ['longitude', -181], + ['latitude', Number.NaN], + ['longitude', Number.POSITIVE_INFINITY], + ]) { + const item = place() + Object.assign(item.availableFields, { [String(field)]: value }) + assert.throws( + () => prepareVenueDrafts(sourceExport([item])), + /invalid (latitude|longitude)/, + ) + } +}) + +test('OSM identity is stable across changing Geoapify IDs and qualified by object type', () => { + const first = place() + const changed = place() + changed.source.geoapifyPlaceId = 'changed-id' + assert.equal( + prepareVenueDrafts(sourceExport([first]))[0].sourceKey, + prepareVenueDrafts(sourceExport([changed]))[0].sourceKey, + ) + assert.throws( + () => prepareVenueDrafts(sourceExport([first, changed])), + /Duplicate source/, + ) + changed.source.osmType = 'w' + assert.equal(prepareVenueDrafts(sourceExport([first, changed])).length, 2) + changed.source.osmId = '' + assert.throws( + () => prepareVenueDrafts(sourceExport([changed])), + /source identity/, + ) +}) + +test('website normalization accepts HTTP and rejects unsafe or malformed URLs', () => { + const item = place() + item.availableFields.websiteUrl = 'http://example.com' + assert.equal( + prepareVenueDrafts(sourceExport([item]))[0].venue.websiteUrl, + 'http://example.com/', + ) + for (const url of [ + 'not-a-url', + 'javascript:alert(1)', + 'https://user:password@example.com', + ]) { + item.availableFields.websiteUrl = url + assert.throws(() => prepareVenueDrafts(sourceExport([item])), /website URL/) + } +}) From 352aeb3fe4a9e21166aae8966ba4140896e7c1e2 Mon Sep 17 00:00:00 2001 From: "Andreas S. Steffensen" Date: Tue, 13 Oct 2026 21:51:54 +0200 Subject: [PATCH 2/2] feat: validate venue imports and complete the Venue model --- .github/workflows/backend-quality.yml | 35 ++ README.md | 5 +- docs/geoapify-data-import.md | 326 +++++++----------- docs/venue-model.md | 122 +++++++ server/README.md | 259 ++++++-------- .../1791907200000_complete-venue-model.js | 48 +++ server/package-lock.json | 8 + server/package.json | 6 +- server/scripts/seed-venues.ts | 279 ++++----------- server/src/data/exportVenues.ts | 40 +++ server/src/data/importVenues.ts | 96 ++++++ server/src/data/prepareVenues.ts | 2 +- server/src/data/sourceExports.ts | 30 ++ server/src/data/venueSeed.ts | 204 +++++++++++ server/src/db/pool.ts | 19 +- server/src/graphql/schema.ts | 47 ++- server/src/index.ts | 34 +- server/src/scripts/checkDatabase.ts | 31 ++ server/src/scripts/exportVenues.ts | 46 +++ server/tests/database.test.ts | 275 +++++++++++++++ server/tests/exportVenues.test.ts | 56 +++ server/tests/fixtures.ts | 62 ++++ server/tests/venueSeed.test.ts | 196 +++++++++++ server/tsconfig.check.json | 10 + 24 files changed, 1604 insertions(+), 632 deletions(-) create mode 100644 .github/workflows/backend-quality.yml create mode 100644 docs/venue-model.md create mode 100644 server/migrations/1791907200000_complete-venue-model.js create mode 100644 server/src/data/exportVenues.ts create mode 100644 server/src/data/importVenues.ts create mode 100644 server/src/data/sourceExports.ts create mode 100644 server/src/data/venueSeed.ts create mode 100644 server/src/scripts/checkDatabase.ts create mode 100644 server/src/scripts/exportVenues.ts create mode 100644 server/tests/database.test.ts create mode 100644 server/tests/exportVenues.test.ts create mode 100644 server/tests/fixtures.ts create mode 100644 server/tests/venueSeed.test.ts create mode 100644 server/tsconfig.check.json diff --git a/.github/workflows/backend-quality.yml b/.github/workflows/backend-quality.yml new file mode 100644 index 0000000..b5f7f3d --- /dev/null +++ b/.github/workflows/backend-quality.yml @@ -0,0 +1,35 @@ +name: Backend quality gate + +on: + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + backend-checks: + name: Backend checks + runs-on: [self-hosted, linux, x64, t07-project-2] + timeout-minutes: 15 + defaults: + run: + working-directory: server + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + cache-dependency-path: server/package-lock.json + - name: Install dependencies + run: npm ci + - name: Check types including scripts and tests + run: npm run typecheck + - name: Run data, migration and GraphQL tests + run: npm run test:data + - name: Build backend + run: npm run build diff --git a/README.md b/README.md index eabc214..cc606c4 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,9 @@ Se [server/README.md](server/README.md) for npm-oppsett, VM-konfigurasjon, lokal testing, databasebruker og migrasjoner. PostgreSQL og den planlagte backendtjenesten kjører direkte på VM-en; Docker brukes ikke. -Se [Geoapify data import](docs/geoapify-data-import.md) for å hente et lite utvalg -ekte utesteder, undersøke tilgjengelige felt og forberede databaseimporten. +See [Venue V1](docs/venue-model.md) for the shared data contract and +[Geoapify data import](docs/geoapify-data-import.md) for retrieval, validation +and database import. Backend checks run on pull requests alongside frontend checks. ## Frontend-deploy diff --git a/docs/geoapify-data-import.md b/docs/geoapify-data-import.md index 9487c14..ee1a9d6 100644 --- a/docs/geoapify-data-import.md +++ b/docs/geoapify-data-import.md @@ -1,20 +1,14 @@ # Geoapify data import -The intended flow is **Geoapify → import script → PostgreSQL on the team VM → -our GraphQL API → frontend**. Importing is a separate operation; browsing the -application should query our database rather than call Geoapify for every visit. +The flow is **Geoapify → local export → PostgreSQL on the team VM → our GraphQL +API → frontend**. Browsing the app should query our database. Retrieval and +refreshing are separate operations, rather than calls to Geoapify on every visit. -## What is available now +[Venue V1](venue-model.md) is the shared field/identity/opening-hours contract. +[Backend setup](../server/README.md) explains credentials, SSH and migrations. +The frontend still uses mock places; its GraphQL integration is separate work. -- PostgreSQL connection code and migration tooling exist in `server/`. -- The implemented GraphQL API currently exposes only `health` and `databaseStatus`. -- The agreed Venue V1 contract is a draft; Venue SQL tables and resolvers are not - implemented in the current branch. -- The frontend still reads mock places. -- `data:preview` fetches a sample and saves original source records locally. It - does not write to PostgreSQL or expose venues through GraphQL. - -## First step: inspect real source data +## 1. Configure retrieval From the repository root: @@ -23,243 +17,163 @@ npm --prefix server ci Copy-Item .env.example server/.env ``` -Only copy the example if `server/.env` does not already exist. Obtain a key from -[Geoapify MyProjects](https://myprojects.geoapify.com/) and replace -`GEOAPIFY_API_KEY=REPLACE_WITH_GEOAPIFY_KEY` in `server/.env`. A database URL is -not needed for this step. Never put the real key in the example file. +Only copy the example if the file does not exist. Add your key from +[Geoapify MyProjects](https://myprojects.geoapify.com/) as `GEOAPIFY_API_KEY` in +`server/.env`. Never commit it. Database access is not needed for steps 1–4. + +## 2. Inspect a sample, then collect query pages ```powershell -npm --prefix server run data:preview -- --limit 20 +npm --prefix server run data:preview -- --trondheim --limit 20 +npm --prefix server run data:preview -- --trondheim --all-pages --limit 50 ``` -This requests one page of bars, pubs and nightclubs within 5 km of -`longitude=10.3951, latitude=63.4305` in central Trondheim. It then requests -`details` for each returned place to obtain available opening hours, website and -source identifiers. The default run makes at most 21 HTTP requests; it is a -sample, not a complete city catalog. Requests stop on an error, and a JSON file -is written only after the entire page succeeds. +Trondheim mode resolves a Norwegian city result once, validates its identity, +then uses its place ID as the boundary filter. Categories are `catering.bar`, +`catering.pub` and `adult.nightclub`. Review the selected boundary and coverage; +finishing pagination cannot guarantee every real venue is mapped. -The output in `server/imports/` includes the retrieval time, search parameters, -source identifiers, original place/detail records and a preview of available -Venue fields. The directory is ignored by Git. No API key is included in the -export. The console lists missing required location fields; this is a coverage -report, not full Venue validation. +The sample makes at most 22 requests with details enabled. Collection keeps +categories and area unchanged and fetches details for unique source identities. +It stops on a short/empty page. The default cap is 10 pages; `--max-pages` accepts +1–20 with `--all-pages`. A capped export is marked partial and exits with code 2. +Failures do not produce a successful partial file. Repeated OSM identities are +skipped, using type plus ID; Geoapify IDs are a fallback during preview only. -Additional commands: +All exports are retained as unique `server/imports/geoapify-*.json` files. This +directory is ignored by Git. Files contain retrieval time, search parameters, +attribution, original place/details and available fields, without the API key. +The console table shows coverage rather than every source field. -```powershell -# Fetch the next sample page, with the same search area. -npm --prefix server run data:preview -- --limit 20 --offset 20 - -# Fetch only the Places page, without per-place detail requests. -npm --prefix server run data:preview -- --limit 20 --no-details +Without `--trondheim`, preview uses a 5 km circle around 10.3951/63.4305. +`--offset`, `--no-details` and `--help` support inspection. Samples, capped files, +nonzero-offset files and details-free exports cannot become the completed seed. -# Show usage without a key or database connection. -npm --prefix server run data:preview -- --help -``` - -### Step 1 toward a Trondheim dataset: check the area boundary +## 3. Inspect intermediate Venue drafts (optional) ```powershell -npm --prefix server run data:preview -- --trondheim --limit 20 +npm --prefix server run data:prepare +npm --prefix server run data:prepare -- --opening-hours ``` -`--trondheim` first resolves `Trondheim, Norway` with the Geocoding API, -restricted to Norwegian city results. It rejects a result without a Trondheim -name, Norwegian country code, city result type or place ID. That place ID is -then used as the Places API boundary filter instead of the 5 km radius. - -The console prints the resolved area, and the export saves `search.filter` and -`search.area`, including the selected geocoding record. This run still requests -only one sample page; it does not fetch every venue in Trondheim. It makes at -most 22 HTTP requests with the default limit and details enabled. Verify the -resolved area's coverage before expanding retrieval. A source boundary or -category search cannot guarantee that every real venue is mapped. +Preparation selects the latest completed Trondheim collection. The first +command keeps opening hours pending; the second uses the pinned `opening_hours` +parser. Each writes a separate draft file without changing the source export. +These nested drafts are intermediate files, not seed input. -Keep the sample limit at 20 while checking the area. Database writes are separate. +Required strings, coordinates, OSM identities and URLs are validated. Unknown +price, rating, description and image values remain null. Initial review count +is 0 for our app, not an external review count. Recognized source categories +become factual tags; generic categories do not invent tags. -### Step 2: collect the remaining query pages +## 4. Export and validate the database input ```powershell -npm --prefix server run data:preview -- --trondheim --all-pages --limit 50 +npm --prefix server run data:export ``` -This starts a fresh collection at offset 0, resolves the area once and requests -consecutive pages of 50 records. It keeps the category/area query unchanged and -stops on a short or empty page. It fetches details for unique places, skips -duplicates by OSM type/ID (or Geoapify ID when OSM identity is missing) and saves -one combined JSON file. Existing exports are retained. - -The default cap is 10 pages, or at most 500 source records with this command. -`--max-pages` accepts 1–20 and requires `--all-pages`. Do not combine -`--all-pages` with a nonzero `--offset`. The console reports progress per page, -and `pagination` in the export records page counts, duplicate counts and whether -a short/empty page ended retrieval. If the cap is reached, `pagination.capped` -is true, the output is explicitly marked partial and the command exits with -code 2. Increase the cap only if needed; a partial export must not be treated as -the finished query. - -This collects the API's results for the selected categories and area. It cannot -guarantee that every real venue is mapped or that the provider returns a complete -catalog. A later request failure stops the run without writing a new export. - -### Step 3: prepare the Venue fields locally +This reads the latest completed source export, converts clear weekly opening +hours and produces a flat JSON array in `server/imports/venue-seed-*.json`. +It makes no API calls or database writes. The console prints the exact path and +a command for a dry run. You can also select the newest generated file: ```powershell -npm --prefix server run data:prepare +$venueSeedFile = Get-ChildItem server/imports/venue-seed-*.json | Sort-Object LastWriteTime -Descending | Select-Object -First 1 +npm --prefix server run seed:venues -- --file $venueSeedFile.FullName --dry-run ``` -This selects the newest completed Trondheim export with all pages and details -enabled. Sample pages, capped exports and details-free exports are skipped. -It validates required fields, coordinate bounds, source identities and website -URLs, then writes a separate `venue-drafts-*.json` file. It makes no API calls -and does not connect to PostgreSQL. The original export remains unchanged. +Each row contains the Venue fields except the database-assigned `id`, plus +`sourceId`, `sourceFetchedAt` and `rawData`. Source types map to `BAR`, `PUB` and +`NIGHTCLUB`. `sourceId` is `openstreetmap::`, independent of a +changing Geoapify place ID. `rawData` retains attribution, source identity, +original place/details, query area, original hours, parser status and warnings. + +Missing/uncertain/unsupported opening hours remain `[]` and can be imported as +unknown. They do not become closed days. Clear week-stable schedules use +calendar days in Europe/Oslo; overnight hours are split at midnight and repeated +weekdays are allowed. See the model document before implementing opening status. +Manually reviewing flagged expressions is a later data-quality task; retain the +original expression and use verified evidence before changing the schedule. + +Validation rejects out-of-range coordinates, invalid clock times, overlaps, +contradictory closed/open entries, unsafe URL schemes/credentials, duplicate +source IDs and invalid price levels. Imported ratings must be null and counts +must be 0 or omitted; provider imports cannot supply app review aggregates. -Each draft contains a `venue` object with the agreed field names except `id`, -which the database must assign. Unknown price level, rating, description and -image values remain null. `reviewCount: 0` is an initial app counter, not a claim -about external reviews; an eventual importer must preserve existing user data. -Recognized source categories become factual tags such as `Internettilgang` or -`Rullestoltilgjengelig`; generic or unknown categories do not invent tags. +A collection on 2026-10-13 yielded 51 venues: 19 bars, 26 pubs and 6 nightclubs. +All had required location fields and stable OSM identity; 34 had websites. +Opening hours normalized for 24, were missing for 19 and required review for 8. +These are observed counts for that collection, not guaranteed future counts. -Opening hours are intentionally still an intermediate step. `openingHoursRaw` -retains the source text and `openingHoursStatus` is `pending` or `missing`. -The draft's `openingHours` list is empty until conversion is implemented. The -file is marked `readyForDatabaseImport: false`; it is not yet a complete API -Venue or the final database import payload. +## 5. Connect, migrate and import -### Step 4: convert opening hours +Get database access from the database owner privately. Keep an SSH tunnel open +for local access and configure `server/.env` for `tapfinder_dev`, as described +in the backend README. Confirm access without writes: ```powershell -npm --prefix server run data:prepare -- --opening-hours +npm --prefix server run db:check ``` -This uses the pinned `opening_hours` library to parse the source's OSM syntax -offline. Clear, week-stable schedules become `OpeningHours` entries. Missing -source hours remain an empty list with status `missing`. Date-dependent schedules, -uncertain closing times, parser warnings and annotations remain an empty list -with status `needs-review`, a reason and the original text. We do not invent -closing times or silently repair source rules. The console lists review cases. - -The result is a new `venue-hours-drafts-*.json` file. Other fields and the source -identity retain the same format. Original exports and earlier drafts are retained. -`readyForDatabaseImport` remains false until database compatibility and review -handling are settled. - -Intervals represent **calendar days in venue-local time (Europe/Oslo)**: - -- Monday `20:00–02:30` becomes Monday `20:00–24:00` plus Tuesday `00:00–02:30`. -- A day can have multiple entries, preserving breaks and morning/night intervals. -- A completely closed day has `closed: true`, `opens: null`, `closes: null`. -- Unknown hours use `[]`; they do not become seven closed days. -- `24:00` means the end of the named day; intervals have `opens < closes`. - -The existing GraphQL list shape permits multiple entries per weekday. Database -validation and frontend opening-status logic must use the same calendar-day -convention and allow these repeated weekdays. A unique `(venue, day)` constraint -would prevent multiple intervals and must not be assumed for this representation. - -The parser evaluates a fixed January reference week after checking that the -expression is week-stable. Only clock strings are exported; no reference dates -are stored as actual opening dates. Holiday exceptions and other variable rules -require an additional representation or an explicit unknown-hours fallback. - -## Mapping to the agreed Venue contract - -| Venue field | Source / handling | -| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | -| `id` | Our own persistent database ID, assigned on first import. | -| `name` | Geoapify `name`; report missing names. | -| `type` | Initially map `catering.bar`, `catering.pub`, `adult.nightclub` to `Bar`, `Pub`, `Nattklubb`. Expand after inspecting the source. | -| `city` | Geoapify `city`; do not infer it from a search radius. | -| `area` | Available `suburb` or `district`; report missing values. | -| `address` | Available street and house number; report a missing street. | -| `latitude`, `longitude` | Geoapify `lat`, `lon`; validate coordinate ranges. | -| `priceLevel` | Not provided by the documented API contract; leave unknown until curated or sourced elsewhere. | -| `rating` | Not provided by the documented API contract; leave unknown until app reviews or another source exist. | -| `reviewCount` | Count our app's reviews; initially zero when a venue has no app reviews. Do not manufacture external review counts. | -| `description` | Available `description`, otherwise null. | -| `openingHours` | Preserve source `opening_hours` for later parsing. Missing hours mean unknown, represented by an empty list in V1; never fabricate closed days. | -| `tags` | Preserve source categories in the preview; choose user-facing tags during normalization. | -| `imageUrl` | Null until an image source has been selected; raw media references remain in the saved response. | -| `websiteUrl` | Available `website`, otherwise null. | - -Geoapify opening hours use OpenStreetMap syntax. Each V1 `OpeningHours` entry -stores one interval; the list can represent multiple intervals on the same day -if consumers support repeated weekdays. Source expressions can also contain -holiday exceptions or open-ended times, which V1 does not represent completely. -Use a suitable parser during normalization and retain the original expression. -Unsupported expressions must remain unknown rather than be misrepresented. - -If required fields such as `area` or `address` are missing, curate them from a -verified source or keep the record out of the public Venue list. Do not satisfy -GraphQL non-null fields with invented values. The preview makes these gaps visible -before database import. - -## Next step: import into PostgreSQL - -1. Reuse any Venue migration/backend branch already in progress. The GraphQL - draft alone does not create a SQL table. Store the 16 public fields plus - internal source identity, original opening-hours expression and refresh time. -2. Use our own persistent Venue ID and a unique external source identity. - Prefer the pair `osm_type` + `osm_id` when present; retain the current - Geoapify place ID for further API requests. A Geoapify ID can change for the - same OSM object. Flag records without reliable identity for review. -3. Normalize the inspected source records and validate required fields. An - importer should insert new venues and update existing source fields using - `ON CONFLICT ... DO UPDATE`, preserving internal IDs and user-generated data. - Reimporting the same source should not create duplicates or reset reviews. -4. Apply the reviewed migration, then import a small sample in a transaction. - Compare row counts and source identities, then rerun the sample to check - duplicate handling. A partial search must never delete absent venues. -5. Implement Venue resolvers against PostgreSQL and replace the frontend's mock - API. Add backend search, filters, sorting and pagination before expanding - the catalog. Schedule refreshes after the manual workflow is verified. - -When importing from a local computer, keep this SSH tunnel open: +After the reviewed migrations are available, apply them, then import the exact +validated seed file: ```powershell -ssh -N -L 5433:127.0.0.1:5432 YOUR_VM_USER@it2810-07.idi.ntnu.no +npm --prefix server run migrate:up +npm --prefix server run seed:venues -- --file $venueSeedFile.FullName ``` -Set `DATABASE_URL` in `server/.env` to the existing VM database, using local -`127.0.0.1:5433`. On the VM itself use `127.0.0.1:5432`. Ask the database owner -for the actual database/user and credentials through the group's normal private -channel. The documented defaults are database `tapfinder`, user `tapfinder_app`; -their existence and access have not been verified by this preview tool. +Imports reject other database names and placeholder credentials. Validation +happens before connecting. Each batch is a transaction: an error rolls back +both Venue and source changes. Run one import at a time. Repeating an import +updates existing source mappings, preserving internal IDs, app ratings and +review counts. Unknown imported price/description/image values preserve existing +curated values. Source retrieval time is stored separately from database update +time. Missing venues are never deleted just because they are absent from a file. + +The default `server/data/venues.json` is still an empty list. Manual files with +the original flat fields remain supported; new optional fields can be omitted. +Manual open intervals must use the same calendar-day convention, including +midnight splits. File paths passed to `--file` are relative to `server/` or +absolute. Keep real exports out of Git; retain the original files for audit. + +Use the model document's GraphQL query to verify all fields after import. Check +counts and rerun the same file to confirm zero new rows and preserved IDs. VM +access, migrations and import must still be verified on the actual environment. -## Tests and attribution +## Tests and remaining integration ```powershell +npm --prefix server run typecheck npm --prefix server run test:data npm --prefix server run build ``` -The preview unit tests use Node's test runner and synthetic API responses; they -need neither a key nor PostgreSQL. They cover missing values, coordinate bounds, -OSM identity preservation, details lookup, pagination inputs, empty results and -API errors, including Trondheim boundary selection and rejection of unexpected -city lookup results. Pagination tests cover fixed search parameters, duplicate -source identities, short/empty pages, page caps and failures on later pages. -Live data coverage and database behavior require separate checks. -Preparation tests cover required and optional fields, source identity, -duplicate rejection, tag mapping, coordinates and website normalization. -Opening-hours tests cover weekday mapping, midnight and Sunday carryover, -multiple intervals, 24/7, closed versus unknown days and unsupported expressions. -The existing frontend uses Vitest; application/API testing remains a later step. - -Retain source attribution with imported records. Display OpenStreetMap -attribution when showing its data, and Geoapify attribution as required by the -API plan. Follow the provider's current terms when publishing the catalog. - -This preview implementation and its unit tests were prepared with Codex. The -Trondheim boundary lookup was checked against Geoapify with one live Places -result on 2026-10-13. Database writes have not been performed. +Retrieval tests use mocked API responses. PGlite runs migrations, constraints, +transaction rollback, repeat import and GraphQL tests against PostgreSQL in +memory. No API key, VM, Docker or external test database is needed. Real-data +coverage and real VM connectivity are separate operational checks. + +Remaining application work includes frontend GraphQL integration with source +attribution, backend search/filter/sort/pagination, app review persistence and +scheduled refreshing after the manual workflow is verified. + +Retain **Powered by Geoapify | © OpenStreetMap contributors** with imported +records and show the required attribution when publishing the data. Current +provider terms and the API plan govern distribution. Raw source metadata is +internal and is not exposed by the Venue API. + +The implementation, tests and documentation were prepared with Codex. Source +retrieval was checked against Geoapify; database behavior was checked locally +without writing to the VM database. Primary documentation: - [Places API](https://apidocs.geoapify.com/docs/places/index.html) - [Place Details API](https://apidocs.geoapify.com/docs/place-details/) -- [Source identifiers and optional details](https://apidocs.geoapify.com/how-to/place-discovery/place-details-by-id-or-coordinates) +- [Source identity and optional fields](https://apidocs.geoapify.com/how-to/place-discovery/place-details-by-id-or-coordinates) +- [City boundary searches](https://apidocs.geoapify.com/how-to/place-discovery/places-in-city-district-postcode/) +- [Pagination](https://apidocs.geoapify.com/how-to/place-discovery/paginate-place-results/) - [Storage and attribution](https://www.geoapify.com/places-api/) - [Opening-hours parser](https://github.com/opening-hours/opening_hours.js) +- [PGlite test database](https://pglite.dev/docs/) diff --git a/docs/venue-model.md b/docs/venue-model.md new file mode 100644 index 0000000..7a6dde4 --- /dev/null +++ b/docs/venue-model.md @@ -0,0 +1,122 @@ +# Venue V1 contract + +This document defines the public model shared by the PostgreSQL backend and +GraphQL consumers. The schema is implemented in `server/src/graphql/schema.ts`. +The original table migration is retained; `complete-venue-model` adds the five +remaining fields to existing databases without replacing records or IDs. + +| GraphQL field | PostgreSQL column | Meaning | +| -------------------------------- | ----------------- | ----------------------------------------------------- | +| `id: ID!` | `id` | Opaque, persistent UUID generated by our database. | +| `name: String!` | `name` | Verified venue name. | +| `type: String!` | `type` | `BAR`, `PUB` or `NIGHTCLUB`. | +| `city: String!` | `city` | Source city; currently Trondheim. | +| `area: String!` | `area` | Source suburb or district. | +| `address: String!` | `address` | Street address. | +| `latitude: Float!` | `latitude` | -90 to 90. | +| `longitude: Float!` | `longitude` | -180 to 180. | +| `priceLevel: Int` | `price_level` | 1–3 (`$`, `$$`, `$$$`); null when unknown. | +| `rating: Float` | `rating` | App review aggregate, 0–5; null before reviews exist. | +| `reviewCount: Int!` | `review_count` | Number of app reviews; initially 0. | +| `description: String` | `description` | Verified description, otherwise null. | +| `openingHours: [OpeningHours!]!` | `opening_hours` | Calendar-day intervals; `[]` means unknown. | +| `tags: [String!]!` | `tags` | Factual, normalized tags; `[]` when none are known. | +| `imageUrl: String` | `image_url` | HTTP(S) image address, otherwise null. | +| `websiteUrl: String` | `website_url` | HTTP(S) venue website, otherwise null. | + +Geoapify does not supply the app's review aggregate. Imports do not update +`rating` or `review_count`. A new record receives their database defaults. The +review-writing feature and aggregate calculation are separate future work. +Unknown imported price, description and image values do not erase existing +curated values. Clearing those values explicitly will belong to an editing API. + +## Opening hours + +```graphql +enum Weekday { + MONDAY + TUESDAY + WEDNESDAY + THURSDAY + FRIDAY + SATURDAY + SUNDAY +} +type OpeningHours { + day: Weekday! + opens: String + closes: String + closed: Boolean! +} +``` + +Times describe **calendar days in Europe/Oslo**, using `HH:mm`. Opening times +range from `00:00` to `23:59`; closing times may also be `24:00`, the end of the +named day. Open intervals require `opens < closes`. For example, Monday +20:00–02:30 is stored as Monday 20:00–24:00 and Tuesday 00:00–02:30. + +Multiple intervals on the same weekday are allowed, with no overlaps. A closed +entry has `closed: true` and both times null; it cannot coexist with an open +entry on that day. An empty list means unknown, not closed. A partial manually +prepared list leaves omitted days unknown. A normalized weekly source schedule +contains all seven days, including explicit closed-day entries. + +The importer does not flatten seasonal rules, holiday exceptions, uncertain +closing times or parser warnings into a misleading recurring week. Such records +use `openingHours: []`; original text and review reasons remain in source data. +Frontend opening-status logic must support repeated weekdays and `24:00`. + +## Source identity and refreshes + +`venue_sources` has a unique `(provider, external_id)` mapping to `venues.id`. +The Geoapify exporter uses `provider = geoapify` and +`external_id = openstreetmap::`. The object type is part of the +identity: node 123 and way 123 are different places. Current Geoapify place IDs +are retained in `raw_data`, since they may change for the same OSM object. + +`raw_data` retains attribution, query area, original place/detail records, +opening-hours expression, normalization status and review reason. `fetched_at` +stores source retrieval time; `venues.updated_at` stores database update time. +Legacy manually prepared seeds without a retrieval timestamp use import time. +An import without raw data preserves any previously stored raw data. + +Run one import at a time. Each file is validated before connecting to PostgreSQL +and written in one transaction. Reimporting updates source fields through the +existing mapping and preserves internal IDs and app reviews. Records absent +from an import are not deleted. Missing required fields must be verified or the +record excluded; non-null fields must not be filled with invented values. + +## API + +The initial queries are `health`, `databaseStatus`, `venues` and `venue(id: ID!)`. +List results are ordered by name and ID. Unknown or malformed IDs return null. +Search, filtering, sorting arguments and pagination are still separate work; +the current list query should not be presented as a completed scalable API. + +```graphql +query { + venues { + id + name + type + city + area + address + latitude + longitude + priceLevel + rating + reviewCount + description + openingHours { + day + opens + closes + closed + } + tags + imageUrl + websiteUrl + } +} +``` diff --git a/server/README.md b/server/README.md index 04e9c00..3876b71 100644 --- a/server/README.md +++ b/server/README.md @@ -1,47 +1,60 @@ -# Backend og database +# Backend and database -Backenden er et TypeScript-prosjekt med Apollo Server (GraphQL) og PostgreSQL. -Både backend og PostgreSQL kjøres direkte på prosjektets VM, uten Docker. -Dette oppsettet inneholder første venue-skjema, GraphQL-lesespørringer og et -idempotent importscript for et kuratert Geoapify-datasett. +The backend uses TypeScript, Apollo Server (GraphQL) and PostgreSQL. The backend +and database run directly on the team VM, without Docker. -For source-data preview and the next database-import steps, see -[Geoapify data import](../docs/geoapify-data-import.md). The preview requires only -`GEOAPIFY_API_KEY` and does not connect to PostgreSQL. +- [Venue V1 contract](../docs/venue-model.md): fields, nulls, stable IDs and opening hours. +- [Geoapify import workflow](../docs/geoapify-data-import.md): source retrieval, seed export and import. -## Krav og npm-oppsett +## Install and configure -- Node.js og npm installert på utviklermaskinen og VM-en. -- PostgreSQL installert som vanlig tjeneste på VM-en. -- `server/package.json` og `server/package-lock.json` versjonskontrollert. - -Etter at backendendringene med lockfila er merget, kjører hvert gruppemedlem én -gang fra repoets rot: +Use the Node version in the root `.nvmrc`. From the repository root: ```powershell npm --prefix server ci +Copy-Item .env.example server/.env +``` + +Only copy the example if `server/.env` does not already exist. Replace the +password placeholder privately; never commit this file or paste credentials +into an issue. URL-encode special characters in the password (e.g. `#` as `%23`). + +On the VM, the connection uses `127.0.0.1:5432/tapfinder_dev`. For a local backend, +keep this SSH tunnel open in a separate terminal: + +```powershell +ssh -N -L 5433:127.0.0.1:5432 YOUR_VM_USER@it2810-07.idi.ntnu.no +``` + +The local `server/.env` then uses: + +```env +DATABASE_URL=postgresql://tapfinder_app:REPLACE_WITH_PASSWORD@127.0.0.1:5433/tapfinder_dev +PORT=4000 +HOST=127.0.0.1 +GEOAPIFY_API_KEY=REPLACE_WITH_GEOAPIFY_KEY +``` + +The Geoapify key is only needed for retrieval, not for an existing-file import +or for GraphQL. Check database access without changing any data: + +```powershell +npm --prefix server run db:check ``` -Dette installerer backendpakkene lokalt fra den versjonslåste `package-lock.json`. -`node_modules/` skal ikke pushes. Når backendens avhengigheter endres, oppdateres -lockfila med `npm --prefix server install`, og `package.json` og -`package-lock.json` committes sammen. På VM-en kjøres `npm --prefix server ci` -etter at ny kode er hentet inn. Dette skjer ikke automatisk ved Git-push før -automatisk utrulling er satt opp. +A connection failure usually means a missing tunnel, incorrect credentials or +an unavailable database. Confirm access with the database owner. The check +rejects placeholder passwords and other database names and does not log secrets. -## Utviklingsdatabase på VM-en +## Initial VM database setup -Kjør dette på VM-en der PostgreSQL er installert. Opprett applikasjonsrollen -uten å skrive passordet inn i SQL-kommandoen. Hvis `tapfinder_app` allerede -finnes fra det tidligere databaseoppsettet, behold rollen og hopp over -`CREATE ROLE` og `\password`: +The database owner runs the following on the VM. Keep PostgreSQL bound to +localhost. Reuse an existing `tapfinder_app` role rather than creating it again. ```bash sudo -u postgres psql ``` -I `psql`: - ```sql CREATE ROLE tapfinder_app LOGIN; \password tapfinder_app @@ -49,172 +62,94 @@ CREATE DATABASE tapfinder_dev OWNER tapfinder_app; \q ``` -`\password` ber om passordet interaktivt. PostgreSQL bør bare lytte på -localhost på VM-en. Ikke åpne port 5432 mot internett. - -## Miljøvariabler - -Backend og migreringsverktøy leser tilkoblingen fra `DATABASE_URL`. `.env.example` -i repo-roten er en mal uten hemmeligheter. På VM-en kopierer du den til -`server/.env` og legger inn det faktiske databasepassordet der. `server/.env` er -ignorert av Git og må ikke committes. Del passordet med gruppa gjennom en avtalt -privat kanal, ikke Git. +`\password` prompts interactively. Each team member configures their own ignored +`server/.env`; share credentials through the group's private channel. -På VM-en, der backend og PostgreSQL kjører på samme maskin, skal adressen bruke -`localhost` og port `5432`: +## Migrations and server -```env -DATABASE_URL=postgresql://tapfinder_app:PASSORD@127.0.0.1:5432/tapfinder_dev -PORT=4000 -HOST=127.0.0.1 -``` - -URL-kod spesialtegn i passordet; for eksempel skal `#` skrives som `%23` i -tilkoblingsadressen. Selve PostgreSQL-passordet endres ikke. - -Hver utvikler har sin egen `server/.env`. Hvis du kjører backend på PC-en, peker -`localhost` på PC-en. For å teste lokalt mot VM-databasen uten å åpne port 5432, -start en SSH-tunnel fra et lokalt PowerShell-vindu: +All database schema changes belong in `server/migrations/`. Do not edit an +already applied migration. Create an additive migration and review it in a PR: ```powershell -ssh -N -L 5433:127.0.0.1:5432 DIN_BRUKER@it2810-07.idi.ntnu.no +npm --prefix server run migrate:create -- add-your-change ``` -Da bruker PC-ens lokale backend port `5433` i sin `DATABASE_URL`. Dette er bare -for lokal testing. Når backend kjører på VM-en, skal den bruke port `5432` -direkte, og denne databasetunnelen trengs ikke. - -## Hente og starte backend på VM-en manuelt - -Inntil automatisk utrulling er konfigurert, må godkjent og merget backendkode -hentes inn på VM-en manuelt. Etter SSH-innlogging går du til repoets klonede mappe -på VM-en og kjører: +Once the code is reviewed and the target database is confirmed, apply pending +migrations. Back up a database that already contains important data first. -```bash -npm --prefix server ci +```powershell +npm --prefix server run migrate:up npm --prefix server run build npm --prefix server run dev ``` -Den siste kommandoen starter utviklingsserveren. SSH-terminalen må stå åpen mens -den kjører; avslut med `Ctrl+C`. En varig tjeneste som starter automatisk etter -omstart av VM-en, og automatisk utrulling etter merge, er ikke konfigurert ennå. +`create-venues` creates the Venue/source tables. `complete-venue-model` adds the +remaining five public fields and range constraints, preserving existing IDs. +Pre-existing rows with invalid coordinates must be corrected before this new +migration can succeed. `migrate:down` reverses a migration and may remove data; +it is not needed for normal setup. -Serveren lytter som standard bare på `127.0.0.1:4000`. For å åpne GraphQL -Sandbox fra PC-en mens serveren kjører på VM-en, bruk en egen lokal tunnel i -PowerShell: +The API listens on `127.0.0.1:4000`. If running on the VM, open a separate API +tunnel from your PC to use the GraphQL Sandbox: ```powershell -ssh -N -L 4000:127.0.0.1:4000 DIN_BRUKER@it2810-07.idi.ntnu.no +ssh -N -L 4000:127.0.0.1:4000 YOUR_VM_USER@it2810-07.idi.ntnu.no ``` -La tunnelen stå åpen og åpne `http://127.0.0.1:4000/` i nettleseren. Dette er en -API-tunnel; den videresender ikke PostgreSQL-porten. - -## GraphQL-kontroll - -I Sandbox kan du kjøre: +Open `http://127.0.0.1:4000/` and run: ```graphql query { health databaseStatus + venues { + id + name + type + reviewCount + } } ``` -`health` svarer `ok` når API-et kjører. `databaseStatus` svarer `connected` når -backenden når PostgreSQL. +`health` returns `ok`; `databaseStatus` returns `connected`. The model document +contains an example requesting all 16 fields. On the VM, `npm --prefix server +run start` runs the compiled server; a persistent backend service and automatic +backend deployment remain separate work. Frontend deployment does not deploy +these backend changes. -## Databaseendringer (migrasjoner) +## Seed import -Alle skjemaendringer skal være migrasjoner i `server/migrations/`, slik at -utviklere og VM-en får de samme endringene i samme rekkefølge. Lag en migrasjon -fra repoets rot: +`npm --prefix server run data:export` produces a validated, ignored seed file +from the latest completed Trondheim export. The original `data/venues.json` +remains an empty default. Provide the desired file explicitly: ```powershell -npm --prefix server run migrate:create -- add-your-change-name +npm --prefix server run seed:venues -- --file imports/venue-seed-YOUR_FILE.json --dry-run +npm --prefix server run seed:venues -- --file imports/venue-seed-YOUR_FILE.json ``` -Review migrasjonen sammen med backendendringen i pull request. Før migrasjoner -kjøres på VM-en, kontroller at VM-ens `server/.env` peker til `tapfinder_dev`. -Før migrering av en database med viktige data: ta backup. Kjør deretter: - -```bash -npm --prefix server run migrate:up -``` - -Migreringen `create_venues` oppretter `venues` og `venue_sources`. Interne venue- -ID-er lages av databasen. Geoapify-ID-en lagres i `venue_sources`, med unik -kombinasjon av leverandør og ekstern ID, slik at importen kan kjøres flere ganger -uten duplikater. - -## Venue-felter og Geoapify-import - -Første versjon inneholder `id`, `name`, `type`, `city`, `area`, `address`, -`latitude`, `longitude`, `openingHours`, `tags` og `websiteUrl`. Image, beskrivelse, -prisnivå, vurdering og antall anmeldelser er utelatt inntil dere har datakilder -for dem. `websiteUrl` kan være `null`. `openingHours` og `tags` er alltid lister; -ukjente verdier representeres som `[]`. - -Importer bare data dere har tillatelse til å lagre og distribuere. Behold -nødvendig Geoapify/OpenStreetMap-attribusjon i produktet og kontroller gjeldende -vilkår før dere committer et datasett til Git. - -Seed-scriptet leser `server/data/venues.json`. Filen er opprettet som en tom liste -og skal fylles med normaliserte poster. Format: - -```json -[ - { - "sourceId": "Geoapify- eller OSM-ID", - "name": "Eksempelbar", - "type": "BAR", - "city": "Trondheim", - "area": "Sentrum", - "address": "Eksempelveien 1", - "latitude": 63.4305, - "longitude": 10.3951, - "openingHours": [ - { "day": "FRIDAY", "opens": "18:00", "closes": "02:00", "closed": false } - ], - "tags": ["bar"], - "websiteUrl": null - } -] -``` +File paths are relative to `server/`; absolute paths also work. Dry runs need +no database connection. Actual imports require `tapfinder_dev`, apply one file +in one transaction and report inserted/updated counts. Reimports preserve IDs, +ratings and review counts. Missing imported price/description/image values +preserve existing curated values. See the import guide for source metadata and +unsupported-hours handling. -Dette er kun et format-eksempel, ikke et virkelig sted. `sourceId` må være stabil -og unik hos Geoapify/OSM. `openingHours` må konverteres fra Geoapifys -åpningstidstekst til ukedag/intervallformat før import; bruk `[]` hvis data -mangler eller ikke er korrekt tolket. `tags` skal være deres normaliserte tagger. -Fyll ut `server/data/venues.json` med de verifiserte postene før seed-kjøring. +## Checks -Kjør importen fra repoets rot på VM-en etter at migreringen er på plass: - -```bash -npm --prefix server run seed:venues +```powershell +npm --prefix server run typecheck +npm --prefix server run test:data +npm --prefix server run build ``` -Scriptet avviser andre databaser enn `tapfinder_dev`, validerer feltene og gjør -alle endringene i én transaksjon. Kjør det gjerne igjen etter at datasettet er -oppdatert; samme Geoapify-ID oppdaterer samme venue. +The test suite covers API retrieval with mocked responses, normalization, +validation, migrations, database constraints, repeat imports, transaction +rollback and all Venue GraphQL fields. PGlite runs PostgreSQL tests in memory; +no VM, Docker, credentials or Geoapify key is needed. Scripts and tests are +included in type checking. Backend CI runs these checks on PRs. The existing +frontend Vitest suite remains separate. -GraphQL Sandbox kan kontrollere dataene med: - -```graphql -query { - venues { - id - name - type - city - area - address - latitude - longitude - openingHours { day opens closes closed } - tags - websiteUrl - } -} -``` +The data-import implementation, tests and documentation were prepared with +Codex and verified locally. A real VM connection and import are a separate +operational check once credentials and the SSH tunnel are available. diff --git a/server/migrations/1791907200000_complete-venue-model.js b/server/migrations/1791907200000_complete-venue-model.js new file mode 100644 index 0000000..b600018 --- /dev/null +++ b/server/migrations/1791907200000_complete-venue-model.js @@ -0,0 +1,48 @@ +/** @param {import('node-pg-migrate').MigrationBuilder} pgm */ +export const up = (pgm) => { + pgm.addColumns('venues', { + price_level: { type: 'integer' }, + rating: { type: 'double precision' }, + review_count: { type: 'integer', notNull: true, default: 0 }, + description: { type: 'text' }, + image_url: { type: 'text' }, + }) + pgm.addConstraint('venues', 'venues_latitude_range', { + check: 'latitude BETWEEN -90 AND 90', + }) + pgm.addConstraint('venues', 'venues_longitude_range', { + check: 'longitude BETWEEN -180 AND 180', + }) + pgm.addConstraint('venues', 'venues_price_level_range', { + check: 'price_level BETWEEN 1 AND 3', + }) + pgm.addConstraint('venues', 'venues_rating_range', { + check: 'rating BETWEEN 0 AND 5', + }) + pgm.addConstraint('venues', 'venues_review_count_range', { + check: 'review_count >= 0', + }) + pgm.addConstraint('venues', 'venues_opening_hours_array', { + check: "jsonb_typeof(opening_hours) = 'array'", + }) +} + +/** @param {import('node-pg-migrate').MigrationBuilder} pgm */ +export const down = (pgm) => { + for (const name of [ + 'venues_latitude_range', + 'venues_longitude_range', + 'venues_price_level_range', + 'venues_rating_range', + 'venues_review_count_range', + 'venues_opening_hours_array', + ]) + pgm.dropConstraint('venues', name) + pgm.dropColumns('venues', [ + 'price_level', + 'rating', + 'review_count', + 'description', + 'image_url', + ]) +} diff --git a/server/package-lock.json b/server/package-lock.json index b2ab0f8..ddfaaa2 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -13,6 +13,7 @@ "pg": "^8.16.3" }, "devDependencies": { + "@electric-sql/pglite": "0.5.9", "@types/node": "^22.15.0", "@types/pg": "^8.15.0", "node-pg-migrate": "^9.0.0", @@ -257,6 +258,13 @@ "node": ">=16" } }, + "node_modules/@electric-sql/pglite": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.5.9.tgz", + "integrity": "sha512-GoQTRMZ/o4J3sMW4mLDtDFGNOm1xy5oH+iHDALWll3uERY6OF4BSnL1t7FZH2Mlw+RPceDBlFJgSaecO73c4EA==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", diff --git a/server/package.json b/server/package.json index 99daea6..7be8131 100644 --- a/server/package.json +++ b/server/package.json @@ -8,7 +8,10 @@ "start": "node dist/index.js", "data:preview": "tsx src/scripts/previewGeoapify.ts", "data:prepare": "tsx src/scripts/prepareVenues.ts", - "test:data": "tsx --test tests/geoapify.test.ts tests/prepareVenues.test.ts tests/openingHours.test.ts", + "data:export": "tsx src/scripts/exportVenues.ts", + "db:check": "tsx src/scripts/checkDatabase.ts", + "typecheck": "tsc -p tsconfig.check.json", + "test:data": "tsx --test tests/*.test.ts", "migrate:create": "node-pg-migrate create", "migrate:up": "node-pg-migrate up", "migrate:down": "node-pg-migrate down", @@ -22,6 +25,7 @@ "pg": "^8.16.3" }, "devDependencies": { + "@electric-sql/pglite": "0.5.9", "@types/node": "^22.15.0", "@types/pg": "^8.15.0", "node-pg-migrate": "^9.0.0", diff --git a/server/scripts/seed-venues.ts b/server/scripts/seed-venues.ts index 201ca18..6d81726 100644 --- a/server/scripts/seed-venues.ts +++ b/server/scripts/seed-venues.ts @@ -1,228 +1,71 @@ -import 'dotenv/config'; -import { readFile } from 'node:fs/promises'; -import { resolve } from 'node:path'; -import { pool } from '../src/db/pool.js'; +import { config } from 'dotenv' +import { readFile } from 'node:fs/promises' +import { resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { importVenues } from '../src/data/importVenues.js' +import { + requireDevelopmentDatabase, + validateVenues, + VenueValidationError, +} from '../src/data/venueSeed.js' -type Weekday = - | 'MONDAY' - | 'TUESDAY' - | 'WEDNESDAY' - | 'THURSDAY' - | 'FRIDAY' - | 'SATURDAY' - | 'SUNDAY'; - -type VenueSeed = { - sourceId: string; - name: string; - type: 'BAR' | 'PUB' | 'NIGHTCLUB'; - city: string; - area: string; - address: string; - latitude: number; - longitude: number; - openingHours: { - day: Weekday; - opens: string | null; - closes: string | null; - closed: boolean; - }[]; - tags: string[]; - websiteUrl: string | null; -}; - -const weekdays = new Set([ - 'MONDAY', - 'TUESDAY', - 'WEDNESDAY', - 'THURSDAY', - 'FRIDAY', - 'SATURDAY', - 'SUNDAY', -]); - -function validateVenue(value: unknown, index: number): VenueSeed { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`venues[${index}] must be an object`); - } - - const venue = value as Record; - const requiredStrings = [ - 'sourceId', - 'name', - 'type', - 'city', - 'area', - 'address', - ] as const; - - for (const key of requiredStrings) { - if (typeof venue[key] !== 'string' || venue[key].trim() === '') { - throw new Error(`venues[${index}].${key} must be a non-empty string`); - } - } - - if (!['BAR', 'PUB', 'NIGHTCLUB'].includes(venue.type as string)) { - throw new Error(`venues[${index}].type must be BAR, PUB, or NIGHTCLUB`); - } - if (typeof venue.latitude !== 'number' || !Number.isFinite(venue.latitude)) { - throw new Error(`venues[${index}].latitude must be a finite number`); - } - if (typeof venue.longitude !== 'number' || !Number.isFinite(venue.longitude)) { - throw new Error(`venues[${index}].longitude must be a finite number`); - } - if (venue.websiteUrl !== null && typeof venue.websiteUrl !== 'string') { - throw new Error(`venues[${index}].websiteUrl must be a URL string or null`); - } - if (!Array.isArray(venue.tags) || !venue.tags.every((tag) => typeof tag === 'string')) { - throw new Error(`venues[${index}].tags must be an array of strings`); +async function main() { + const serverRoot = fileURLToPath(new URL('../', import.meta.url)) + config({ path: resolve(serverRoot, '.env') }) + const args = process.argv.slice(2) + if (args.includes('--help')) { + console.log( + 'Usage: npm --prefix server run seed:venues -- [--file PATH] [--dry-run]\nPaths are relative to server/. Dry runs validate without connecting to PostgreSQL.', + ) + return } - if (!Array.isArray(venue.openingHours)) { - throw new Error(`venues[${index}].openingHours must be an array (use [] when unknown)`); + let file = 'data/venues.json' + let dryRun = false + for (let index = 0; index < args.length; index++) { + if (args[index] === '--dry-run') dryRun = true + else if ( + args[index] === '--file' && + args[index + 1] && + !args[index + 1].startsWith('--') + ) + file = args[++index] + else + throw new VenueValidationError( + 'Unknown or incomplete option. Use --help for usage.', + ) } - - const openingHours = venue.openingHours.map((hours, hoursIndex) => { - if (!hours || typeof hours !== 'object' || Array.isArray(hours)) { - throw new Error(`venues[${index}].openingHours[${hoursIndex}] must be an object`); - } - const item = hours as Record; - if (!weekdays.has(item.day as Weekday)) { - throw new Error(`venues[${index}].openingHours[${hoursIndex}].day is invalid`); - } - if (item.opens !== null && typeof item.opens !== 'string') { - throw new Error(`venues[${index}].openingHours[${hoursIndex}].opens must be a string or null`); - } - if (item.closes !== null && typeof item.closes !== 'string') { - throw new Error(`venues[${index}].openingHours[${hoursIndex}].closes must be a string or null`); - } - if (typeof item.closed !== 'boolean') { - throw new Error(`venues[${index}].openingHours[${hoursIndex}].closed must be boolean`); - } - return { - day: item.day as Weekday, - opens: item.opens as string | null, - closes: item.closes as string | null, - closed: item.closed, - }; - }); - - return { - sourceId: venue.sourceId as string, - name: venue.name as string, - type: venue.type as VenueSeed['type'], - city: venue.city as string, - area: venue.area as string, - address: venue.address as string, - latitude: venue.latitude, - longitude: venue.longitude, - openingHours, - tags: venue.tags as string[], - websiteUrl: venue.websiteUrl as string | null, - }; -} - -async function seed() { - const databaseUrl = process.env.DATABASE_URL; - if (!databaseUrl || new URL(databaseUrl).pathname !== '/tapfinder_dev') { - throw new Error( - 'Refusing to seed: DATABASE_URL must point to the tapfinder_dev database.', - ); - } - - const inputPath = resolve(process.cwd(), 'data/venues.json'); - const parsed: unknown = JSON.parse(await readFile(inputPath, 'utf8')); - if (!Array.isArray(parsed)) { - throw new Error(`${inputPath} must contain a JSON array`); + let payload: unknown + try { + payload = JSON.parse(await readFile(resolve(serverRoot, file), 'utf8')) + } catch { + throw new VenueValidationError( + 'Could not read the seed file as JSON. Check --file.', + ) } - - const venues = parsed.map(validateVenue); - if (venues.length === 0) { - console.log('No venues in data/venues.json; nothing to import.'); - return; + const venues = validateVenues(payload) + console.log(`Validated ${venues.length} venue(s).`) + if (dryRun || !venues.length) { + console.log('No database writes were made.') + return } - - const client = await pool.connect(); + requireDevelopmentDatabase(process.env.DATABASE_URL) + const { pool } = await import('../src/db/pool.js') try { - await client.query('BEGIN'); - for (const venue of venues) { - const source = await client.query<{ venue_id: string }>( - `SELECT venue_id - FROM venue_sources - WHERE provider = 'geoapify' AND external_id = $1 - FOR UPDATE`, - [venue.sourceId], - ); - - let venueId = source.rows[0]?.venue_id; - if (venueId) { - await client.query( - `UPDATE venues - SET name = $2, type = $3, city = $4, area = $5, address = $6, - latitude = $7, longitude = $8, opening_hours = $9::jsonb, - tags = $10, website_url = $11, updated_at = NOW() - WHERE id = $1`, - [ - venueId, - venue.name, - venue.type, - venue.city, - venue.area, - venue.address, - venue.latitude, - venue.longitude, - JSON.stringify(venue.openingHours), - venue.tags, - venue.websiteUrl, - ], - ); - await client.query( - `UPDATE venue_sources SET fetched_at = NOW() - WHERE provider = 'geoapify' AND external_id = $1`, - [venue.sourceId], - ); - } else { - const inserted = await client.query<{ id: string }>( - `INSERT INTO venues - (name, type, city, area, address, latitude, longitude, - opening_hours, tags, website_url) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10) - RETURNING id`, - [ - venue.name, - venue.type, - venue.city, - venue.area, - venue.address, - venue.latitude, - venue.longitude, - JSON.stringify(venue.openingHours), - venue.tags, - venue.websiteUrl, - ], - ); - venueId = inserted.rows[0].id; - await client.query( - `INSERT INTO venue_sources (venue_id, provider, external_id) - VALUES ($1, 'geoapify', $2)`, - [venueId, venue.sourceId], - ); - } - } - await client.query('COMMIT'); - console.log(`Imported or updated ${venues.length} venue(s).`); - } catch (error) { - await client.query('ROLLBACK'); - throw error; + const result = await importVenues(pool, venues) + console.log( + `Inserted ${result.inserted}; updated ${result.updated}. Existing IDs and app reviews were preserved.`, + ) } finally { - client.release(); + await pool.end() } } -try { - await seed(); -} catch (error) { - console.error('Venue import failed:', error); - process.exitCode = 1; -} finally { - await pool.end(); -} +main().catch((error: unknown) => { + // Driver errors may include connection details. Only validation messages are public. + console.error( + error instanceof VenueValidationError + ? error.message + : 'Venue import failed. Check database access and apply the Venue migrations.', + ) + process.exitCode = 1 +}) diff --git a/server/src/data/exportVenues.ts b/server/src/data/exportVenues.ts new file mode 100644 index 0000000..ca6298e --- /dev/null +++ b/server/src/data/exportVenues.ts @@ -0,0 +1,40 @@ +import { prepareVenueDrafts } from './prepareVenues.js' +import { isRecord, validateVenues, type VenueSeed } from './venueSeed.js' + +export function exportVenueSeeds( + payload: unknown, + sourceFile: string, +): VenueSeed[] { + const drafts = prepareVenueDrafts(payload, { openingHours: true }) + const source = payload as Record + if ( + typeof source.fetchedAt !== 'string' || + !isRecord(source.search) || + typeof source.attribution !== 'string' + ) + throw new Error( + 'Source export is missing its retrieval time or attribution.', + ) + const places = source.places as Record[] + const typeMap = { Bar: 'BAR', Pub: 'PUB', Nattklubb: 'NIGHTCLUB' } as const + return validateVenues( + drafts.map((draft, index) => ({ + ...draft.venue, + sourceId: draft.sourceKey, + type: typeMap[draft.venue.type as keyof typeof typeMap], + sourceFetchedAt: source.fetchedAt, + rawData: { + provider: 'Geoapify', + attribution: source.attribution, + sourceFile, + search: source.search, + source: draft.source, + openingHoursRaw: draft.openingHoursRaw, + openingHoursStatus: draft.openingHoursStatus, + openingHoursReviewReason: draft.openingHoursReviewReason, + openingHoursWarnings: draft.openingHoursWarnings, + raw: places[index].raw ?? null, + }, + })), + ) +} diff --git a/server/src/data/importVenues.ts b/server/src/data/importVenues.ts new file mode 100644 index 0000000..dcbb0a3 --- /dev/null +++ b/server/src/data/importVenues.ts @@ -0,0 +1,96 @@ +import { validateVenues } from './venueSeed.js' + +export interface SqlExecutor { + query( + sql: string, + values?: unknown[], + ): Promise<{ rows: Record[] }> +} +export interface ImportDatabase { + connect(): Promise +} + +export async function importVenues(database: ImportDatabase, payload: unknown) { + const venues = validateVenues(payload) + if (!venues.length) return { inserted: 0, updated: 0 } + const client = await database.connect() + let inserted = 0 + let updated = 0 + try { + await client.query('BEGIN') + for (const venue of venues) { + const source = await client.query( + `SELECT venue_id FROM venue_sources + WHERE provider = 'geoapify' AND external_id = $1 FOR UPDATE`, + [venue.sourceId], + ) + let venueId = source.rows[0]?.venue_id + const values = [ + venue.name, + venue.type, + venue.city, + venue.area, + venue.address, + venue.latitude, + venue.longitude, + JSON.stringify(venue.openingHours), + venue.tags, + venue.websiteUrl, + venue.priceLevel, + venue.description, + venue.imageUrl, + ] + if (venueId) { + await client.query( + `UPDATE venues SET name = $1, type = $2, city = $3, area = $4, + address = $5, latitude = $6, longitude = $7, opening_hours = $8::jsonb, + tags = $9, website_url = $10, + price_level = COALESCE($11, price_level), + description = COALESCE($12, description), + image_url = COALESCE($13, image_url), updated_at = NOW() + WHERE id = $14`, + [...values, venueId], + ) + await client.query( + `UPDATE venue_sources SET + fetched_at = COALESCE($2::timestamptz, NOW()), + raw_data = COALESCE($3::jsonb, raw_data) + WHERE provider = 'geoapify' AND external_id = $1`, + [ + venue.sourceId, + venue.sourceFetchedAt, + venue.rawData === null ? null : JSON.stringify(venue.rawData), + ], + ) + updated++ + } else { + const result = await client.query( + `INSERT INTO venues (name, type, city, area, address, latitude, longitude, + opening_hours, tags, website_url, price_level, description, image_url) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10, $11, $12, $13) + RETURNING id`, + values, + ) + venueId = result.rows[0].id + await client.query( + `INSERT INTO venue_sources (venue_id, provider, external_id, fetched_at, raw_data) + VALUES ($1, 'geoapify', $2, COALESCE($3::timestamptz, NOW()), $4::jsonb)`, + [ + venueId, + venue.sourceId, + venue.sourceFetchedAt, + venue.rawData === null ? null : JSON.stringify(venue.rawData), + ], + ) + inserted++ + } + } + await client.query('COMMIT') + return { inserted, updated } + } catch (error) { + await client.query('ROLLBACK') + throw error + } finally { + client.release() + } +} diff --git a/server/src/data/prepareVenues.ts b/server/src/data/prepareVenues.ts index d061cf4..d8f8f7f 100644 --- a/server/src/data/prepareVenues.ts +++ b/server/src/data/prepareVenues.ts @@ -6,7 +6,7 @@ import { type JsonRecord = Record -// Intermediate data only: database IDs and opening-hours conversion come later. +// Intermediate data: source metadata is separate from the public Venue fields. export interface VenueDraft { sourceKey: string source: { osmType: string; osmId: string; geoapifyPlaceId: string } diff --git a/server/src/data/sourceExports.ts b/server/src/data/sourceExports.ts new file mode 100644 index 0000000..54ab745 --- /dev/null +++ b/server/src/data/sourceExports.ts @@ -0,0 +1,30 @@ +import { readFile, readdir, stat } from 'node:fs/promises' +import { join } from 'node:path' +import { isCompletedTrondheimExport } from './prepareVenues.js' + +export async function loadCompletedTrondheimExport(directory: string) { + const candidates = await Promise.all( + (await readdir(directory)) + .filter((name) => /^geoapify-\d.*\.json$/.test(name)) + .map(async (name) => ({ + name, + modified: (await stat(join(directory, name))).mtimeMs, + })), + ) + candidates.sort((a, b) => b.modified - a.modified) + for (const candidate of candidates) { + let payload: unknown + try { + payload = JSON.parse( + await readFile(join(directory, candidate.name), 'utf8'), + ) + } catch { + continue + } + if (isCompletedTrondheimExport(payload)) + return { payload, name: candidate.name } + } + throw new Error( + 'No completed Trondheim export found. Run data:preview with --trondheim --all-pages and details enabled first.', + ) +} diff --git a/server/src/data/venueSeed.ts b/server/src/data/venueSeed.ts new file mode 100644 index 0000000..0c812c9 --- /dev/null +++ b/server/src/data/venueSeed.ts @@ -0,0 +1,204 @@ +import { weekdays, type OpeningHours, type Weekday } from './openingHours.js' + +export class VenueValidationError extends Error {} + +export interface VenueSeed { + sourceId: string + name: string + type: 'BAR' | 'PUB' | 'NIGHTCLUB' + city: string + area: string + address: string + latitude: number + longitude: number + priceLevel: number | null + rating: null + reviewCount: 0 + description: string | null + openingHours: OpeningHours[] + tags: string[] + imageUrl: string | null + websiteUrl: string | null + sourceFetchedAt: string | null + rawData: Record | null +} + +export function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +export function validateVenues(payload: unknown): VenueSeed[] { + if (!Array.isArray(payload)) + throw new VenueValidationError('The seed file must contain a JSON array.') + const sources = new Set() + return payload.map((value, index) => { + const prefix = `venues[${index}]` + const fail = (message: string): never => { + throw new VenueValidationError(`${prefix}.${message}`) + } + if (!isRecord(value)) fail('record must be an object.') + const venue = value as Record + const requiredText = (key: string) => { + const result = venue[key] + if (typeof result !== 'string' || !result.trim()) + return fail(`${key} must be a non-empty string.`) + return result.trim() + } + const optionalText = (key: string) => { + const result = venue[key] + if (result === null || result === undefined) return null + if (typeof result !== 'string' || !result.trim()) + return fail(`${key} must be a non-empty string or null.`) + return result.trim() + } + const url = (key: string) => { + const text = optionalText(key) + if (text === null) return null + try { + const parsed = new URL(text) + if ( + !['https:', 'http:'].includes(parsed.protocol) || + parsed.username || + parsed.password + ) + throw new Error() + return parsed.href + } catch { + return fail( + `${key} must be an HTTP(S) URL without credentials, or null.`, + ) + } + } + const coordinate = (key: string, bound: number) => { + const result = venue[key] + if ( + typeof result !== 'number' || + !Number.isFinite(result) || + Math.abs(result) > bound + ) + return fail(`${key} must be between -${bound} and ${bound}.`) + return result + } + const sourceId = requiredText('sourceId') + if (sources.has(sourceId)) fail('sourceId duplicates another record.') + sources.add(sourceId) + const type = requiredText('type') + if (!['BAR', 'PUB', 'NIGHTCLUB'].includes(type)) + fail('type must be BAR, PUB, or NIGHTCLUB.') + const priceLevel = venue.priceLevel ?? null + if ( + priceLevel !== null && + (!Number.isInteger(priceLevel) || + typeof priceLevel !== 'number' || + priceLevel < 1 || + priceLevel > 3) + ) + fail('priceLevel must be 1, 2, 3, or null.') + // Ratings and counts belong to our application, never to provider imports. + if (venue.rating !== undefined && venue.rating !== null) + fail('rating must be null; app reviews own the rating.') + if (venue.reviewCount !== undefined && venue.reviewCount !== 0) + fail('reviewCount must be 0; app reviews own the review count.') + if ( + !Array.isArray(venue.tags) || + !venue.tags.every((tag) => typeof tag === 'string' && tag.trim()) + ) + fail('tags must be an array of non-empty strings.') + if (!Array.isArray(venue.openingHours)) + fail('openingHours must be an array (use [] when unknown).') + const openingHours = (venue.openingHours as unknown[]).map( + (hours, hourIndex) => { + const path = `openingHours[${hourIndex}]` + if (!isRecord(hours)) fail(`${path} must be an object.`) + const item = hours as Record + if (!weekdays.includes(item.day as Weekday)) + fail(`${path}.day is invalid.`) + if (typeof item.closed !== 'boolean') + fail(`${path}.closed must be boolean.`) + if (item.closed) { + if (item.opens !== null || item.closes !== null) + fail(`${path} must have null times when closed.`) + } else { + if ( + typeof item.opens !== 'string' || + !/^(?:[01]\d|2[0-3]):[0-5]\d$/.test(item.opens) + ) + fail(`${path}.opens must be HH:mm from 00:00 to 23:59.`) + if ( + typeof item.closes !== 'string' || + !/^(?:(?:[01]\d|2[0-3]):[0-5]\d|24:00)$/.test(item.closes) + ) + fail(`${path}.closes must be HH:mm from 00:00 to 24:00.`) + if ((item.opens as string) >= (item.closes as string)) + fail( + `${path} must stay within one calendar day; split overnight hours at midnight.`, + ) + } + return { + day: item.day as Weekday, + opens: item.opens as string | null, + closes: item.closes as string | null, + closed: item.closed as boolean, + } + }, + ) + for (const day of weekdays) { + const entries = openingHours.filter((item) => item.day === day) + if (entries.length > 1 && entries.some((item) => item.closed)) + fail(`openingHours has conflicting closed/open entries for ${day}.`) + const ranges = entries + .filter((item) => !item.closed) + .sort((a, b) => a.opens!.localeCompare(b.opens!)) + for (let i = 1; i < ranges.length; i++) + if (ranges[i].opens! < ranges[i - 1].closes!) + fail(`openingHours has overlapping intervals for ${day}.`) + } + const sourceFetchedAt = optionalText('sourceFetchedAt') + if ( + sourceFetchedAt && + (!/^\d{4}-\d{2}-\d{2}T/.test(sourceFetchedAt) || + !Number.isFinite(Date.parse(sourceFetchedAt))) + ) + fail('sourceFetchedAt must be an ISO timestamp or null.') + const rawData = venue.rawData ?? null + if (rawData !== null && !isRecord(rawData)) + fail('rawData must be an object or null.') + return { + sourceId, + name: requiredText('name'), + type: type as VenueSeed['type'], + city: requiredText('city'), + area: requiredText('area'), + address: requiredText('address'), + latitude: coordinate('latitude', 90), + longitude: coordinate('longitude', 180), + priceLevel: priceLevel as number | null, + rating: null, + reviewCount: 0, + description: optionalText('description'), + openingHours, + tags: [...new Set((venue.tags as string[]).map((tag) => tag.trim()))], + imageUrl: url('imageUrl'), + websiteUrl: url('websiteUrl'), + sourceFetchedAt, + rawData: rawData as Record | null, + } + }) +} + +export function requireDevelopmentDatabase(value: string | undefined) { + try { + const url = new URL(value ?? '') + if ( + !['postgres:', 'postgresql:'].includes(url.protocol) || + url.pathname !== '/tapfinder_dev' || + !url.password || + /REPLACE|PASSORD|YOUR_/i.test(decodeURIComponent(url.password)) + ) + throw new Error() + } catch { + throw new VenueValidationError( + 'Configure DATABASE_URL with real credentials for tapfinder_dev before importing.', + ) + } +} diff --git a/server/src/db/pool.ts b/server/src/db/pool.ts index e4fe85c..1805ea1 100644 --- a/server/src/db/pool.ts +++ b/server/src/db/pool.ts @@ -1,14 +1,19 @@ -import 'dotenv/config'; -import { Pool } from 'pg'; +import 'dotenv/config' +import { Pool } from 'pg' -const connectionString = process.env.DATABASE_URL; +const connectionString = process.env.DATABASE_URL if (!connectionString) { - throw new Error('DATABASE_URL is not set. Copy .env.example to server/.env and configure it.'); + throw new Error( + 'DATABASE_URL is not set. Copy .env.example to server/.env and configure it.', + ) } -export const pool = new Pool({ connectionString }); +export const pool = new Pool({ + connectionString, + connectionTimeoutMillis: 5000, +}) pool.on('error', (error) => { - console.error('Unexpected PostgreSQL pool error:', error); -}); + console.error('Unexpected PostgreSQL pool error:', error) +}) diff --git a/server/src/graphql/schema.ts b/server/src/graphql/schema.ts index f2aec66..380e1ef 100644 --- a/server/src/graphql/schema.ts +++ b/server/src/graphql/schema.ts @@ -1,4 +1,4 @@ -import { pool } from '../db/pool.js'; +import type { SqlExecutor } from '../data/importVenues.js' export const typeDefs = ` enum Weekday { @@ -27,8 +27,13 @@ export const typeDefs = ` address: String! latitude: Float! longitude: Float! + priceLevel: Int + rating: Float + reviewCount: Int! + description: String openingHours: [OpeningHours!]! tags: [String!]! + imageUrl: String websiteUrl: String } @@ -42,35 +47,41 @@ export const typeDefs = ` venues: [Venue!]! venue(id: ID!): Venue } -`; +` -export const resolvers = { +const venueColumns = `id, name, type, city, area, address, latitude, longitude, + price_level AS "priceLevel", rating, review_count AS "reviewCount", description, + opening_hours AS "openingHours", tags, image_url AS "imageUrl", website_url AS "websiteUrl"` + +export const createResolvers = (database: SqlExecutor) => ({ Query: { health: () => 'ok', databaseStatus: async () => { - await pool.query('SELECT 1'); - return 'connected'; + await database.query('SELECT 1') + return 'connected' }, venues: async () => { - const { rows } = await pool.query( - `SELECT id, name, type, city, area, address, latitude, longitude, - opening_hours AS "openingHours", tags, - website_url AS "websiteUrl" + const { rows } = await database.query( + `SELECT ${venueColumns} FROM venues ORDER BY name, id`, - ); - return rows; + ) + return rows }, venue: async (_parent: unknown, args: { id: string }) => { - const { rows } = await pool.query( - `SELECT id, name, type, city, area, address, latitude, longitude, - opening_hours AS "openingHours", tags, - website_url AS "websiteUrl" + if ( + !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( + args.id, + ) + ) + return null + const { rows } = await database.query( + `SELECT ${venueColumns} FROM venues WHERE id = $1`, [args.id], - ); - return rows[0] ?? null; + ) + return rows[0] ?? null }, }, -}; +}) diff --git a/server/src/index.ts b/server/src/index.ts index ce3b1e3..9951465 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,32 +1,32 @@ -import 'dotenv/config'; -import { ApolloServer } from '@apollo/server'; -import { startStandaloneServer } from '@apollo/server/standalone'; -import { pool } from './db/pool.js'; -import { resolvers, typeDefs } from './graphql/schema.js'; +import 'dotenv/config' +import { ApolloServer } from '@apollo/server' +import { startStandaloneServer } from '@apollo/server/standalone' +import { pool } from './db/pool.js' +import { createResolvers, typeDefs } from './graphql/schema.js' -const port = Number(process.env.PORT ?? 4000); -const host = process.env.HOST ?? '127.0.0.1'; +const port = Number(process.env.PORT ?? 4000) +const host = process.env.HOST ?? '127.0.0.1' if (!Number.isInteger(port) || port < 1 || port > 65535) { - throw new Error('PORT must be a valid TCP port number.'); + throw new Error('PORT must be a valid TCP port number.') } -const server = new ApolloServer({ typeDefs, resolvers }); +const server = new ApolloServer({ typeDefs, resolvers: createResolvers(pool) }) try { const { url } = await startStandaloneServer(server, { listen: { host, port }, - }); - console.log(`GraphQL API ready at ${url}`); + }) + console.log(`GraphQL API ready at ${url}`) } catch (error) { - await pool.end(); - throw error; + await pool.end() + throw error } for (const signal of ['SIGINT', 'SIGTERM'] as const) { process.once(signal, async () => { - await server.stop(); - await pool.end(); - process.exit(0); - }); + await server.stop() + await pool.end() + process.exit(0) + }) } diff --git a/server/src/scripts/checkDatabase.ts b/server/src/scripts/checkDatabase.ts new file mode 100644 index 0000000..8cb68d7 --- /dev/null +++ b/server/src/scripts/checkDatabase.ts @@ -0,0 +1,31 @@ +import { config } from 'dotenv' +import { fileURLToPath } from 'node:url' +import { + requireDevelopmentDatabase, + VenueValidationError, +} from '../data/venueSeed.js' + +async function main() { + config({ path: fileURLToPath(new URL('../../.env', import.meta.url)) }) + requireDevelopmentDatabase(process.env.DATABASE_URL) + const { pool } = await import('../db/pool.js') + try { + const result = await pool.query( + 'SELECT current_database() AS database, current_user AS user', + ) + console.log( + `Connected to ${result.rows[0].database} as ${result.rows[0].user}. No database writes were made.`, + ) + } finally { + await pool.end() + } +} + +main().catch((error: unknown) => { + console.error( + error instanceof VenueValidationError + ? error.message + : 'Could not connect. Check the SSH tunnel, DATABASE_URL and database access with the database owner.', + ) + process.exitCode = 1 +}) diff --git a/server/src/scripts/exportVenues.ts b/server/src/scripts/exportVenues.ts new file mode 100644 index 0000000..e446ce7 --- /dev/null +++ b/server/src/scripts/exportVenues.ts @@ -0,0 +1,46 @@ +import { randomUUID } from 'node:crypto' +import { writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { exportVenueSeeds } from '../data/exportVenues.js' +import { loadCompletedTrondheimExport } from '../data/sourceExports.js' + +async function main() { + if (process.argv.includes('--help')) { + console.log( + 'Usage: npm --prefix server run data:export\nConverts the latest completed Trondheim export into a validated seed file. No API calls or database writes.', + ) + return + } + if (process.argv.length > 2) + throw new Error('Unknown option. Use --help for usage.') + const directory = fileURLToPath(new URL('../../imports/', import.meta.url)) + const source = await loadCompletedTrondheimExport(directory) + const venues = exportVenueSeeds(source.payload, source.name) + const name = `venue-seed-${new Date().toISOString().replaceAll(':', '-')}-${randomUUID()}.json` + await writeFile( + join(directory, name), + `${JSON.stringify(venues, null, 2)}\n`, + { flag: 'wx' }, + ) + console.log(`Source: ${source.name}`) + console.log( + `Saved ${venues.length} validated venues to ${join(directory, name)}`, + ) + for (const status of ['normalized', 'missing', 'needs-review']) + console.log( + `Opening hours ${status}: ${venues.filter((venue) => venue.rawData?.openingHoursStatus === status).length}.`, + ) + console.log( + 'Missing or uncertain hours remain []; original expressions and review reasons are retained in rawData.', + ) + console.log( + `Validate without database access: npm --prefix server run seed:venues -- --file imports/${name} --dry-run`, + ) + console.log('No API calls or database writes were made.') +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : 'Venue export failed.') + process.exitCode = 1 +}) diff --git a/server/tests/database.test.ts b/server/tests/database.test.ts new file mode 100644 index 0000000..42b9577 --- /dev/null +++ b/server/tests/database.test.ts @@ -0,0 +1,275 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { PGlite } from '@electric-sql/pglite' +import { MigrationBuilder } from 'node-pg-migrate' +import { ApolloServer } from '@apollo/server' +import { + up as createVenueTables, + down as dropVenueTables, +} from '../migrations/1791904375317_create-venues.js' +import { + up as completeModel, + down as removeModelFields, +} from '../migrations/1791907200000_complete-venue-model.js' +import { importVenues, type ImportDatabase } from '../src/data/importVenues.js' +import { createResolvers, typeDefs } from '../src/graphql/schema.js' +import { venueFixture } from './fixtures.js' + +async function migrate(db: PGlite, migration: typeof createVenueTables) { + const builder = new MigrationBuilder( + {} as ConstructorParameters[0], + undefined, + false, + console, + ) + await migration(builder) + await db.exec(builder.getSql()) +} + +function adapter(db: PGlite): ImportDatabase { + return { + connect: async () => ({ + query: (sql, values) => db.query>(sql, values), + release() {}, + }), + } +} + +async function database() { + const db = new PGlite() + await migrate(db, createVenueTables) + await migrate(db, completeModel) + return db +} + +test('the additive migration preserves legacy IDs and initializes nullable fields and review counts', async () => { + const db = new PGlite() + try { + await migrate(db, createVenueTables) + const before = await db.query<{ id: string }>(`INSERT INTO venues + (name, type, city, area, address, latitude, longitude) + VALUES ('Legacy fixture', 'PUB', 'Trondheim', 'Midtbyen', 'Fixture 1', 63.43, 10.39) RETURNING id`) + await migrate(db, completeModel) + const after = await db.query>( + 'SELECT id, price_level, rating, review_count, description, image_url FROM venues', + ) + assert.deepEqual(after.rows[0], { + id: before.rows[0].id, + price_level: null, + rating: null, + review_count: 0, + description: null, + image_url: null, + }) + await migrate(db, removeModelFields) + assert.equal( + (await db.query>('SELECT id FROM venues')).rows[0] + .id, + before.rows[0].id, + ) + await migrate(db, dropVenueTables) + assert.equal( + ( + await db.query>( + "SELECT to_regclass('venues') AS table_name", + ) + ).rows[0].table_name, + null, + ) + } finally { + await db.close() + } +}) + +test('repeat imports update source data while preserving IDs, app reviews and curated optional fields', async () => { + const db = await database() + try { + const venue = venueFixture({ + priceLevel: 2, + description: 'Curated description', + imageUrl: 'https://example.com/image.jpg', + }) + assert.deepEqual(await importVenues(adapter(db), [venue]), { + inserted: 1, + updated: 0, + }) + const initial = ( + await db.query>('SELECT id FROM venues') + ).rows[0].id + await db.exec('UPDATE venues SET rating = 4.5, review_count = 3') + const update = venueFixture({ + name: 'Updated fixture', + rawData: { openingHoursRaw: 'Mo off', geoapifyPlaceId: 'changed-id' }, + }) + assert.deepEqual(await importVenues(adapter(db), [update]), { + inserted: 0, + updated: 1, + }) + const row = ( + await db.query>('SELECT * FROM venues') + ).rows[0] + assert.equal(row.id, initial) + assert.equal(row.name, 'Updated fixture') + assert.equal(row.rating, 4.5) + assert.equal(row.review_count, 3) + assert.equal(row.price_level, 2) + assert.equal(row.description, 'Curated description') + assert.equal(row.image_url, 'https://example.com/image.jpg') + const source = ( + await db.query>('SELECT * FROM venue_sources') + ).rows[0] + assert.equal( + new Date(source.fetched_at as string).toISOString(), + venue.sourceFetchedAt, + ) + assert.deepEqual(source.raw_data, update.rawData) + await importVenues(adapter(db), [venueFixture({ rawData: null })]) + assert.deepEqual( + ( + await db.query>( + 'SELECT raw_data FROM venue_sources', + ) + ).rows[0].raw_data, + update.rawData, + ) + assert.equal( + ( + await db.query>( + 'SELECT COUNT(*)::int AS count FROM venues', + ) + ).rows[0].count, + 1, + ) + } finally { + await db.close() + } +}) + +test('a mid-import failure rolls back earlier updates, new venues and source records together', async () => { + const db = await database() + try { + await importVenues(adapter(db), [venueFixture()]) + let released = false + const failing: ImportDatabase = { + connect: async () => ({ + query: async (sql, values) => { + if (sql.includes('INSERT INTO venue_sources')) + throw new Error('Injected failure') + return db.query>(sql, values) + }, + release() { + released = true + }, + }), + } + await assert.rejects( + importVenues(failing, [ + venueFixture({ name: 'Should roll back' }), + venueFixture({ sourceId: 'openstreetmap:n:456' }), + ]), + /Injected failure/, + ) + assert.equal(released, true) + assert.equal( + (await db.query>('SELECT name FROM venues')) + .rows[0].name, + 'Fixture pub', + ) + assert.equal( + ( + await db.query>( + 'SELECT COUNT(*)::int AS count FROM venues', + ) + ).rows[0].count, + 1, + ) + assert.equal( + ( + await db.query>( + 'SELECT COUNT(*)::int AS count FROM venue_sources', + ) + ).rows[0].count, + 1, + ) + } finally { + await db.close() + } +}) + +test('PostgreSQL constraints reject invalid coordinates, prices, ratings and counts', async () => { + const db = await database() + try { + await importVenues(adapter(db), [venueFixture()]) + for (const assignment of [ + 'latitude = 160', + 'longitude = 240', + 'price_level = 4', + 'rating = 6', + 'review_count = -1', + "opening_hours = '{}'::jsonb", + ]) + await assert.rejects( + db.exec(`UPDATE venues SET ${assignment}`), + /check constraint/, + ) + } finally { + await db.close() + } +}) + +test('GraphQL exposes all 16 fields, multiple daily intervals and null for unknown or malformed IDs', async () => { + const db = await database() + const server = new ApolloServer({ + typeDefs, + resolvers: createResolvers({ + query: (sql, values) => db.query>(sql, values), + }), + }) + try { + await importVenues(adapter(db), [ + venueFixture({ + openingHours: [ + { day: 'MONDAY', opens: '00:00', closes: '02:30', closed: false }, + { day: 'MONDAY', opens: '18:00', closes: '24:00', closed: false }, + ], + }), + ]) + const query = + '{ health databaseStatus venues { id name type city area address latitude longitude priceLevel rating reviewCount description openingHours { day opens closes closed } tags imageUrl websiteUrl } }' + const result = await server.executeOperation({ query }) + assert.equal(result.body.kind, 'single') + if (result.body.kind !== 'single') + throw new Error('Expected single response') + assert.equal(result.body.singleResult.errors, undefined) + const data = result.body.singleResult.data as Record + assert.equal(data.health, 'ok') + assert.equal(data.databaseStatus, 'connected') + const venues = data.venues as Record[] + assert.equal(Object.keys(venues[0]).length, 16) + assert.equal(venues[0].rating, null) + assert.equal(venues[0].reviewCount, 0) + assert.equal((venues[0].openingHours as unknown[]).length, 2) + for (const id of [ + venues[0].id, + '00000000-0000-0000-0000-000000000000', + 'not-a-uuid', + ]) { + const response = await server.executeOperation({ + query: 'query($id: ID!) { venue(id: $id) { id } }', + variables: { id }, + }) + assert.equal(response.body.kind, 'single') + if (response.body.kind !== 'single') + throw new Error('Expected single response') + assert.equal(response.body.singleResult.errors, undefined) + const actual = response.body.singleResult.data?.venue as { + id: string + } | null + if (id === venues[0].id) assert.equal(actual?.id, id) + else assert.equal(actual, null) + } + } finally { + await server.stop() + await db.close() + } +}) diff --git a/server/tests/exportVenues.test.ts b/server/tests/exportVenues.test.ts new file mode 100644 index 0000000..9ffaff8 --- /dev/null +++ b/server/tests/exportVenues.test.ts @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { exportVenueSeeds } from '../src/data/exportVenues.js' +import { exportFixture } from './fixtures.js' + +test('completed source exports become flat seed rows with stable IDs and original metadata', () => { + const source = exportFixture() + const result = exportVenueSeeds(source, 'source.json')[0] + assert.equal(result.sourceId, 'openstreetmap:n:123') + assert.equal(result.type, 'PUB') + assert.equal(result.sourceFetchedAt, source.fetchedAt) + assert.equal(result.rawData?.attribution, source.attribution) + assert.deepEqual(result.rawData?.raw, source.places[0].raw) + assert.equal(result.rawData?.openingHoursRaw, 'Mo 20:00-02:30') + assert.equal(result.rawData?.openingHoursStatus, 'normalized') + assert.deepEqual(result.openingHours.slice(0, 2), [ + { day: 'MONDAY', opens: '20:00', closes: '24:00', closed: false }, + { day: 'TUESDAY', opens: '00:00', closes: '02:30', closed: false }, + ]) + assert.ok(!Object.hasOwn(result, 'id')) +}) + +test('uncertain hours remain unknown but retain the expression and review reason', () => { + const source = exportFixture() + source.places[0].availableFields.openingHoursRaw = 'Sa 12:00+' + const result = exportVenueSeeds(source, 'source.json')[0] + assert.deepEqual(result.openingHours, []) + assert.equal(result.rawData?.openingHoursStatus, 'needs-review') + assert.equal(result.rawData?.openingHoursRaw, 'Sa 12:00+') + assert.equal(result.rawData?.openingHoursReviewReason, 'uncertain-interval') +}) + +test('incomplete exports and missing retrieval metadata cannot become seeds', () => { + assert.throws(() => + exportVenueSeeds( + { ...exportFixture(), pagination: { exhausted: false, capped: true } }, + 'source.json', + ), + ) + assert.throws( + () => + exportVenueSeeds( + { ...exportFixture(), fetchedAt: undefined }, + 'source.json', + ), + /retrieval time/, + ) + assert.throws( + () => + exportVenueSeeds( + { ...exportFixture(), attribution: undefined }, + 'source.json', + ), + /attribution/, + ) +}) diff --git a/server/tests/fixtures.ts b/server/tests/fixtures.ts new file mode 100644 index 0000000..689173e --- /dev/null +++ b/server/tests/fixtures.ts @@ -0,0 +1,62 @@ +import type { VenueSeed } from '../src/data/venueSeed.js' + +export function venueFixture(overrides: Partial = {}): VenueSeed { + return { + sourceId: 'openstreetmap:n:123', + name: 'Fixture pub', + type: 'PUB', + city: 'Trondheim', + area: 'Midtbyen', + address: 'Fixture street 1', + latitude: 63.43, + longitude: 10.39, + priceLevel: null, + rating: null, + reviewCount: 0, + description: null, + openingHours: [], + tags: [], + imageUrl: null, + websiteUrl: null, + sourceFetchedAt: '2026-10-13T15:35:36.455Z', + rawData: { + openingHoursStatus: 'missing', + openingHoursRaw: null, + attribution: 'Powered by Geoapify | © OpenStreetMap contributors', + }, + ...overrides, + } +} + +export function exportFixture() { + return { + version: 1, + provider: 'Geoapify', + fetchedAt: '2026-10-13T15:35:36.455Z', + attribution: 'Powered by Geoapify | © OpenStreetMap contributors', + search: { trondheim: true, allPages: true, details: true, offset: 0 }, + pagination: { exhausted: true, capped: false }, + places: [ + { + source: { osmType: 'n', osmId: '123', geoapifyPlaceId: 'fixture-id' }, + availableFields: { + name: 'Fixture pub', + type: 'Pub', + city: 'Trondheim', + area: 'Midtbyen', + address: 'Fixture street 1', + latitude: 63.43, + longitude: 10.39, + description: null, + websiteUrl: null, + categories: ['catering.pub'], + openingHoursRaw: 'Mo 20:00-02:30' as string | null, + }, + raw: { + place: { osm_id: 123 }, + details: { opening_hours: 'Mo 20:00-02:30' }, + }, + }, + ], + } +} diff --git a/server/tests/venueSeed.test.ts b/server/tests/venueSeed.test.ts new file mode 100644 index 0000000..07e30f6 --- /dev/null +++ b/server/tests/venueSeed.test.ts @@ -0,0 +1,196 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + requireDevelopmentDatabase, + validateVenues, +} from '../src/data/venueSeed.js' +import { importVenues } from '../src/data/importVenues.js' +import { venueFixture } from './fixtures.js' + +test('legacy seed fields remain supported and unknown optional values remain null', () => { + const { + priceLevel, + rating, + reviewCount, + description, + imageUrl, + sourceFetchedAt, + rawData, + ...legacy + } = venueFixture() + const result = validateVenues([legacy])[0] + assert.equal(result.priceLevel, null) + assert.equal(result.rating, null) + assert.equal(result.reviewCount, 0) + assert.equal(result.description, null) + assert.equal(result.imageUrl, null) + assert.equal(result.rawData, null) + assert.deepEqual(result.openingHours, []) +}) + +test('the entire batch is validated before a database connection is acquired', async () => { + let connected = false + const database = { + connect: async (): Promise => { + connected = true + throw new Error('unexpected connection') + }, + } + await assert.rejects( + importVenues(database, [ + venueFixture(), + venueFixture({ sourceId: 'second', latitude: 160 }), + ]), + /latitude/, + ) + assert.equal(connected, false) + assert.deepEqual(await importVenues(database, []), { + inserted: 0, + updated: 0, + }) + assert.equal(connected, false) +}) + +test('coordinates reject the invalid values accepted by the original seed script', () => { + for (const override of [ + { latitude: 160 }, + { longitude: 240 }, + { latitude: -91 }, + { longitude: -181 }, + { latitude: NaN }, + { longitude: Infinity }, + ]) + assert.throws( + () => validateVenues([venueFixture(override)]), + /latitude|longitude/, + ) + assert.doesNotThrow(() => + validateVenues([venueFixture({ latitude: -90, longitude: 180 })]), + ) +}) + +test('calendar-day hours allow midnight splits and multiple non-overlapping intervals', () => { + const hours = [ + { day: 'MONDAY' as const, opens: '20:00', closes: '24:00', closed: false }, + { day: 'TUESDAY' as const, opens: '00:00', closes: '02:30', closed: false }, + { day: 'TUESDAY' as const, opens: '18:00', closes: '24:00', closed: false }, + { day: 'WEDNESDAY' as const, opens: null, closes: null, closed: true }, + ] + assert.deepEqual( + validateVenues([venueFixture({ openingHours: hours })])[0].openingHours, + hours, + ) +}) + +test('invalid, unsplit overnight, overlapping and contradictory opening hours fail', () => { + const open = { + day: 'MONDAY' as const, + opens: '18:00', + closes: '23:00', + closed: false, + } + const invalid: unknown[][] = [ + [{ ...open, opens: '99:99' }], + [{ ...open, closes: 'not-a-time' }], + [{ ...open, closes: '24:01' }], + [{ ...open, opens: '24:00' }], + [{ ...open, closes: '02:00' }], + [{ ...open, opens: null }], + [{ ...open, closed: true }], + [open, { ...open, opens: '22:00' }], + [open, { ...open, opens: null, closes: null, closed: true }], + [{ ...open, day: 'UNKNOWN' }], + ] + for (const hours of invalid) + assert.throws( + () => validateVenues([{ ...venueFixture(), openingHours: hours }]), + /openingHours/, + ) +}) + +test('URLs reject scripts, malformed addresses and embedded credentials', () => { + for (const key of ['websiteUrl', 'imageUrl']) { + for (const value of [ + 'javascript:alert(1)', + 'not-a-url', + 'ftp://example.com', + 'https://user:secret@example.com', + ]) + assert.throws( + () => validateVenues([{ ...venueFixture(), [key]: value }]), + new RegExp(key), + ) + } + assert.equal( + validateVenues([venueFixture({ websiteUrl: 'https://example.com' })])[0] + .websiteUrl, + 'https://example.com/', + ) +}) + +test('missing text, duplicate identities and invalid optional values are rejected', () => { + for (const key of ['sourceId', 'name', 'city', 'area', 'address']) + assert.throws( + () => validateVenues([{ ...venueFixture(), [key]: ' ' }]), + new RegExp(key), + ) + assert.throws( + () => + validateVenues([ + venueFixture(), + venueFixture({ sourceId: ' openstreetmap:n:123 ' }), + ]), + /duplicates/, + ) + for (const priceLevel of [0, 4, 1.5, '2']) + assert.throws( + () => validateVenues([{ ...venueFixture(), priceLevel }]), + /priceLevel/, + ) + assert.throws( + () => validateVenues([{ ...venueFixture(), sourceFetchedAt: 'yesterday' }]), + /sourceFetchedAt/, + ) + assert.throws( + () => validateVenues([{ ...venueFixture(), rawData: [] }]), + /rawData/, + ) + assert.throws(() => validateVenues([venueFixture({ tags: [''] })]), /tags/) + assert.throws(() => validateVenues({ venues: [] }), /JSON array/) +}) + +test('imports cannot supply external ratings or review counts', () => { + assert.throws( + () => validateVenues([{ ...venueFixture(), rating: 4.5 }]), + /app reviews/, + ) + assert.throws( + () => validateVenues([{ ...venueFixture(), reviewCount: 10 }]), + /app reviews/, + ) +}) + +test('database guard rejects placeholders and other targets without exposing credentials', () => { + for (const url of [ + undefined, + 'invalid', + 'https://user:secret@example.com/tapfinder_dev', + 'postgresql://user:secret@localhost/tapfinder', + 'postgresql://user:REPLACE_WITH_PASSWORD@localhost/tapfinder_dev', + ]) { + assert.throws( + () => requireDevelopmentDatabase(url), + (error: unknown) => { + assert.ok(error instanceof Error) + assert.match(error.message, /tapfinder_dev/) + assert.doesNotMatch(error.message, /secret|REPLACE_WITH_PASSWORD/) + return true + }, + ) + } + assert.doesNotThrow(() => + requireDevelopmentDatabase( + 'postgresql://user:configured@localhost:5433/tapfinder_dev', + ), + ) +}) diff --git a/server/tsconfig.check.json b/server/tsconfig.check.json new file mode 100644 index 0000000..64eb057 --- /dev/null +++ b/server/tsconfig.check.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": ".", + "noEmit": true, + "allowJs": true, + "checkJs": false + }, + "include": ["src/**/*.ts", "scripts/**/*.ts", "tests/**/*.ts"] +}