From f534c1e89d39a5243a8bf33c843e599c8d76ac07 Mon Sep 17 00:00:00 2001 From: Lea Tveteraas Date: Thu, 8 Oct 2026 10:48:08 +0200 Subject: [PATCH 1/2] =?UTF-8?q?oppgave/Endret=20kategorier=20og=20henter?= =?UTF-8?q?=20data=20med=20importscript=20+=20endring=20i=20index.ts=20for?= =?UTF-8?q?=20=C3=A5=20passe=20med=20de=20samlede=20katgoriene?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../db/migrations/003_add_food_category.sql | 14 + backend/db/seeds/food_categories.sql | 76 ++++ backend/scripts/import.ts | 421 ++++++++++++++++-- 3 files changed, 478 insertions(+), 33 deletions(-) create mode 100644 backend/db/migrations/003_add_food_category.sql create mode 100644 backend/db/seeds/food_categories.sql diff --git a/backend/db/migrations/003_add_food_category.sql b/backend/db/migrations/003_add_food_category.sql new file mode 100644 index 0000000..d9e9e1d --- /dev/null +++ b/backend/db/migrations/003_add_food_category.sql @@ -0,0 +1,14 @@ +-- 003_add_food_category.sql +-- Wine not? – grovere matkategorier til Find wine by food +-- Kjør med: psql -d winenot -v ON_ERROR_STOP=1 -f backend/db/migrations/003_add_food_category.sql +-- +-- Hver matverdi fra X-Wines (f.eks. 'Rich Fish', 'Shellfish') får én kategori (f.eks. 'Fish & seafood'). +-- Selve grupperingen ligger i db/seeds/food_categories.sql og kjøres av importskriptet. + +BEGIN; + +ALTER TABLE foods ADD COLUMN category text; + +CREATE INDEX foods_category_idx ON foods (category); + +COMMIT; diff --git a/backend/db/seeds/food_categories.sql b/backend/db/seeds/food_categories.sql new file mode 100644 index 0000000..96361c1 --- /dev/null +++ b/backend/db/seeds/food_categories.sql @@ -0,0 +1,76 @@ +-- food_categories.sql +-- Grupperer matverdiene fra X-Wines (kolonnen Harmonize) i kategorier for Find wine by food. +-- +-- Kjøres automatisk av importskriptet etter at maten er satt inn, og kan også kjøres alene: +-- psql -d winenot -1 -v ON_ERROR_STOP=1 -f backend/db/seeds/food_categories.sql +-- (-1 kjører hele filen som én transaksjon. Filen har ikke egen BEGIN/COMMIT, +-- fordi importskriptet kjører den inne i sin egen transaksjon.) +-- +-- Endre en kategori: bytt navnet i høyre kolonne og kjør filen på nytt. +-- Ny matverdi uten kategori: importskriptet lister dem opp. Legg dem til her. +-- Matverdier som ikke står her, får category = NULL og vises ikke i Find wine by food. + +UPDATE foods SET category = NULL; + +UPDATE foods AS f +SET category = m.category +FROM (VALUES + -- Kjøtt + ('Beef', 'Meat'), + ('Lamb', 'Meat'), + ('Veal', 'Meat'), + ('Pork', 'Meat'), + ('Game Meat', 'Meat'), + ('Barbecue', 'Meat'), + ('Grilled', 'Meat'), + + -- Fjærkre + ('Poultry', 'Poultry'), + ('Chicken', 'Poultry'), + + -- Fisk og sjømat + ('Fish', 'Fish & seafood'), + ('Rich Fish', 'Fish & seafood'), + ('Lean Fish', 'Fish & seafood'), + ('Seafood', 'Fish & seafood'), + ('Shellfish', 'Fish & seafood'), + + -- Ost + ('Maturated Cheese', 'Cheese'), + ('Soft Cheese', 'Cheese'), + ('Hard Cheese', 'Cheese'), + ('Blue Cheese', 'Cheese'), + ('Goat Cheese', 'Cheese'), + + -- Pasta, pizza og gryter + ('Pasta', 'Pasta, pizza & stews'), + ('Pizza', 'Pasta, pizza & stews'), + ('Risotto', 'Pasta, pizza & stews'), + ('Tomato Dishes', 'Pasta, pizza & stews'), + ('Light Stews', 'Pasta, pizza & stews'), + + -- Vegetar + ('Vegetarian', 'Vegetarian'), + ('Salad', 'Vegetarian'), + ('Mushrooms', 'Vegetarian'), + + -- Sterk mat + ('Spicy Food', 'Spicy food'), + + -- Småretter og aperitiff + ('Appetizer', 'Snacks & starters'), + ('Snack', 'Snacks & starters'), + ('Aperitif', 'Snacks & starters'), + ('Cured Meat', 'Snacks & starters'), + ('Cold Cuts', 'Snacks & starters'), + ('French Fries', 'Snacks & starters'), + + -- Dessert + ('Sweet Dessert', 'Dessert'), + ('Fruit Dessert', 'Dessert'), + ('Cake', 'Dessert'), + ('Chocolate', 'Dessert'), + ('Fruit', 'Dessert'), + ('Cream', 'Dessert') +) AS m(food, category) +WHERE f.name = m.food; diff --git a/backend/scripts/import.ts b/backend/scripts/import.ts index e8b9d77..bfacfe3 100644 --- a/backend/scripts/import.ts +++ b/backend/scripts/import.ts @@ -1,46 +1,401 @@ -import { createReadStream } from "node:fs"; +// Importerer X-Wines-datasettet til PostgreSQL. +// +// Bruk (fra backend/): +// npm run import testfilene i data/ +// npm run import -- f.eks. full versjon +// +// Skriptet kan kjøres flere ganger. Viner oppdateres, og favoritter beholdes. + +import { createReadStream, readFileSync } from "node:fs"; import { parse } from "csv-parse"; +import type { PoolClient } from "pg"; import { pool } from "../src/db.js"; -const WINES_FILE = "data/XWines_Test_100_wines.csv"; +const WINES_FILE = process.argv[2] ?? "data/XWines_Test_100_wines.csv"; +const RATINGS_FILE = process.argv[3] ?? "data/XWines_Test_1K_ratings.csv"; +const BATCH_SIZE = 1000; +const FOOD_CATEGORIES_FILE = "db/seeds/food_categories.sql"; +const CURRENT_YEAR = new Date().getFullYear(); -async function main() { - const parser = createReadStream(WINES_FILE).pipe(parse({ columns: true })); +const BODY_LEVELS = new Set([ + "Very light-bodied", + "Light-bodied", + "Medium-bodied", + "Full-bodied", + "Very full-bodied", +]); +const ACIDITY_LEVELS = new Set(["Low", "Medium", "High"]); + +type CsvRow = Record; + +interface Wine { + id: number; + name: string; + type: string; + elaborate: string | null; + abv: number | null; + body: string | null; + acidity: string | null; + wineryId: number; + wineryName: string; + website: string | null; + regionId: number; + regionName: string; + countryCode: string; + countryName: string; + grapes: string[]; + foods: string[]; + vintages: number[]; +} - let count = 0; - for await (const row of parser) { - await pool.query( - `INSERT INTO countries (code, name) VALUES ($1, $2) - ON CONFLICT (code) DO NOTHING`, - [row.Code, row.Country] - ); +type Validation = { ok: true; wine: Wine; warnings: string[] } | { ok: false; reason: string }; - await pool.query( - `INSERT INTO regions (id, name, country_code) VALUES ($1, $2, $3) - ON CONFLICT (id) DO NOTHING`, - [row.RegionID, row.RegionName, row.Code] - ); +// ---------- Hjelpefunksjoner for rensing ---------- - await pool.query( - `INSERT INTO wineries (id, name, website) VALUES ($1, $2, $3) - ON CONFLICT (id) DO NOTHING`, - [row.WineryID, row.WineryName, row.Website || null] - ); +function text(value: string | undefined): string { + return (value ?? "").trim(); +} - await pool.query( - `INSERT INTO wines - (id, name, type, elaborate, abv, body, acidity, winery_id, region_id, country_code) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (id) DO NOTHING`, - [row.WineID, row.WineName, row.Type, row.Elaborate, row.ABV, - row.Body, row.Acidity, row.WineryID, row.RegionID, row.Code] - ); +function orNull(value: string): string | null { + return value === "" ? null : value; +} - count++; +function toId(value: string | undefined): number | null { + const n = Number(text(value)); + return Number.isInteger(n) && n > 0 ? n : null; +} + +// "['Merlot', \"Nero d'Avola\"]" -> ["Merlot", "Nero d'Avola"] +function parseList(value: string): string[] { + const items: string[] = []; + for (const match of value.matchAll(/'([^']*)'|"([^"]*)"/g)) { + const item = (match[1] ?? match[2] ?? "").trim(); + if (item !== "") items.push(item); } + return [...new Set(items)]; +} + +// "[2020, 2019, 'N.V.']" -> [2020, 2019] +function parseVintages(value: string): number[] { + const years = (value.match(/\d{4}/g) ?? []) + .map(Number) + .filter((year) => year >= 1800 && year <= CURRENT_YEAR); + return [...new Set(years)].sort((a, b) => b - a); +} + +// ---------- Validering av én vinrad ---------- + +function validateWine(row: CsvRow): Validation { + const fail = (reason: string): Validation => ({ ok: false, reason }); + const warnings: string[] = []; + + const id = toId(row.WineID); + if (id === null) return fail("ugyldig WineID"); + + const name = text(row.WineName); + if (name === "") return fail("mangler WineName"); + + const type = text(row.Type); + if (type === "") return fail("mangler Type"); + + const wineryId = toId(row.WineryID); + const wineryName = text(row.WineryName); + if (wineryId === null || wineryName === "") return fail("mangler produsent"); + + const regionId = toId(row.RegionID); + const regionName = text(row.RegionName); + if (regionId === null || regionName === "") return fail("mangler region"); + + const countryCode = text(row.Code).toUpperCase(); + const countryName = text(row.Country); + if (countryCode === "" || countryName === "") return fail("mangler land"); + + // Valgfrie felt: ugyldig verdi blir NULL, vinen beholdes. + const abvText = text(row.ABV); + const abvNumber = Number(abvText); + let abv: number | null = null; + if (abvText !== "") { + if (Number.isFinite(abvNumber) && abvNumber > 0 && abvNumber <= 50) { + abv = Math.round(abvNumber * 10) / 10; + } else { + warnings.push("ugyldig ABV"); + } + } + + const bodyText = text(row.Body); + const body = BODY_LEVELS.has(bodyText) ? bodyText : null; + if (bodyText !== "" && body === null) warnings.push("ugyldig Body"); + + const acidityText = text(row.Acidity); + const acidity = ACIDITY_LEVELS.has(acidityText) ? acidityText : null; + if (acidityText !== "" && acidity === null) warnings.push("ugyldig Acidity"); + + return { + ok: true, + warnings, + wine: { + id, + name, + type, + elaborate: orNull(text(row.Elaborate)), + abv, + body, + acidity, + wineryId, + wineryName, + website: orNull(text(row.Website)), + regionId, + regionName, + countryCode, + countryName, + grapes: parseList(text(row.Grapes)), + foods: parseList(text(row.Harmonize)), + vintages: parseVintages(text(row.Vintages)), + }, + }; +} + +// ---------- Lesing av filene ---------- + +function countReason(map: Map, reason: string) { + map.set(reason, (map.get(reason) ?? 0) + 1); +} + +async function readWines() { + const wines: Wine[] = []; + const seenIds = new Set(); + const skipped = new Map(); + const warnings = new Map(); + const examples: string[] = []; + let line = 1; // linje 1 er overskriften + + const parser = createReadStream(WINES_FILE).pipe(parse({ columns: true, relax_column_count: true })); + for await (const row of parser as AsyncIterable) { + line++; + let result = validateWine(row); + if (result.ok && seenIds.has(result.wine.id)) result = { ok: false, reason: "duplikat WineID" }; + + if (!result.ok) { + countReason(skipped, result.reason); + if (examples.length < 10) examples.push(`linje ${line}: ${result.reason} (WineID=${text(row.WineID) || "tom"})`); + continue; + } + + for (const warning of result.warnings) countReason(warnings, warning); + seenIds.add(result.wine.id); + wines.push(result.wine); + } + + return { wines, skipped, warnings, examples }; +} + +async function readRatings(wineIds: Set) { + const totals = new Map(); + const skipped = new Map(); + let read = 0; - console.log(`Importerte ${count} viner`); - await pool.end(); + const parser = createReadStream(RATINGS_FILE).pipe(parse({ columns: true, relax_column_count: true })); + for await (const row of parser as AsyncIterable) { + read++; + if (read % 1_000_000 === 0) console.log(` ... ${read.toLocaleString("nb-NO")} vurderinger lest`); + + const wineId = toId(row.WineID); + const rating = Number(text(row.Rating)); + + if (wineId === null || !wineIds.has(wineId)) { + countReason(skipped, "ukjent vin"); + continue; + } + if (!Number.isFinite(rating) || rating < 1 || rating > 5) { + countReason(skipped, "ugyldig rating"); + continue; + } + + const total = totals.get(wineId) ?? { sum: 0, count: 0 }; + total.sum += rating; + total.count++; + totals.set(wineId, total); + } + + return { totals, skipped, read }; +} + +// ---------- Skriving til databasen ---------- + +// Sender radene som JSON i bolker på BATCH_SIZE, i stedet for én spørring per rad. +async function insertInBatches(client: PoolClient, sql: string, rows: object[]) { + for (let i = 0; i < rows.length; i += BATCH_SIZE) { + await client.query(sql, [JSON.stringify(rows.slice(i, i + BATCH_SIZE))]); + } +} + +function uniqueBy(items: T[], key: (item: T) => string | number): T[] { + const map = new Map(); + for (const item of items) if (!map.has(key(item))) map.set(key(item), item); + return [...map.values()]; +} + +async function writeToDatabase(client: PoolClient, wines: Wine[], totals: Map) { + // 1. Oppslagstabellene, i den rekkefølgen fremmednøklene krever + await insertInBatches( + client, + `INSERT INTO countries (code, name) + SELECT code, name FROM jsonb_to_recordset($1::jsonb) AS x(code text, name text) + ON CONFLICT DO NOTHING`, + uniqueBy(wines, (w) => w.countryCode).map((w) => ({ code: w.countryCode, name: w.countryName })) + ); + + await insertInBatches( + client, + `INSERT INTO regions (id, name, country_code) + SELECT id, name, country_code FROM jsonb_to_recordset($1::jsonb) AS x(id int, name text, country_code text) + ON CONFLICT DO NOTHING`, + uniqueBy(wines, (w) => w.regionId).map((w) => ({ id: w.regionId, name: w.regionName, country_code: w.countryCode })) + ); + + await insertInBatches( + client, + `INSERT INTO wineries (id, name, website) + SELECT id, name, website FROM jsonb_to_recordset($1::jsonb) AS x(id int, name text, website text) + ON CONFLICT DO NOTHING`, + uniqueBy(wines, (w) => w.wineryId).map((w) => ({ id: w.wineryId, name: w.wineryName, website: w.website })) + ); + + // 2. Vinene. ON CONFLICT DO UPDATE gjør at en ny kjøring oppdaterer vinene + // i stedet for å slette dem, slik at favoritter ikke forsvinner. + await insertInBatches( + client, + `INSERT INTO wines (id, name, type, elaborate, abv, body, acidity, winery_id, region_id, country_code, vintages) + SELECT id, name, type, elaborate, abv, body, acidity, winery_id, region_id, country_code, vintages + FROM jsonb_to_recordset($1::jsonb) AS x( + id int, name text, type text, elaborate text, abv numeric, body body_level, acidity acidity_level, + winery_id int, region_id int, country_code text, vintages int[]) + ON CONFLICT (id) DO UPDATE SET + name = EXCLUDED.name, type = EXCLUDED.type, elaborate = EXCLUDED.elaborate, abv = EXCLUDED.abv, + body = EXCLUDED.body, acidity = EXCLUDED.acidity, winery_id = EXCLUDED.winery_id, + region_id = EXCLUDED.region_id, country_code = EXCLUDED.country_code, vintages = EXCLUDED.vintages`, + wines.map((w) => ({ + id: w.id, + name: w.name, + type: w.type, + elaborate: w.elaborate, + abv: w.abv, + body: w.body, + acidity: w.acidity, + winery_id: w.wineryId, + region_id: w.regionId, + country_code: w.countryCode, + vintages: w.vintages, + })) + ); + + // 3. Druer og mat, og koblingene til vinene + const grapeLinks = wines.flatMap((w) => w.grapes.map((name) => ({ wine_id: w.id, name }))); + const foodLinks = wines.flatMap((w) => w.foods.map((name) => ({ wine_id: w.id, name }))); + + await insertInBatches( + client, + `INSERT INTO grapes (name) SELECT name FROM jsonb_to_recordset($1::jsonb) AS x(name text) ON CONFLICT DO NOTHING`, + uniqueBy(grapeLinks, (l) => l.name).map((l) => ({ name: l.name })) + ); + await insertInBatches( + client, + `INSERT INTO foods (name) SELECT name FROM jsonb_to_recordset($1::jsonb) AS x(name text) ON CONFLICT DO NOTHING`, + uniqueBy(foodLinks, (l) => l.name).map((l) => ({ name: l.name })) + ); + + await insertInBatches( + client, + `INSERT INTO wine_grapes (wine_id, grape_id) + SELECT x.wine_id, g.id FROM jsonb_to_recordset($1::jsonb) AS x(wine_id int, name text) + JOIN grapes g ON g.name = x.name + ON CONFLICT DO NOTHING`, + grapeLinks + ); + await insertInBatches( + client, + `INSERT INTO wine_foods (wine_id, food_id) + SELECT x.wine_id, f.id FROM jsonb_to_recordset($1::jsonb) AS x(wine_id int, name text) + JOIN foods f ON f.name = x.name + ON CONFLICT DO NOTHING`, + foodLinks + ); + + // 4. Rating: snitt og antall per vin + await insertInBatches( + client, + `UPDATE wines AS w + SET avg_rating = round(x.sum / x.count, 2), rating_count = x.count + FROM jsonb_to_recordset($1::jsonb) AS x(id int, sum numeric, count int) + WHERE w.id = x.id`, + [...totals].map(([id, t]) => ({ id, sum: t.sum, count: t.count })) + ); + + // 5. Matkategorier: grupperingen ligger i en egen SQL-fil, så den er lett å endre + await client.query(readFileSync(FOOD_CATEGORIES_FILE, "utf8")); + + return { grapeLinks: grapeLinks.length, foodLinks: foodLinks.length }; +} + +// ---------- Oppsummering ---------- + +function printCounts(title: string, map: Map) { + if (map.size === 0) return; + console.log(title); + for (const [reason, n] of map) console.log(` ${reason}: ${n}`); +} + +async function main() { + const started = Date.now(); + + console.log(`Leser viner fra ${WINES_FILE}`); + const { wines, skipped, warnings, examples } = await readWines(); + console.log(` ${wines.length} gyldige viner`); + + console.log(`Leser vurderinger fra ${RATINGS_FILE}`); + const ratings = await readRatings(new Set(wines.map((w) => w.id))); + console.log(` ${ratings.read} vurderinger lest, ${ratings.totals.size} viner har rating`); + + console.log("Skriver til databasen"); + const client = await pool.connect(); + try { + await client.query("BEGIN"); + const links = await writeToDatabase(client, wines, ratings.totals); + await client.query("COMMIT"); + console.log(` ${links.grapeLinks} drue-koblinger og ${links.foodLinks} mat-koblinger`); + } catch (error) { + await client.query("ROLLBACK"); + console.error("Importen feilet, ingenting ble lagret."); + throw error; + } finally { + client.release(); + } + + console.log(""); + printCounts("Viner som ble hoppet over:", skipped); + if (examples.length > 0) { + console.log("Eksempler:"); + for (const example of examples) console.log(` ${example}`); + } + printCounts("Felt satt til NULL på grunn av ugyldig verdi:", warnings); + printCounts("Vurderinger som ble hoppet over:", ratings.skipped); + + const uncategorized = await pool.query( + `SELECT f.name, count(*)::int AS wines + FROM foods f JOIN wine_foods wf ON wf.food_id = f.id + WHERE f.category IS NULL + GROUP BY f.name ORDER BY wines DESC` + ); + if (uncategorized.rows.length > 0) { + console.log(`Mat uten kategori (legg dem til i ${FOOD_CATEGORIES_FILE}):`); + for (const row of uncategorized.rows) console.log(` ${row.name}: ${row.wines} viner`); + } + console.log(`Ferdig på ${((Date.now() - started) / 1000).toFixed(1)} s`); } -main(); \ No newline at end of file +main() + .catch((error) => { + console.error(error); + process.exitCode = 1; + }) + .finally(() => pool.end()); \ No newline at end of file From c49d8d0a06142422fb001ac3b424ef9ae532cb90 Mon Sep 17 00:00:00 2001 From: Lea Tveteraas Date: Thu, 8 Oct 2026 10:49:48 +0200 Subject: [PATCH 2/2] =?UTF-8?q?Endringer=20i=20index.ts=20for=20GraphQL-sp?= =?UTF-8?q?=C3=B8rringer=20til=20matkategorier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/src/index.ts | 81 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index bc261f3..bfa85f1 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -3,9 +3,6 @@ import { startStandaloneServer } from "@apollo/server/standalone"; import { pool } from "./db.js"; const typeDefs = `#graphql - type Query { - health: String! - } type Wine { id: ID! @@ -26,6 +23,11 @@ const typeDefs = `#graphql totalCount: Int! } + type FoodCategory { + name: String! + wineCount: Int! + } + type Query { health: String! @@ -33,6 +35,14 @@ const typeDefs = `#graphql limit: Int = 20 offset: Int = 0 ): WinePage! + + foodCategories: [FoodCategory!]! + + winesByFoodCategory( + category: String! + limit: Int = 20 + offset: Int = 0 + ): WinePage! } `; @@ -81,6 +91,71 @@ const resolvers = { totalCount: Number(countResult.rows[0].count), }; }, + + foodCategories: async () => { + const result = await pool.query( + ` + SELECT f.category AS name, count(DISTINCT wf.wine_id)::int AS "wineCount" + FROM foods f + JOIN wine_foods wf ON wf.food_id = f.id + WHERE f.category IS NOT NULL + GROUP BY f.category + ORDER BY f.category + ` + ); + return result.rows; + }, + + winesByFoodCategory: async ( + _: unknown, + args: { category: string; limit?: number; offset?: number } + ) => { + const limit = args.limit ?? 20; + const offset = args.offset ?? 0; + const inCategory = ` + EXISTS ( + SELECT 1 + FROM wine_foods wf + JOIN foods f ON f.id = wf.food_id + WHERE wf.wine_id = w.id AND f.category = $1 + )`; + + const winesResult = await pool.query( + ` + SELECT + w.id, + w.name, + w.type, + w.abv, + w.body, + w.acidity, + wi.name AS winery, + r.name AS region, + c.name AS country, + w.avg_rating AS "avgRating", + w.rating_count AS "ratingCount" + FROM wines w + JOIN wineries wi ON w.winery_id = wi.id + JOIN regions r ON w.region_id = r.id + JOIN countries c ON w.country_code = c.code + WHERE ${inCategory} + ORDER BY w.name, w.id + LIMIT $2 + OFFSET $3 + `, + [args.category, limit, offset] + ); + + const countResult = await pool.query( + `SELECT count(*)::int AS count FROM wines w WHERE ${inCategory}`, + [args.category] + ); + + return { + items: winesResult.rows, + totalCount: countResult.rows[0].count, + }; + }, }, };