From 580a8a3972b48851b3fdab12791b26c6ed9521e0 Mon Sep 17 00:00:00 2001 From: albdert Date: Wed, 21 Oct 2026 10:39:00 +0200 Subject: [PATCH 1/2] new import script and review table --- db/grants.sql | 2 ++ db/load.py | 47 +++++++++++++++++++++++++++++++---------------- db/schema.sql | 5 ++--- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/db/grants.sql b/db/grants.sql index 733b740..7353a9c 100644 --- a/db/grants.sql +++ b/db/grants.sql @@ -10,4 +10,6 @@ GRANT INSERT ON users TO webapp; GRANT USAGE ON SEQUENCE users_id_seq TO webapp; GRANT INSERT ON reviews TO webapp; +GRANT UPDATE (rating, body, hours_played) ON reviews TO webapp; +GRANT DELETE ON reviews TO webapp; GRANT USAGE ON SEQUENCE reviews_id_seq TO webapp; diff --git a/db/load.py b/db/load.py index 3e6f825..c1f23b5 100644 --- a/db/load.py +++ b/db/load.py @@ -1,3 +1,4 @@ +import html import os import sys import tempfile @@ -10,6 +11,8 @@ PARQUET = os.environ.get("PARQUET_PATH", str(Path.home() / "data" / "videogames_data.parquet")) DATABASE_URL = os.environ["DATABASE_URL"] FORCE = "--force" in sys.argv +# Tags used by fewer games than this are dropped (long tail of one-off, noisy tags). +MIN_TAG_GAMES = int(os.environ.get("MIN_TAG_GAMES", "50")) # (parquet column, lookup table, junction table, junction fk column) RELATIONS = [ @@ -23,20 +26,28 @@ ] GAME_COLUMNS = ( - "id, slug, name, name_original, alternative_names, search_names, released, " + "id, slug, name, alternative_names, search_names, released, " "background_image, background_image_additional, screenshots, rating, ratings_count, " - "added, metacritic, playtime, esrb_rating, description, website, dominant_color" + "added, metacritic, playtime, esrb_rating, description, website" ) def clean(col: str) -> str: - """Empty strings and the text 'None' (a Python null written as a string) become NULL.""" - return f"NULLIF(NULLIF(trim({col}), ''), 'None')" + """Empty strings and the texts 'None'/'nan' (nulls written as strings) become NULL.""" + return f"CASE WHEN lower(trim({col})) IN ('', 'none', 'nan') THEN NULL ELSE trim({col}) END" -# Pipe-separated names, trimmed, empty entries removed. +# Pipe-separated names, trimmed, empty and null-like entries removed. ALT_NAMES = ( f"list_filter(list_transform(string_split({clean('alternative_names')}, '|'), s -> trim(s)), " - "s -> s NOT IN ('', 'None'))" + "s -> lower(s) NOT IN ('', 'none', 'nan'))" +) + +# description_raw is plain text; description is the fallback (tags stripped, HTML entities kept). +# Both get entities decoded and leftover Markdown headings / HTML tags removed. +DESCRIPTION = ( + "NULLIF(trim(regexp_replace(unescape(COALESCE(" + f"{clean('description_raw')}, {clean('description')})), " + r"'#{2,}\s*|<[^>]+>', '', 'g')), '')" ) # short_screenshots holds Python-style dicts ({'id': 123, 'image': '...'}), not JSON. @@ -68,10 +79,11 @@ def pg_array(list_expr: str) -> str: def export_csvs(out: Path) -> None: con = duckdb.connect() + con.create_function("unescape", html.unescape, ["VARCHAR"], "VARCHAR") con.execute( f"""CREATE VIEW src AS SELECT * FROM read_parquet('{PARQUET}') - WHERE added > 0 AND name IS NOT NULL""" + WHERE added > 0 AND name IS NOT NULL AND trim(name) <> ''""" ) con.execute( f"""COPY ( @@ -79,11 +91,11 @@ def export_csvs(out: Path) -> None: id, slug, name, - NULLIF({clean('name_original')}, name) AS name_original, {pg_array(ALT_NAMES)} AS alternative_names, - concat_ws(' | ', name, NULLIF({clean('name_original')}, name), - array_to_string({ALT_NAMES}, ' | ')) AS search_names, - TRY_CAST(released AS DATE) AS released, + concat_ws(' | ', name, array_to_string({ALT_NAMES}, ' | ')) AS search_names, + -- Dates far in the future are placeholders, not real release dates. + CASE WHEN TRY_CAST(released AS DATE) <= current_date + INTERVAL 5 YEAR + THEN TRY_CAST(released AS DATE) END AS released, {clean('background_image')} AS background_image, {clean('background_image_additional')} AS background_image_additional, {pg_array(SCREENSHOTS)} AS screenshots, @@ -93,9 +105,8 @@ def export_csvs(out: Path) -> None: TRY_CAST(metacritic AS INTEGER) AS metacritic, TRY_CAST(playtime AS INTEGER) AS playtime, {ESRB} AS esrb_rating, - {clean('description_raw')} AS description, - {clean('website')} AS website, - {clean('dominant_color')} AS dominant_color + {DESCRIPTION} AS description, + {clean('website')} AS website FROM src ) TO '{out}/games.csv' (FORMAT csv, HEADER)""" ) @@ -105,10 +116,14 @@ def export_csvs(out: Path) -> None: SELECT id AS game_id, trim(unnest(string_split({col}, '|'))) AS name FROM src WHERE {col} IS NOT NULL""" ) + min_games = MIN_TAG_GAMES if table == "tags" else 1 con.execute( - """CREATE OR REPLACE TEMP TABLE lk AS + f"""CREATE OR REPLACE TEMP TABLE lk AS SELECT row_number() OVER (ORDER BY name) AS id, name - FROM (SELECT DISTINCT name FROM pairs WHERE name NOT IN ('', 'None'))""" + FROM (SELECT name FROM pairs + WHERE name NOT IN ('', 'None') + GROUP BY name + HAVING count(DISTINCT game_id) >= {min_games})""" ) con.execute(f"COPY (SELECT id, name FROM lk) TO '{out}/{table}.csv' (FORMAT csv, HEADER)") con.execute( diff --git a/db/schema.sql b/db/schema.sql index cbf51ed..e98637a 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -8,7 +8,6 @@ CREATE TABLE games ( id BIGINT PRIMARY KEY, slug TEXT, name TEXT NOT NULL, - name_original TEXT, alternative_names TEXT[], search_names TEXT NOT NULL, released DATE, @@ -22,8 +21,7 @@ CREATE TABLE games ( playtime INTEGER, esrb_rating TEXT, description TEXT, - website TEXT, - dominant_color TEXT + website TEXT ); CREATE TABLE genres (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); @@ -84,6 +82,7 @@ CREATE TABLE reviews ( user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, rating SMALLINT NOT NULL CHECK (rating BETWEEN 1 AND 5), body TEXT NOT NULL CHECK (char_length(body) BETWEEN 1 AND 2000), + hours_played SMALLINT CHECK (hours_played BETWEEN 1 AND 10000), created_at TIMESTAMPTZ NOT NULL DEFAULT now(), UNIQUE (game_id, user_id) ); From b02658f03010c6eed52cd341b3b002d2684b5bf5 Mon Sep 17 00:00:00 2001 From: Albert Paulsen Lindberg Date: Wed, 21 Oct 2026 10:56:26 +0200 Subject: [PATCH 2/2] requirement numpy --- db/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/db/requirements.txt b/db/requirements.txt index c3eb184..99a49dc 100644 --- a/db/requirements.txt +++ b/db/requirements.txt @@ -1,2 +1,3 @@ duckdb +numpy psycopg[binary]