Database/data cleanup

db/grants.sql

@@ -10,4 +10,6 @@ GRANT INSERT ON users TO webapp;
 GRANT USAGE ON SEQUENCE users_id_seq TO webapp;
 
 GRANT INSERT ON reviews TO webapp;
+GRANT UPDATE (rating, body, hours_played) ON reviews TO webapp;
+GRANT DELETE ON reviews TO webapp;
 GRANT USAGE ON SEQUENCE reviews_id_seq TO webapp;

db/load.py

@@ -1,3 +1,4 @@
+import html
 import os
 import sys
 import tempfile
@@ -10,6 +11,8 @@
 PARQUET = os.environ.get("PARQUET_PATH", str(Path.home() / "data" / "videogames_data.parquet"))
 DATABASE_URL = os.environ["DATABASE_URL"]
 FORCE = "--force" in sys.argv
+# Tags used by fewer games than this are dropped (long tail of one-off, noisy tags).
+MIN_TAG_GAMES = int(os.environ.get("MIN_TAG_GAMES", "50"))
 
 # (parquet column, lookup table, junction table, junction fk column)
 RELATIONS = [
@@ -23,20 +26,28 @@
 ]
 
 GAME_COLUMNS = (
-    "id, slug, name, name_original, alternative_names, search_names, released, "
+    "id, slug, name, alternative_names, search_names, released, "
     "background_image, background_image_additional, screenshots, rating, ratings_count, "
-    "added, metacritic, playtime, esrb_rating, description, website, dominant_color"
+    "added, metacritic, playtime, esrb_rating, description, website"
 )
 
 def clean(col: str) -> str:
-    """Empty strings and the text 'None' (a Python null written as a string) become NULL."""
-    return f"NULLIF(NULLIF(trim({col}), ''), 'None')"
+    """Empty strings and the texts 'None'/'nan' (nulls written as strings) become NULL."""
+    return f"CASE WHEN lower(trim({col})) IN ('', 'none', 'nan') THEN NULL ELSE trim({col}) END"
 
 
-# Pipe-separated names, trimmed, empty entries removed.
+# Pipe-separated names, trimmed, empty and null-like entries removed.
 ALT_NAMES = (
     f"list_filter(list_transform(string_split({clean('alternative_names')}, '|'), s -> trim(s)), "
-    "s -> s NOT IN ('', 'None'))"
+    "s -> lower(s) NOT IN ('', 'none', 'nan'))"
+)
+
+# description_raw is plain text; description is the fallback (tags stripped, HTML entities kept).
+# Both get entities decoded and leftover Markdown headings / HTML tags removed.
+DESCRIPTION = (
+    "NULLIF(trim(regexp_replace(unescape(COALESCE("
+    f"{clean('description_raw')}, {clean('description')})), "
+    r"'#{2,}\s*|<[^>]+>', '', 'g')), '')"
 )
 
 # short_screenshots holds Python-style dicts ({'id': 123, 'image': '...'}), not JSON.
@@ -68,22 +79,23 @@ def pg_array(list_expr: str) -> str:
 
 def export_csvs(out: Path) -> None:
     con = duckdb.connect()
+    con.create_function("unescape", html.unescape, ["VARCHAR"], "VARCHAR")
     con.execute(
         f"""CREATE VIEW src AS
             SELECT * FROM read_parquet('{PARQUET}')
-            WHERE added > 0 AND name IS NOT NULL"""
+            WHERE added > 0 AND name IS NOT NULL AND trim(name) <> ''"""
     )
     con.execute(
         f"""COPY (
             SELECT
                 id,
                 slug,
                 name,
-                NULLIF({clean('name_original')}, name) AS name_original,
                 {pg_array(ALT_NAMES)} AS alternative_names,
-                concat_ws(' | ', name, NULLIF({clean('name_original')}, name),
-                          array_to_string({ALT_NAMES}, ' | ')) AS search_names,
-                TRY_CAST(released AS DATE) AS released,
+                concat_ws(' | ', name, array_to_string({ALT_NAMES}, ' | ')) AS search_names,
+                -- Dates far in the future are placeholders, not real release dates.
+                CASE WHEN TRY_CAST(released AS DATE) <= current_date + INTERVAL 5 YEAR
+                     THEN TRY_CAST(released AS DATE) END AS released,
                 {clean('background_image')} AS background_image,
                 {clean('background_image_additional')} AS background_image_additional,
                 {pg_array(SCREENSHOTS)} AS screenshots,
@@ -93,9 +105,8 @@ def export_csvs(out: Path) -> None:
                 TRY_CAST(metacritic AS INTEGER) AS metacritic,
                 TRY_CAST(playtime AS INTEGER) AS playtime,
                 {ESRB} AS esrb_rating,
-                {clean('description_raw')} AS description,
-                {clean('website')} AS website,
-                {clean('dominant_color')} AS dominant_color
+                {DESCRIPTION} AS description,
+                {clean('website')} AS website
             FROM src
         ) TO '{out}/games.csv' (FORMAT csv, HEADER)"""
     )
@@ -105,10 +116,14 @@ def export_csvs(out: Path) -> None:
                 SELECT id AS game_id, trim(unnest(string_split({col}, '|'))) AS name
                 FROM src WHERE {col} IS NOT NULL"""
         )
+        min_games = MIN_TAG_GAMES if table == "tags" else 1
         con.execute(
-            """CREATE OR REPLACE TEMP TABLE lk AS
+            f"""CREATE OR REPLACE TEMP TABLE lk AS
                SELECT row_number() OVER (ORDER BY name) AS id, name
-               FROM (SELECT DISTINCT name FROM pairs WHERE name NOT IN ('', 'None'))"""
+               FROM (SELECT name FROM pairs
+                     WHERE name NOT IN ('', 'None')
+                     GROUP BY name
+                     HAVING count(DISTINCT game_id) >= {min_games})"""
         )
         con.execute(f"COPY (SELECT id, name FROM lk) TO '{out}/{table}.csv' (FORMAT csv, HEADER)")
         con.execute(

db/requirements.txt

@@ -1,2 +1,3 @@
 duckdb
+numpy
 psycopg[binary]

db/schema.sql

@@ -8,7 +8,6 @@ CREATE TABLE games (
     id                          BIGINT PRIMARY KEY,
     slug                        TEXT,
     name                        TEXT NOT NULL,
-    name_original               TEXT,
     alternative_names           TEXT[],
     search_names                TEXT NOT NULL,
     released                    DATE,
@@ -22,8 +21,7 @@ CREATE TABLE games (
     playtime                    INTEGER,
     esrb_rating                 TEXT,
     description                 TEXT,
-    website                     TEXT,
-    dominant_color              TEXT
+    website                     TEXT
 );
 
 CREATE TABLE genres           (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
@@ -84,6 +82,7 @@ CREATE TABLE reviews (
     user_id    BIGINT   NOT NULL REFERENCES users(id) ON DELETE CASCADE,
     rating     SMALLINT NOT NULL CHECK (rating BETWEEN 1 AND 5),
     body       TEXT     NOT NULL CHECK (char_length(body) BETWEEN 1 AND 2000),
+    hours_played SMALLINT CHECK (hours_played BETWEEN 1 AND 10000),
     created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
     UNIQUE (game_id, user_id)
 );