flere felter i tabellene

db/grants.sql

@@ -1,3 +1,6 @@
+-- Rettigheter for backend rollen webapp. Kjør etter hver import!
+-- Rollen skal opprettes èn gang i postgres før scriptet kjøres:
+--   CREATE ROLE webapp WITH LOGIN PASSWORD '<password>';
 GRANT CONNECT ON DATABASE gamesdb TO webapp;
 GRANT USAGE ON SCHEMA public TO webapp;
 

db/indexes.sql

@@ -1,19 +1,17 @@
+CREATE INDEX games_search_trgm ON games USING gin (search_names gin_trgm_ops);
 
-CREATE INDEX games_name_trgm   ON games USING gin (name gin_trgm_ops);
-
 CREATE INDEX games_added_id    ON games (added DESC, id DESC);
 CREATE INDEX games_rating_id   ON games (rating DESC NULLS LAST, id DESC);
 CREATE INDEX games_released_id ON games (released DESC NULLS LAST, id DESC);
 CREATE INDEX games_meta_id     ON games (metacritic DESC NULLS LAST, id DESC);
 CREATE INDEX games_name_id     ON games (name, id);
-
-CREATE INDEX game_genres_rev     ON game_genres     (genre_id, game_id);
-CREATE INDEX game_platforms_rev  ON game_platforms  (platform_id, game_id);
-CREATE INDEX game_stores_rev     ON game_stores     (store_id, game_id);
-CREATE INDEX game_developers_rev ON game_developers (developer_id, game_id);
-CREATE INDEX game_publishers_rev ON game_publishers (publisher_id, game_id);
-CREATE INDEX game_tags_rev       ON game_tags       (tag_id, game_id);
-
-CREATE INDEX reviews_game_created ON reviews (game_id, created_at DESC);
-
 
+CREATE INDEX game_genres_rev           ON game_genres           (genre_id, game_id);
+CREATE INDEX game_platforms_rev        ON game_platforms        (platform_id, game_id);
+CREATE INDEX game_parent_platforms_rev ON game_parent_platforms (parent_platform_id, game_id);
+CREATE INDEX game_stores_rev           ON game_stores           (store_id, game_id);
+CREATE INDEX game_developers_rev       ON game_developers       (developer_id, game_id);
+CREATE INDEX game_publishers_rev       ON game_publishers       (publisher_id, game_id);
+CREATE INDEX game_tags_rev             ON game_tags             (tag_id, game_id);
+
+CREATE INDEX reviews_game_created ON reviews (game_id, created_at DESC);

db/load.py

@@ -15,66 +15,95 @@
 RELATIONS = [
     ("genres", "genres", "game_genres", "genre_id"),
     ("platforms", "platforms", "game_platforms", "platform_id"),
+    ("parent_platforms", "parent_platforms", "game_parent_platforms", "parent_platform_id"),
     ("stores", "stores", "game_stores", "store_id"),
     ("developers", "developers", "game_developers", "developer_id"),
     ("publishers", "publishers", "game_publishers", "publisher_id"),
     ("tags", "tags", "game_tags", "tag_id"),
 ]
 
-REQUIRED_COLUMNS = [
-    "id", "slug", "name", "released", "background_image", "rating", "ratings_count",
-    "added", "metacritic", "playtime", "esrb_rating", "description_raw", "website",
-    "dominant_color", "genres", "platforms", "stores", "developers", "publishers", "tags",
-]
-
 GAME_COLUMNS = (
-    "id, slug, name, released, background_image, rating, ratings_count, added, "
-    "metacritic, playtime, esrb_rating, description, website, dominant_color"
+    "id, slug, name, name_original, alternative_names, search_names, released, "
+    "background_image, background_image_additional, screenshots, rating, ratings_count, "
+    "added, metacritic, playtime, esrb_rating, description, website, dominant_color"
 )
 
+def clean(col: str) -> str:
+    """Empty strings and the text 'None' (a Python null written as a string) become NULL."""
+    return f"NULLIF(NULLIF(trim({col}), ''), 'None')"
 
-def export_csvs(out: Path) -> None:
-    con = duckdb.connect()
 
-    found = {r[0] for r in con.execute(f"DESCRIBE SELECT * FROM read_parquet('{PARQUET}')").fetchall()}
-    missing = [c for c in REQUIRED_COLUMNS if c not in found]
-    if missing:
-        sys.exit(f"Parquet file is missing expected columns: {missing}\nAvailable: {sorted(found)}")
+# Pipe-separated names, trimmed, empty entries removed.
+ALT_NAMES = (
+    f"list_filter(list_transform(string_split({clean('alternative_names')}, '|'), s -> trim(s)), "
+    "s -> s NOT IN ('', 'None'))"
+)
+
+# short_screenshots holds Python-style dicts ({'id': 123, 'image': '...'}), not JSON.
+# \d+ skips the id -1 entry, which duplicates background_image.
+SCREENSHOTS = (
+    "regexp_extract_all(" + clean("short_screenshots")
+    + r""", '''id'': (\d+), ''image'': ''([^'']+)''', 2)"""
+)
+
+# esrb_rating may be JSON, a Python-style dict, a plain name or 'None'.
+ESRB = (
+    f"CASE WHEN {clean('esrb_rating')} IS NULL THEN NULL "
+    f"WHEN trim(esrb_rating) LIKE '{{%' THEN "
+    + r"""NULLIF(regexp_extract(esrb_rating, '[''"]name[''"]\s*:\s*[''"]([^''"]+)', 1), '') """
+    + f"ELSE {clean('esrb_rating')} END"
+)
+
+
+def pg_array(list_expr: str) -> str:
+    """DuckDB list -> Postgres array literal ({"a","b"}), NULL when empty."""
+    quoted = (
+        "list_transform(" + list_expr + r""", x -> '"' || replace(replace(x, '\', '\\'), '"', '\"') || '"')"""
+    )
+    return (
+        f"CASE WHEN len({list_expr}) > 0 "
+        f"THEN '{{' || array_to_string({quoted}, ',') || '}}' END"
+    )
+
 
-    # The dataset stores missing values as the text 'None'; convert to real NULLs.
+def export_csvs(out: Path) -> None:
+    con = duckdb.connect()
+    con.execute(
+        f"""CREATE VIEW src AS
+            SELECT * FROM read_parquet('{PARQUET}')
+            WHERE added > 0 AND name IS NOT NULL"""
+    )
     con.execute(
-        f"""CREATE TEMP TABLE src AS
+        f"""COPY (
             SELECT
                 id,
-                NULLIF(slug, 'None') AS slug,
+                slug,
                 name,
+                NULLIF({clean('name_original')}, name) AS name_original,
+                {pg_array(ALT_NAMES)} AS alternative_names,
+                concat_ws(' | ', name, NULLIF({clean('name_original')}, name),
+                          array_to_string({ALT_NAMES}, ' | ')) AS search_names,
                 TRY_CAST(released AS DATE) AS released,
-                NULLIF(background_image, 'None') AS background_image,
+                {clean('background_image')} AS background_image,
+                {clean('background_image_additional')} AS background_image_additional,
+                {pg_array(SCREENSHOTS)} AS screenshots,
                 NULLIF(rating, 0) AS rating,
                 COALESCE(TRY_CAST(ratings_count AS INTEGER), 0) AS ratings_count,
                 TRY_CAST(added AS INTEGER) AS added,
                 TRY_CAST(metacritic AS INTEGER) AS metacritic,
                 TRY_CAST(playtime AS INTEGER) AS playtime,
-                NULLIF(regexp_extract(esrb_rating, '"name":[ ]*"([^"]*)"', 1), '') AS esrb_rating,
-                NULLIF(description_raw, 'None') AS description,
-                NULLIF(website, 'None') AS website,
-                NULLIF(dominant_color, 'None') AS dominant_color,
-                genres, platforms, stores, developers, publishers, tags
-            FROM read_parquet('{PARQUET}')
-            WHERE added > 0 AND name IS NOT NULL"""
-    )
-
-    con.execute(
-        f"COPY (SELECT {GAME_COLUMNS} FROM src) TO '{out}/games.csv' (FORMAT csv, HEADER)"
+                {ESRB} AS esrb_rating,
+                {clean('description_raw')} AS description,
+                {clean('website')} AS website,
+                {clean('dominant_color')} AS dominant_color
+            FROM src
+        ) TO '{out}/games.csv' (FORMAT csv, HEADER)"""
     )
-
     for col, table, junction, fk in RELATIONS:
         con.execute(
             f"""CREATE OR REPLACE TEMP TABLE pairs AS
-                SELECT game_id, trim(item) AS name FROM (
-                    SELECT id AS game_id, unnest(string_split({col}, '|')) AS item
-                    FROM src WHERE {col} IS NOT NULL
-                ) t"""
+                SELECT id AS game_id, trim(unnest(string_split({col}, '|'))) AS name
+                FROM src WHERE {col} IS NOT NULL"""
         )
         con.execute(
             """CREATE OR REPLACE TEMP TABLE lk AS

db/schema.sql

@@ -1,31 +1,37 @@
 DROP TABLE IF EXISTS reviews CASCADE;
-DROP TABLE IF EXISTS game_genres, game_platforms, game_stores, game_developers, game_publishers, game_tags CASCADE;
-DROP TABLE IF EXISTS genres, platforms, stores, developers, publishers, tags CASCADE;
+DROP TABLE IF EXISTS game_genres, game_platforms, game_parent_platforms, game_stores, game_developers, game_publishers, game_tags CASCADE;
+DROP TABLE IF EXISTS genres, platforms, parent_platforms, stores, developers, publishers, tags CASCADE;
 DROP TABLE IF EXISTS games CASCADE;
 
 CREATE TABLE games (
-    id               BIGINT PRIMARY KEY,
-    slug             TEXT,
-    name             TEXT NOT NULL,
-    released         DATE,
-    background_image TEXT,
-    rating           REAL,
-    ratings_count    INTEGER NOT NULL DEFAULT 0,
-    added            INTEGER NOT NULL DEFAULT 0,
-    metacritic       INTEGER,
-    playtime         INTEGER,
-    esrb_rating      TEXT,
-    description      TEXT,
-    website          TEXT,
-    dominant_color   TEXT
+    id                          BIGINT PRIMARY KEY,
+    slug                        TEXT,
+    name                        TEXT NOT NULL,
+    name_original               TEXT,
+    alternative_names           TEXT[],
+    search_names                TEXT NOT NULL,
+    released                    DATE,
+    background_image            TEXT,
+    background_image_additional TEXT,
+    screenshots                 TEXT[],
+    rating                      REAL,
+    ratings_count               INTEGER NOT NULL DEFAULT 0,
+    added                       INTEGER NOT NULL DEFAULT 0,
+    metacritic                  INTEGER,
+    playtime                    INTEGER,
+    esrb_rating                 TEXT,
+    description                 TEXT,
+    website                     TEXT,
+    dominant_color              TEXT
 );
 
-CREATE TABLE genres     (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
-CREATE TABLE platforms  (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
-CREATE TABLE stores     (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
-CREATE TABLE developers (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
-CREATE TABLE publishers (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
-CREATE TABLE tags       (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
+CREATE TABLE genres           (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
+CREATE TABLE platforms        (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
+CREATE TABLE parent_platforms (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
+CREATE TABLE stores           (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
+CREATE TABLE developers       (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
+CREATE TABLE publishers       (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
+CREATE TABLE tags             (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
 
 CREATE TABLE game_genres (
     game_id  BIGINT  NOT NULL REFERENCES games(id)  ON DELETE CASCADE,
@@ -37,6 +43,11 @@ CREATE TABLE game_platforms (
     platform_id INTEGER NOT NULL REFERENCES platforms(id) ON DELETE CASCADE,
     PRIMARY KEY (game_id, platform_id)
 );
+CREATE TABLE game_parent_platforms (
+    game_id            BIGINT  NOT NULL REFERENCES games(id)            ON DELETE CASCADE,
+    parent_platform_id INTEGER NOT NULL REFERENCES parent_platforms(id) ON DELETE CASCADE,
+    PRIMARY KEY (game_id, parent_platform_id)
+);
 CREATE TABLE game_stores (
     game_id  BIGINT  NOT NULL REFERENCES games(id)  ON DELETE CASCADE,
     store_id INTEGER NOT NULL REFERENCES stores(id) ON DELETE CASCADE,