From 48a11b96dcf8168dc1f2cba4ba32e07af430c703 Mon Sep 17 00:00:00 2001 From: Albert Paulsen Lindberg Date: Fri, 9 Oct 2026 14:47:22 +0200 Subject: [PATCH 1/2] flere felter i tabellene --- db/grants.sql | 10 ++++++ db/indexes.sql | 22 ++++++------ db/load.py | 97 ++++++++++++++++++++++++++++++++------------------ db/schema.sql | 55 ++++++++++++++++------------ 4 files changed, 116 insertions(+), 68 deletions(-) create mode 100644 db/grants.sql diff --git a/db/grants.sql b/db/grants.sql new file mode 100644 index 00000000..c74af3db --- /dev/null +++ b/db/grants.sql @@ -0,0 +1,10 @@ +-- Privileges for the backend role. Run after every import, since load.py recreates the tables. +-- The role itself is created once as postgres: +-- CREATE ROLE webapp WITH LOGIN PASSWORD ''; + +GRANT CONNECT ON DATABASE gamesdb TO webapp; +GRANT USAGE ON SCHEMA public TO webapp; + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO webapp; +GRANT INSERT ON reviews TO webapp; +GRANT USAGE ON SEQUENCE reviews_id_seq TO webapp; diff --git a/db/indexes.sql b/db/indexes.sql index 52263486..929a7042 100644 --- a/db/indexes.sql +++ b/db/indexes.sql @@ -1,19 +1,17 @@ +CREATE INDEX games_search_trgm ON games USING gin (search_names gin_trgm_ops); -CREATE INDEX games_name_trgm ON games USING gin (name gin_trgm_ops); - CREATE INDEX games_added_id ON games (added DESC, id DESC); CREATE INDEX games_rating_id ON games (rating DESC NULLS LAST, id DESC); CREATE INDEX games_released_id ON games (released DESC NULLS LAST, id DESC); CREATE INDEX games_meta_id ON games (metacritic DESC NULLS LAST, id DESC); CREATE INDEX games_name_id ON games (name, id); - -CREATE INDEX game_genres_rev ON game_genres (genre_id, game_id); -CREATE INDEX game_platforms_rev ON game_platforms (platform_id, game_id); -CREATE INDEX game_stores_rev ON game_stores (store_id, game_id); -CREATE INDEX game_developers_rev ON game_developers (developer_id, game_id); -CREATE INDEX game_publishers_rev ON game_publishers (publisher_id, game_id); -CREATE INDEX game_tags_rev ON game_tags (tag_id, game_id); - -CREATE INDEX reviews_game_created ON reviews (game_id, created_at DESC); - +CREATE INDEX game_genres_rev ON game_genres (genre_id, game_id); +CREATE INDEX game_platforms_rev ON game_platforms (platform_id, game_id); +CREATE INDEX game_parent_platforms_rev ON game_parent_platforms (parent_platform_id, game_id); +CREATE INDEX game_stores_rev ON game_stores (store_id, game_id); +CREATE INDEX game_developers_rev ON game_developers (developer_id, game_id); +CREATE INDEX game_publishers_rev ON game_publishers (publisher_id, game_id); +CREATE INDEX game_tags_rev ON game_tags (tag_id, game_id); + +CREATE INDEX reviews_game_created ON reviews (game_id, created_at DESC); diff --git a/db/load.py b/db/load.py index 4ee04537..3e6f8253 100644 --- a/db/load.py +++ b/db/load.py @@ -15,66 +15,95 @@ RELATIONS = [ ("genres", "genres", "game_genres", "genre_id"), ("platforms", "platforms", "game_platforms", "platform_id"), + ("parent_platforms", "parent_platforms", "game_parent_platforms", "parent_platform_id"), ("stores", "stores", "game_stores", "store_id"), ("developers", "developers", "game_developers", "developer_id"), ("publishers", "publishers", "game_publishers", "publisher_id"), ("tags", "tags", "game_tags", "tag_id"), ] -REQUIRED_COLUMNS = [ - "id", "slug", "name", "released", "background_image", "rating", "ratings_count", - "added", "metacritic", "playtime", "esrb_rating", "description_raw", "website", - "dominant_color", "genres", "platforms", "stores", "developers", "publishers", "tags", -] - GAME_COLUMNS = ( - "id, slug, name, released, background_image, rating, ratings_count, added, " - "metacritic, playtime, esrb_rating, description, website, dominant_color" + "id, slug, name, name_original, alternative_names, search_names, released, " + "background_image, background_image_additional, screenshots, rating, ratings_count, " + "added, metacritic, playtime, esrb_rating, description, website, dominant_color" ) +def clean(col: str) -> str: + """Empty strings and the text 'None' (a Python null written as a string) become NULL.""" + return f"NULLIF(NULLIF(trim({col}), ''), 'None')" -def export_csvs(out: Path) -> None: - con = duckdb.connect() - found = {r[0] for r in con.execute(f"DESCRIBE SELECT * FROM read_parquet('{PARQUET}')").fetchall()} - missing = [c for c in REQUIRED_COLUMNS if c not in found] - if missing: - sys.exit(f"Parquet file is missing expected columns: {missing}\nAvailable: {sorted(found)}") +# Pipe-separated names, trimmed, empty entries removed. +ALT_NAMES = ( + f"list_filter(list_transform(string_split({clean('alternative_names')}, '|'), s -> trim(s)), " + "s -> s NOT IN ('', 'None'))" +) + +# short_screenshots holds Python-style dicts ({'id': 123, 'image': '...'}), not JSON. +# \d+ skips the id -1 entry, which duplicates background_image. +SCREENSHOTS = ( + "regexp_extract_all(" + clean("short_screenshots") + + r""", '''id'': (\d+), ''image'': ''([^'']+)''', 2)""" +) + +# esrb_rating may be JSON, a Python-style dict, a plain name or 'None'. +ESRB = ( + f"CASE WHEN {clean('esrb_rating')} IS NULL THEN NULL " + f"WHEN trim(esrb_rating) LIKE '{{%' THEN " + + r"""NULLIF(regexp_extract(esrb_rating, '[''"]name[''"]\s*:\s*[''"]([^''"]+)', 1), '') """ + + f"ELSE {clean('esrb_rating')} END" +) + + +def pg_array(list_expr: str) -> str: + """DuckDB list -> Postgres array literal ({"a","b"}), NULL when empty.""" + quoted = ( + "list_transform(" + list_expr + r""", x -> '"' || replace(replace(x, '\', '\\'), '"', '\"') || '"')""" + ) + return ( + f"CASE WHEN len({list_expr}) > 0 " + f"THEN '{{' || array_to_string({quoted}, ',') || '}}' END" + ) + - # The dataset stores missing values as the text 'None'; convert to real NULLs. +def export_csvs(out: Path) -> None: + con = duckdb.connect() + con.execute( + f"""CREATE VIEW src AS + SELECT * FROM read_parquet('{PARQUET}') + WHERE added > 0 AND name IS NOT NULL""" + ) con.execute( - f"""CREATE TEMP TABLE src AS + f"""COPY ( SELECT id, - NULLIF(slug, 'None') AS slug, + slug, name, + NULLIF({clean('name_original')}, name) AS name_original, + {pg_array(ALT_NAMES)} AS alternative_names, + concat_ws(' | ', name, NULLIF({clean('name_original')}, name), + array_to_string({ALT_NAMES}, ' | ')) AS search_names, TRY_CAST(released AS DATE) AS released, - NULLIF(background_image, 'None') AS background_image, + {clean('background_image')} AS background_image, + {clean('background_image_additional')} AS background_image_additional, + {pg_array(SCREENSHOTS)} AS screenshots, NULLIF(rating, 0) AS rating, COALESCE(TRY_CAST(ratings_count AS INTEGER), 0) AS ratings_count, TRY_CAST(added AS INTEGER) AS added, TRY_CAST(metacritic AS INTEGER) AS metacritic, TRY_CAST(playtime AS INTEGER) AS playtime, - NULLIF(regexp_extract(esrb_rating, '"name":[ ]*"([^"]*)"', 1), '') AS esrb_rating, - NULLIF(description_raw, 'None') AS description, - NULLIF(website, 'None') AS website, - NULLIF(dominant_color, 'None') AS dominant_color, - genres, platforms, stores, developers, publishers, tags - FROM read_parquet('{PARQUET}') - WHERE added > 0 AND name IS NOT NULL""" - ) - - con.execute( - f"COPY (SELECT {GAME_COLUMNS} FROM src) TO '{out}/games.csv' (FORMAT csv, HEADER)" + {ESRB} AS esrb_rating, + {clean('description_raw')} AS description, + {clean('website')} AS website, + {clean('dominant_color')} AS dominant_color + FROM src + ) TO '{out}/games.csv' (FORMAT csv, HEADER)""" ) - for col, table, junction, fk in RELATIONS: con.execute( f"""CREATE OR REPLACE TEMP TABLE pairs AS - SELECT game_id, trim(item) AS name FROM ( - SELECT id AS game_id, unnest(string_split({col}, '|')) AS item - FROM src WHERE {col} IS NOT NULL - ) t""" + SELECT id AS game_id, trim(unnest(string_split({col}, '|'))) AS name + FROM src WHERE {col} IS NOT NULL""" ) con.execute( """CREATE OR REPLACE TEMP TABLE lk AS diff --git a/db/schema.sql b/db/schema.sql index 43affb0f..43d074c4 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -1,31 +1,37 @@ DROP TABLE IF EXISTS reviews CASCADE; -DROP TABLE IF EXISTS game_genres, game_platforms, game_stores, game_developers, game_publishers, game_tags CASCADE; -DROP TABLE IF EXISTS genres, platforms, stores, developers, publishers, tags CASCADE; +DROP TABLE IF EXISTS game_genres, game_platforms, game_parent_platforms, game_stores, game_developers, game_publishers, game_tags CASCADE; +DROP TABLE IF EXISTS genres, platforms, parent_platforms, stores, developers, publishers, tags CASCADE; DROP TABLE IF EXISTS games CASCADE; CREATE TABLE games ( - id BIGINT PRIMARY KEY, - slug TEXT, - name TEXT NOT NULL, - released DATE, - background_image TEXT, - rating REAL, - ratings_count INTEGER NOT NULL DEFAULT 0, - added INTEGER NOT NULL DEFAULT 0, - metacritic INTEGER, - playtime INTEGER, - esrb_rating TEXT, - description TEXT, - website TEXT, - dominant_color TEXT + id BIGINT PRIMARY KEY, + slug TEXT, + name TEXT NOT NULL, + name_original TEXT, + alternative_names TEXT[], + search_names TEXT NOT NULL, + released DATE, + background_image TEXT, + background_image_additional TEXT, + screenshots TEXT[], + rating REAL, + ratings_count INTEGER NOT NULL DEFAULT 0, + added INTEGER NOT NULL DEFAULT 0, + metacritic INTEGER, + playtime INTEGER, + esrb_rating TEXT, + description TEXT, + website TEXT, + dominant_color TEXT ); -CREATE TABLE genres (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); -CREATE TABLE platforms (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); -CREATE TABLE stores (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); -CREATE TABLE developers (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); -CREATE TABLE publishers (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); -CREATE TABLE tags (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); +CREATE TABLE genres (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); +CREATE TABLE platforms (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); +CREATE TABLE parent_platforms (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); +CREATE TABLE stores (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); +CREATE TABLE developers (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); +CREATE TABLE publishers (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); +CREATE TABLE tags (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); CREATE TABLE game_genres ( game_id BIGINT NOT NULL REFERENCES games(id) ON DELETE CASCADE, @@ -37,6 +43,11 @@ CREATE TABLE game_platforms ( platform_id INTEGER NOT NULL REFERENCES platforms(id) ON DELETE CASCADE, PRIMARY KEY (game_id, platform_id) ); +CREATE TABLE game_parent_platforms ( + game_id BIGINT NOT NULL REFERENCES games(id) ON DELETE CASCADE, + parent_platform_id INTEGER NOT NULL REFERENCES parent_platforms(id) ON DELETE CASCADE, + PRIMARY KEY (game_id, parent_platform_id) +); CREATE TABLE game_stores ( game_id BIGINT NOT NULL REFERENCES games(id) ON DELETE CASCADE, store_id INTEGER NOT NULL REFERENCES stores(id) ON DELETE CASCADE, From 30fc3d9855867e58687421f7b11aab645e1b1083 Mon Sep 17 00:00:00 2001 From: Albert Paulsen Lindberg Date: Sun, 11 Oct 2026 16:20:50 +0200 Subject: [PATCH 2/2] update grants and schema import script --- db/grants.sql | 6 +++++- db/schema.sql | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/db/grants.sql b/db/grants.sql index c74af3db..8c2bddb4 100644 --- a/db/grants.sql +++ b/db/grants.sql @@ -1,4 +1,4 @@ --- Privileges for the backend role. Run after every import, since load.py recreates the tables. +-- Privileges for the backend role. Run after every import or migration, since load.py recreates the tables. -- The role itself is created once as postgres: -- CREATE ROLE webapp WITH LOGIN PASSWORD ''; @@ -6,5 +6,9 @@ GRANT CONNECT ON DATABASE gamesdb TO webapp; GRANT USAGE ON SCHEMA public TO webapp; GRANT SELECT ON ALL TABLES IN SCHEMA public TO webapp; + +GRANT INSERT ON users TO webapp; +GRANT USAGE ON SEQUENCE users_id_seq TO webapp; + GRANT INSERT ON reviews TO webapp; GRANT USAGE ON SEQUENCE reviews_id_seq TO webapp; diff --git a/db/schema.sql b/db/schema.sql index 43d074c4..cbf51ed0 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS reviews CASCADE; +DROP TABLE IF EXISTS users CASCADE; DROP TABLE IF EXISTS game_genres, game_platforms, game_parent_platforms, game_stores, game_developers, game_publishers, game_tags CASCADE; DROP TABLE IF EXISTS genres, platforms, parent_platforms, stores, developers, publishers, tags CASCADE; DROP TABLE IF EXISTS games CASCADE; @@ -69,11 +70,21 @@ CREATE TABLE game_tags ( PRIMARY KEY (game_id, tag_id) ); +CREATE TABLE users ( + id BIGSERIAL PRIMARY KEY, + username TEXT NOT NULL CHECK (username ~ '^[A-Za-z0-9_]{3,30}$'), + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX users_username_lower ON users (lower(username)); + CREATE TABLE reviews ( id BIGSERIAL PRIMARY KEY, game_id BIGINT NOT NULL REFERENCES games(id) ON DELETE CASCADE, - author TEXT NOT NULL CHECK (char_length(author) BETWEEN 1 AND 50), + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, rating SMALLINT NOT NULL CHECK (rating BETWEEN 1 AND 5), body TEXT NOT NULL CHECK (char_length(body) BETWEEN 1 AND 2000), - created_at TIMESTAMPTZ NOT NULL DEFAULT now() + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (game_id, user_id) ); +CREATE INDEX reviews_user_created ON reviews (user_id, created_at DESC);