diff --git a/backend/src/resolvers.ts b/backend/src/resolvers.ts index 757dc29..fabb301 100644 --- a/backend/src/resolvers.ts +++ b/backend/src/resolvers.ts @@ -34,7 +34,6 @@ const GAME_COLS = ` ratings_count AS "ratingsCount", playtime, esrb_rating AS "esrbRating", - dominant_color AS "dominantColor", background_image AS "backgroundImage", background_image_additional AS "backgroundImageAdditional", COALESCE(screenshots, '{}') AS "screenshots" diff --git a/backend/src/schema.ts b/backend/src/schema.ts index 9d2621e..a141ac2 100644 --- a/backend/src/schema.ts +++ b/backend/src/schema.ts @@ -33,7 +33,6 @@ export const typeDefs = `#graphql ratingsCount: Int!, playtime: Int, esrbRating: String, - dominantColor: String, backgroundImage: String, backgroundImageAdditional: String, screenshots: [String!]! diff --git a/db/load.py b/db/load.py index c1f23b5..524c8e4 100644 --- a/db/load.py +++ b/db/load.py @@ -11,6 +11,7 @@ PARQUET = os.environ.get("PARQUET_PATH", str(Path.home() / "data" / "videogames_data.parquet")) DATABASE_URL = os.environ["DATABASE_URL"] FORCE = "--force" in sys.argv +WIPE_USERS = "--wipe-user-data" in sys.argv # Tags used by fewer games than this are dropped (long tail of one-off, noisy tags). MIN_TAG_GAMES = int(os.environ.get("MIN_TAG_GAMES", "50")) @@ -140,18 +141,30 @@ def copy_csv(cur: psycopg.Cursor, table: str, columns: str, path: Path) -> None: while chunk := f.read(1 << 20): cp.write(chunk) +def has_user_data(cur) -> bool: + for table in ("users", "reviews"): + cur.execute("SELECT to_regclass(%s)", (f"public.{table}",)) + if cur.fetchone()[0] is None: + continue + cur.execute(f"SELECT EXISTS (SELECT 1 FROM {table})") + if cur.fetchone()[0]: + return True + return False def main() -> None: - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) - print("Exporting CSVs from Parquet...") - export_csvs(out) + with psycopg.connect(DATABASE_URL) as conn: + cur = conn.cursor() + cur.execute("SELECT to_regclass('public.games')") + if cur.fetchone()[0] is not None and not FORCE: + sys.exit("games table already exists. Rerun with --force to force reload.") - with psycopg.connect(DATABASE_URL) as conn: - cur = conn.cursor() - cur.execute("SELECT to_regclass('public.games')") - if cur.fetchone()[0] is not None and not FORCE: - sys.exit("games table already exists. Rerun with --force to wipe and reload (deletes reviews too).") + if has_user_data(cur) and not WIPE_USERS: + sys.exit("users/reviews contain data. Add --wipe-user-data to delete it.") + + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + print("Exporting CSVs from Parquet...") + export_csvs(out) print("Creating tables...") cur.execute((HERE / "schema.sql").read_text()) @@ -162,17 +175,15 @@ def main() -> None: copy_csv(cur, table, "id, name", out / f"{table}.csv") copy_csv(cur, junction, f"game_id, {fk}", out / f"{junction}.csv") - print("Creating indexes...") - cur.execute((HERE / "indexes.sql").read_text()) - conn.commit() - - cur.execute("ANALYZE") - conn.commit() - - for table in ["games"] + [r[1] for r in RELATIONS] + [r[2] for r in RELATIONS]: - cur.execute(f"SELECT count(*) FROM {table}") - print(f"{table}: {cur.fetchone()[0]}") + print("Creating indexes...") + cur.execute((HERE / "indexes.sql").read_text()) + conn.commit() + cur.execute("ANALYZE") + conn.commit() + for table in ["games"] + [r[1] for r in RELATIONS] + [r[2] for r in RELATIONS]: + cur.execute(f"SELECT count(*) FROM {table}") + print(f"{table}: {cur.fetchone()[0]}") if __name__ == "__main__": main()