diff --git a/backend/.gitignore b/backend/.gitignore index deed335..b5b1337 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -1,3 +1,4 @@ node_modules/ dist/ .env +data/last-import.txt \ No newline at end of file diff --git a/backend/src/scripts/import.ts b/backend/src/scripts/import.ts index 98696a4..646325e 100644 --- a/backend/src/scripts/import.ts +++ b/backend/src/scripts/import.ts @@ -1,5 +1,5 @@ import "dotenv/config"; -import { writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; const API_KEY = process.env.KASSALAPP_API_KEY; @@ -8,9 +8,18 @@ if (!API_KEY) { } const BASE_URL = "https://kassal.app/api/v1/"; -const OUTPUT_PATH = "data/kassalapp-dump.json"; +const OUTPUT_PATH = process.env.IMPORT_OUTPUT ?? "data/kassalapp-dump.json"; +const LAST_IMPORT_PATH = "data/last-import.txt"; const UNCATEGORIZED_CATEGORY_ID = -1; +const since: Date | null = await readFile(LAST_IMPORT_PATH, "utf-8") + .then((text) => { + const date = new Date(text.trim()); + return Number.isNaN(date.getTime()) ? null : date; + }) + .catch(() => null); + + type KassalappCategory = { id: number; parent_id?: number | null; @@ -92,13 +101,13 @@ type ImportedProduct = { }; async function getProductsPage(page: number): Promise { - const response = await get(`products?size=100&page=${page}`); + const response = await get(`products?size=100&page=${page}&sort=date_desc`); const productData = response as ProductsResponse; return productData.data ?? []; } -async function getProducts(): Promise { +async function getProducts(since: Date | null): Promise { const allProducts: KassalappProduct[] = []; const MAX_PAGES = 1000; @@ -118,6 +127,13 @@ async function getProducts(): Promise { console.log( `Side ${page}: ${products.length} produkter hentet. Totalt: ${allProducts.length}`, ); + + const oldest = products[products.length - 1]?.updated_at; + + if (since && oldest && new Date(oldest) < since) { + console.log("Nådde produkter fra før forrige import. Stopper."); + break; + } } if (allProducts.length > 0) { @@ -263,6 +279,15 @@ async function get(path: string): Promise { } } +const startedAt = new Date().toISOString(); + + +console.log( + since + ? `Henter endringer siden ${since.toISOString()}` + : "Fant ingen forrige import. Henter alt.", +); + console.log("Tester tilkobling til Kassalapp..."); const categories = await get("categories"); @@ -279,7 +304,7 @@ console.log( console.log("\nTester henting av flere produktsider..."); -const products = await getProducts(); +const products = await getProducts(since); console.log(`Totalt antall produkter hentet: ${products.length}`); @@ -315,7 +340,7 @@ const withoutCategory = normalizedProducts.filter( console.log(`Produkter uten kategori: ${withoutCategory}`); const dump = { - importedAt: new Date().toISOString(), + importedAt: startedAt, categories: normalizedCategories, products: normalizedProducts, }; diff --git a/backend/src/scripts/seed.ts b/backend/src/scripts/seed.ts index 1b1921a..0251fcf 100644 --- a/backend/src/scripts/seed.ts +++ b/backend/src/scripts/seed.ts @@ -1,4 +1,4 @@ -import { readFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { pool } from "../db.js"; type ImportedCategory = { @@ -38,7 +38,7 @@ type KassalappDump = { products: ImportedProduct[]; }; -const DUMP_PATH = "data/kassalapp-dump.json"; +const DUMP_PATH = process.env.IMPORT_OUTPUT ?? "data/kassalapp-dump.json"; const rawDump = await readFile(DUMP_PATH, "utf-8"); const dump = JSON.parse(rawDump) as KassalappDump; @@ -127,7 +127,7 @@ try { nutrition = EXCLUDED.nutrition, allergens = EXCLUDED.allergens, labels = EXCLUDED.labels, - category_id = EXCLUDED.category_id + category_id = COALESCE(NULLIF(EXCLUDED.category_id, -1), products.category_id) `, [ product.ean, @@ -200,6 +200,7 @@ try { console.log(`Seedet ${insertedPrices} prisoppføringer.`); await client.query("COMMIT"); + await writeFile("data/last-import.txt", dump.importedAt, "utf-8"); console.log("Seeding fullført uten feil."); } catch (error) {