Nattlig import henter bare endringer siden forrige kjøring (få dev oppdatert)

backend/.gitignore

@@ -1,3 +1,4 @@
 node_modules/
 dist/
 .env
+data/last-import.txt

backend/src/scripts/import.ts

@@ -1,5 +1,5 @@
 import "dotenv/config";
-import { writeFile } from "node:fs/promises";
+import { readFile, writeFile } from "node:fs/promises";
 
 const API_KEY = process.env.KASSALAPP_API_KEY;
 
@@ -8,9 +8,18 @@ if (!API_KEY) {
 }
 
 const BASE_URL = "https://kassal.app/api/v1/";
-const OUTPUT_PATH = "data/kassalapp-dump.json";
+const OUTPUT_PATH = process.env.IMPORT_OUTPUT ?? "data/kassalapp-dump.json";
+const LAST_IMPORT_PATH = "data/last-import.txt";
 const UNCATEGORIZED_CATEGORY_ID = -1;
 
+const since: Date | null = await readFile(LAST_IMPORT_PATH, "utf-8")
+  .then((text) => {
+    const date = new Date(text.trim());
+    return Number.isNaN(date.getTime()) ? null : date;
+  })
+  .catch(() => null);
+
+
 type KassalappCategory = {
   id: number;
   parent_id?: number | null;
@@ -92,13 +101,13 @@ type ImportedProduct = {
 };
 
 async function getProductsPage(page: number): Promise<KassalappProduct[]> {
-  const response = await get(`products?size=100&page=${page}`);
+  const response = await get(`products?size=100&page=${page}&sort=date_desc`);
   const productData = response as ProductsResponse;
 
   return productData.data ?? [];
 }
 
-async function getProducts(): Promise<KassalappProduct[]> {
+async function getProducts(since: Date | null): Promise<KassalappProduct[]> {
   const allProducts: KassalappProduct[] = [];
 
   const MAX_PAGES = 1000;
@@ -118,6 +127,13 @@ async function getProducts(): Promise<KassalappProduct[]> {
     console.log(
       `Side ${page}: ${products.length} produkter hentet. Totalt: ${allProducts.length}`,
     );
+
+    const oldest = products[products.length - 1]?.updated_at;
+
+    if (since && oldest && new Date(oldest) < since) {
+      console.log("Nådde produkter fra før forrige import. Stopper.");
+     break;
+    }
   }
 
   if (allProducts.length > 0) {
@@ -263,6 +279,15 @@ async function get(path: string): Promise<unknown> {
   }
 }
 
+const startedAt = new Date().toISOString();
+
+
+console.log(
+  since
+    ? `Henter endringer siden ${since.toISOString()}`
+    : "Fant ingen forrige import. Henter alt.",
+);
+
 console.log("Tester tilkobling til Kassalapp...");
 
 const categories = await get("categories");
@@ -279,7 +304,7 @@ console.log(
 
 console.log("\nTester henting av flere produktsider...");
 
-const products = await getProducts();
+const products = await getProducts(since);
 
 console.log(`Totalt antall produkter hentet: ${products.length}`);
 
@@ -315,7 +340,7 @@ const withoutCategory = normalizedProducts.filter(
 console.log(`Produkter uten kategori: ${withoutCategory}`);
 
 const dump = {
-  importedAt: new Date().toISOString(),
+  importedAt: startedAt, 
   categories: normalizedCategories,
   products: normalizedProducts,
 };

backend/src/scripts/seed.ts

@@ -1,4 +1,4 @@
-import { readFile } from "node:fs/promises";
+import { readFile, writeFile } from "node:fs/promises";
 import { pool } from "../db.js";
 
 type ImportedCategory = {
@@ -38,7 +38,7 @@ type KassalappDump = {
   products: ImportedProduct[];
 };
 
-const DUMP_PATH = "data/kassalapp-dump.json";
+const DUMP_PATH = process.env.IMPORT_OUTPUT ?? "data/kassalapp-dump.json";
 
 const rawDump = await readFile(DUMP_PATH, "utf-8");
 const dump = JSON.parse(rawDump) as KassalappDump;
@@ -127,7 +127,7 @@ try {
           nutrition = EXCLUDED.nutrition,
           allergens = EXCLUDED.allergens,
           labels = EXCLUDED.labels,
-          category_id = EXCLUDED.category_id
+          category_id = COALESCE(NULLIF(EXCLUDED.category_id, -1), products.category_id)
       `,
         [
             product.ean,
@@ -200,6 +200,7 @@ try {
   console.log(`Seedet ${insertedPrices} prisoppføringer.`);
 
   await client.query("COMMIT");
+  await writeFile("data/last-import.txt", dump.importedAt, "utf-8");
 
   console.log("Seeding fullført uten feil.");
 } catch (error) {