Scale dataset for 1000 artworks

backend/src/importMetArtworks.ts

@@ -1,6 +1,9 @@
 import { MongoClient } from "mongodb";
 
 const mongoUri = process.env.MONGODB_URI;
+const IMPORT_LIMIT = 1000;
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
 
 if (!mongoUri) {
   throw new Error("MONGODB_URI mangler");
@@ -24,55 +27,69 @@ async function importArtworks() {
     const database = client.db("artex");
     const collection = database.collection("met_artworks");
 
-    const searchResponse = await fetch(
-      "https://collectionapi.metmuseum.org/public/collection/v1.1/search?hasImages=true&q=painting&limit=100",
-    );
+    let imported = 0;
+    let offset = 0;
+    const PAGE_SIZE = 100;
 
-    if (!searchResponse.ok) {
-      throw new Error(`Search failed: ${searchResponse.status}`);
-    }
+    while (imported < IMPORT_LIMIT) {
+      const searchResponse = await fetch(
+        `https://collectionapi.metmuseum.org/public/collection/v1.1/search?hasImages=true&q=painting&offset=${offset}&limit=${PAGE_SIZE}`,
+      );
 
-    const searchData = (await searchResponse.json()) as {
-      objectIDs: number[] | null;
-    };
+      if (!searchResponse.ok) {
+        throw new Error(`Search failed: ${searchResponse.status}`);
+      }
 
-    if (!searchData.objectIDs) {
-      throw new Error("Ingen kunstverk funnet");
-    }
+      const searchData = (await searchResponse.json()) as {
+        objectIDs: number[] | null;
+      };
 
-    let imported = 0;
+      const objectIDs = searchData.objectIDs ?? [];
+
+      if (objectIDs.length === 0) {
+        console.log("No more artworks found.");
+        break;
+      }
 
-    for (const id of searchData.objectIDs) {
-      if (imported >= 20) break;
+      console.log(`Checking ${objectIDs.length} IDs from offset ${offset}`);
 
-      const response = await fetch(
-        `https://collectionapi.metmuseum.org/public/collection/v1/objects/${id}`,
-      );
+      for (const id of objectIDs) {
+        if (imported >= IMPORT_LIMIT) break;
 
-      if (!response.ok) continue;
+        await sleep(500);
 
-      const artwork = (await response.json()) as MetArtwork;
+        const response = await fetch(
+          `https://collectionapi.metmuseum.org/public/collection/v1/objects/${id}`,
+        );
 
-      if (!artwork.isPublicDomain || !artwork.primaryImageSmall) {
-        continue;
-      }
+        if (!response.ok) continue;
+
+        const artwork = (await response.json()) as MetArtwork;
 
-      await collection.updateOne(
-        { id: artwork.objectID },
-        {
-          $set: {
-            id: artwork.objectID,
-            title: artwork.title,
-            artist_display: artwork.artistDisplayName,
-            date_display: artwork.objectDate,
-            image_url: artwork.primaryImageSmall,
+        if (!artwork.isPublicDomain || !artwork.primaryImageSmall) {
+          continue;
+        }
+
+        await collection.updateOne(
+          { id: artwork.objectID },
+          {
+            $set: {
+              id: artwork.objectID,
+              title: artwork.title,
+              artist_display: artwork.artistDisplayName,
+              date_display: artwork.objectDate,
+              image_url: artwork.primaryImageSmall,
+            },
           },
-        },
-        { upsert: true },
-      );
+          { upsert: true },
+        );
+
+        imported++;
+        console.log(`Imported ${imported}/${IMPORT_LIMIT}: ${artwork.title}`);
+      }
 
-      imported++;
-      console.log(`Imported ${imported}/20: ${artwork.title}`);
+      offset += PAGE_SIZE;
+      await sleep(2000);
     }
 
     console.log(`Completed! Imported ${imported} artworks.`);