Updated URLCharityScraper

Time for scraping went down from approx 2 hours to around 20-30 mins, and is still reliable. Thread.sleep() seems to not be required after all.

helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/scraperComponents/URLCharityScraper.java

@@ -70,7 +70,7 @@ public URLCharityScraper(String url, WebDriver driver) {
    * @return the {@code WebDriverWait} object to be used in the methods
    */
   protected WebDriverWait createWait() {
-    return new WebDriverWait(driver, Duration.ofSeconds(30));
+    return new WebDriverWait(driver, Duration.ofSeconds(10));
   }
 
   /**
@@ -118,7 +118,7 @@ protected void updateDescription() {
       wait.until(
           ExpectedConditions.numberOfElementsToBeMoreThan(By.cssSelector(".information"), 0));
 
-      Thread.sleep(5000);
+      // Thread.sleep(5000);
       List<WebElement> firstDescription = findElements(By.cssSelector(".information"));
 
       for (WebElement element : firstDescription) {
@@ -139,7 +139,7 @@ void updateLogo() {
     try {
       WebDriverWait wait = createWait();
       wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector(".logo > img")));
-      Thread.sleep(5000);
+      // Thread.sleep(5000);
 
       WebElement logo = findElement(By.cssSelector(".logo > img"));
       this.logoURL = logo.getAttribute("src");
@@ -155,7 +155,7 @@ void updateCategories() {
       WebDriverWait wait = createWait();
 
       wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector(".tag-label")));
-      Thread.sleep(5000);
+      // Thread.sleep(5000);
 
       List<WebElement> elements = findElements(By.cssSelector(".tag-label"));
 
@@ -183,7 +183,7 @@ void updateKeyValues() {
           ExpectedConditions.visibilityOfElementLocated(
               By.xpath(
                   "//li[.//h2[normalize-space()='Innsamlingsprosent']]//div[@class='graph']")));
-      Thread.sleep(5000);
+      // Thread.sleep(5000);
       element =
           findElement(
               By.xpath("//li[.//h2[normalize-space()='Innsamlingsprosent']]//div[@class='graph']"));
@@ -224,9 +224,9 @@ public void scrapeCharityPage() {
       updateLogo();
       updateCategories();
       updateKeyValues();
-      Thread.sleep(1000);
+      // Thread.sleep(1000);
 
-    } catch (InterruptedException e) {
+    } catch (Exception e) {
       throw new RuntimeException(e);
     } finally {
       closeDriver();