-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added class to scrape confirmed organizations from InnsamlingsKontrollen.
- Loading branch information
Showing
1 changed file
with
96 additions
and
0 deletions.
There are no files selected for viewing
96 changes: 96 additions & 0 deletions
96
...helpapplication/src/main/java/ntnu/sytemutvikling/team6/models/IKOrganizationScraper.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| package ntnu.sytemutvikling.team6.models; | ||
|
|
||
| import java.time.Duration; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import org.openqa.selenium.By; | ||
| import org.openqa.selenium.WebDriver; | ||
| import org.openqa.selenium.WebElement; | ||
| import org.openqa.selenium.chrome.ChromeDriver; | ||
| import org.openqa.selenium.chrome.ChromeOptions; | ||
| import org.openqa.selenium.support.ui.ExpectedConditions; | ||
| import org.openqa.selenium.support.ui.WebDriverWait; | ||
|
|
||
| public class IKOrganizationScraper { | ||
| private final List<Organization> organizationData; | ||
|
|
||
| public IKOrganizationScraper() { | ||
| this.organizationData = new ArrayList<>(); | ||
| } | ||
|
|
||
| public boolean updateData() { | ||
| // Configure headless chrome browser | ||
| ChromeOptions options = new ChromeOptions(); | ||
| options.addArguments("--headless=new"); | ||
| options.addArguments("--window-size=1920,1080"); | ||
| options.addArguments("--disable-gpu"); | ||
| options.addArguments("--no-sandbox"); | ||
| options.addArguments("--disable-dev-shm-usage"); | ||
|
|
||
| WebDriver driver = new ChromeDriver(options); | ||
|
|
||
| //URL for godkjente organisasjoner | ||
| driver.get("https://www.innsamlingskontrollen.no/organisasjoner/"); | ||
|
|
||
| // Wait to ensure tabular data is loaded first | ||
| WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(10)); | ||
| wait.until(ExpectedConditions.presenceOfElementLocated(By.tagName("table"))); | ||
|
|
||
| // Loops through table rows and columns | ||
| List<WebElement> rows = driver.findElements(By.cssSelector("table tbody tr")); | ||
|
|
||
| String name = null; | ||
| String telephone = null; | ||
| String location = null; | ||
| String status = null; | ||
|
|
||
| // Clear old data before updating | ||
| this.organizationData.clear(); | ||
|
|
||
| for (WebElement row : rows) { | ||
| List<WebElement> columns = row.findElements(By.tagName("td")); | ||
|
|
||
| for (int i = 0; i < columns.size(); i++) { | ||
|
|
||
| WebElement column = columns.get(i); | ||
|
|
||
| // Non-verification columns | ||
| if (i == 0) { | ||
| name = column.getText(); | ||
| } | ||
|
|
||
| if (i == 1) { | ||
| telephone = columns.get(i).getText(); | ||
| } | ||
|
|
||
| if (i == 2) { | ||
| location = columns.get(i).getText(); | ||
| } | ||
|
|
||
| // Verification column | ||
| if (i == 3) { | ||
| if (!column.findElements(By.cssSelector(".status-pre-approved")).isEmpty()) { | ||
| status = "Monitored"; | ||
| } else if (!column.findElements(By.cssSelector(".status-approved")).isEmpty()) { | ||
| status = "Approved"; | ||
| } else { | ||
| status = "Unknown"; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| var organization = new Organization(name, telephone, location, status); | ||
| this.organizationData.add(organization); | ||
| } | ||
| driver.quit(); | ||
|
|
||
| // Removes the first empty row that lists only the categories, aka no relevant data | ||
| this.organizationData.removeFirst(); | ||
|
|
||
| return true; | ||
| } | ||
|
|
||
| public List<Organization> getData() { | ||
| return this.organizationData; | ||
| } | ||
| } |