Skip to content

Commit

Permalink
Feat: Created a file FullCharityScrape basically combines apiScarper …
Browse files Browse the repository at this point in the history
…and urlScaraper to produce one CharityRegistry
  • Loading branch information
AdrianBalunan committed Apr 13, 2026
1 parent 60deef4 commit 926660c
Show file tree
Hide file tree
Showing 7 changed files with 53 additions and 6 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package ntnu.systemutvikling.team6.scraper;

import ntnu.systemutvikling.team6.models.Charity;
import ntnu.systemutvikling.team6.models.registry.CharityRegistry;
import ntnu.systemutvikling.team6.scraper.scraperComponents.APICharityScraper;
import ntnu.systemutvikling.team6.scraper.scraperComponents.URLCharityScraper;

import java.io.IOException;
import java.net.URISyntaxException;
import java.net.http.HttpClient;

public class FullCharityScrape {
private APICharityScraper apiScraper;

public FullCharityScrape() throws URISyntaxException {
HttpClient https = HttpClient.newHttpClient();
this.apiScraper = new APICharityScraper(https);
}

public CharityRegistry getAPIAndURLCharityData() throws IOException, InterruptedException {
CharityRegistry charityRegistry = null;
if (apiScraper.checkConnection()) {
charityRegistry = apiScraper.parseJSON(apiScraper.getJSONData());
for (Charity charity : charityRegistry.getAllCharities()) {
System.out.println(charity.getName());
}
// Scrapes description, logo, categories, and key values from IK
int charityCounter = 0;
for (Charity charity : charityRegistry.getAllCharities()) {
charityCounter++;

System.out.println("Scraping charity vanity details: " + charityCounter + " of " + charityRegistry.getAllCharities().size());
URLCharityScraper urlScraper = new URLCharityScraper(charity.getURL());
urlScraper.scrapeCharityPage();

charity.setDescription(urlScraper.getDescription());
charity.setCategory(urlScraper.getCategories());
charity.setLogoURL(urlScraper.getLogoURL());
charity.setKeyValues(urlScraper.getKeyValues());
byte[] logoBlob = LogoDownloader.downloadImageAsBlob(charity.getLogoURL());
charity.setLogoBlob(logoBlob);
}
}
return charityRegistry;
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package ntnu.systemutvikling.team6.scraper;
package ntnu.systemutvikling.team6.scraper.scraperComponents;

import ntnu.systemutvikling.team6.database.DatabaseSetup;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package ntnu.systemutvikling.team6.scraper;
package ntnu.systemutvikling.team6.scraper.scraperComponents;

import com.google.gson.Gson;
import java.io.IOException;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package ntnu.systemutvikling.team6.scraper;
package ntnu.systemutvikling.team6.scraper.scraperComponents;

import java.time.Duration;
import java.util.ArrayList;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package ntnu.systemutvikling.team6.scraper;
package ntnu.systemutvikling.team6.scraper.scraperComponents;

import static org.junit.jupiter.api.Assertions.*;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package ntnu.systemutvikling.team6.scraper;
package ntnu.systemutvikling.team6.scraper.scraperComponents;

import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
package ntnu.systemutvikling.team6.scraper;
package ntnu.systemutvikling.team6.scraper.scraperComponents;

import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;

import java.util.List;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.openqa.selenium.By;
Expand Down

0 comments on commit 926660c

Please sign in to comment.