diff --git a/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/FullCharityScrape.java b/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/FullCharityScrape.java new file mode 100644 index 0000000..c88fb11 --- /dev/null +++ b/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/FullCharityScrape.java @@ -0,0 +1,46 @@ +package ntnu.systemutvikling.team6.scraper; + +import ntnu.systemutvikling.team6.models.Charity; +import ntnu.systemutvikling.team6.models.registry.CharityRegistry; +import ntnu.systemutvikling.team6.scraper.scraperComponents.APICharityScraper; +import ntnu.systemutvikling.team6.scraper.scraperComponents.URLCharityScraper; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.http.HttpClient; + +public class FullCharityScrape { + private APICharityScraper apiScraper; + + public FullCharityScrape() throws URISyntaxException { + HttpClient https = HttpClient.newHttpClient(); + this.apiScraper = new APICharityScraper(https); + } + + public CharityRegistry getAPIAndURLCharityData() throws IOException, InterruptedException { + CharityRegistry charityRegistry = null; + if (apiScraper.checkConnection()) { + charityRegistry = apiScraper.parseJSON(apiScraper.getJSONData()); + for (Charity charity : charityRegistry.getAllCharities()) { + System.out.println(charity.getName()); + } + // Scrapes description, logo, categories, and key values from IK + int charityCounter = 0; + for (Charity charity : charityRegistry.getAllCharities()) { + charityCounter++; + + System.out.println("Scraping charity vanity details: " + charityCounter + " of " + charityRegistry.getAllCharities().size()); + URLCharityScraper urlScraper = new URLCharityScraper(charity.getURL()); + urlScraper.scrapeCharityPage(); + + charity.setDescription(urlScraper.getDescription()); + charity.setCategory(urlScraper.getCategories()); + charity.setLogoURL(urlScraper.getLogoURL()); + charity.setKeyValues(urlScraper.getKeyValues()); + byte[] logoBlob = LogoDownloader.downloadImageAsBlob(charity.getLogoURL()); + charity.setLogoBlob(logoBlob); + } + } + return charityRegistry; + } +} diff --git a/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/APICharityData.java b/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/scraperComponents/APICharityData.java similarity index 97% rename from helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/APICharityData.java rename to helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/scraperComponents/APICharityData.java index 0cf6da8..35b10d5 100644 --- a/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/APICharityData.java +++ b/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/scraperComponents/APICharityData.java @@ -1,4 +1,4 @@ -package ntnu.systemutvikling.team6.scraper; +package ntnu.systemutvikling.team6.scraper.scraperComponents; import ntnu.systemutvikling.team6.database.DatabaseSetup; diff --git a/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/APICharityScraper.java b/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/scraperComponents/APICharityScraper.java similarity index 98% rename from helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/APICharityScraper.java rename to helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/scraperComponents/APICharityScraper.java index 389106f..16be61f 100644 --- a/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/APICharityScraper.java +++ b/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/scraperComponents/APICharityScraper.java @@ -1,4 +1,4 @@ -package ntnu.systemutvikling.team6.scraper; +package ntnu.systemutvikling.team6.scraper.scraperComponents; import com.google.gson.Gson; import java.io.IOException; diff --git a/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/URLCharityScraper.java b/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/scraperComponents/URLCharityScraper.java similarity index 99% rename from helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/URLCharityScraper.java rename to helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/scraperComponents/URLCharityScraper.java index 88be3dd..d7b087e 100644 --- a/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/URLCharityScraper.java +++ b/helpmehelpapplication/src/main/java/ntnu/systemutvikling/team6/scraper/scraperComponents/URLCharityScraper.java @@ -1,4 +1,4 @@ -package ntnu.systemutvikling.team6.scraper; +package ntnu.systemutvikling.team6.scraper.scraperComponents; import java.time.Duration; import java.util.ArrayList; diff --git a/helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/APICharityDataTest.java b/helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/scraperComponents/APICharityDataTest.java similarity index 97% rename from helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/APICharityDataTest.java rename to helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/scraperComponents/APICharityDataTest.java index cd90feb..31dc502 100644 --- a/helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/APICharityDataTest.java +++ b/helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/scraperComponents/APICharityDataTest.java @@ -1,4 +1,4 @@ -package ntnu.systemutvikling.team6.scraper; +package ntnu.systemutvikling.team6.scraper.scraperComponents; import static org.junit.jupiter.api.Assertions.*; diff --git a/helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/APICharityScraperTest.java b/helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/scraperComponents/APICharityScraperTest.java similarity index 98% rename from helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/APICharityScraperTest.java rename to helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/scraperComponents/APICharityScraperTest.java index ad5e4a8..04e9a06 100644 --- a/helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/APICharityScraperTest.java +++ b/helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/scraperComponents/APICharityScraperTest.java @@ -1,4 +1,4 @@ -package ntnu.systemutvikling.team6.scraper; +package ntnu.systemutvikling.team6.scraper.scraperComponents; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; diff --git a/helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/URLCharityScraperTest.java b/helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/scraperComponents/URLCharityScraperTest.java similarity index 98% rename from helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/URLCharityScraperTest.java rename to helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/scraperComponents/URLCharityScraperTest.java index 82c28cb..bd5ffb7 100644 --- a/helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/URLCharityScraperTest.java +++ b/helpmehelpapplication/src/test/java/ntnu/systemutvikling/team6/scraper/scraperComponents/URLCharityScraperTest.java @@ -1,9 +1,10 @@ -package ntnu.systemutvikling.team6.scraper; +package ntnu.systemutvikling.team6.scraper.scraperComponents; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; import java.util.List; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.openqa.selenium.By;