Skip to content

Commit

Permalink
Merge pull request #13 from torave/tora
Browse files Browse the repository at this point in the history 
Tora
  • Loading branch information
@hannhegg
hannhegg authored and GitHub Enterprise committed Apr 22, 2025
2 parents 1e2b69f + 8caff5a commit cf4f89b
Show file tree
Hide file tree
Showing 16 changed files with 676 additions and 1,157 deletions.
275 changes: 105 additions & 170 deletions notebooks/notebook_compare_one_day_data.ipynb
View file

Large diffs are not rendered by default.

289 changes: 110 additions & 179 deletions notebooks/notebook_compare_one_week_data.ipynb
View file

Large diffs are not rendered by default.

243 changes: 92 additions & 151 deletions notebooks/notebook_compare_statistic_data.ipynb
View file

Large diffs are not rendered by default.

12 changes: 5 additions & 7 deletions notebooks/notebook_current_data.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,13 @@
"sys.path.append(os.path.abspath(\"../src\"))\n",
"\n",
"# Now we can import the fucntion from the module\n",
"from my_package.fetch_current_data import fetch_current_data\n",
"from my_package.data import fetch_current_data\n",
"\n",
"# Import function to replace nordic (æøå)\n",
"from my_package.util import replace_nordic\n",
"# Import function to for input_place, replace æøå\n",
"from my_package.util import input_place\n",
"\n",
"# User input the city, for the weather\n",
"city_name = input(\"Enter a city in Norway: \")\n",
"\n",
"city_name = replace_nordic(city_name)\n",
"city_name = input_place()\n",
"\n",
"# Stores the return of the function\n",
"data, folder = fetch_current_data(city_name)"
Expand Down Expand Up @@ -66,7 +64,7 @@
"# Gets the absolute path to the src folder\n",
"sys.path.append(os.path.abspath(\"../src\"))\n",
"\n",
"from my_package.write_data import write_data\n",
"from my_package.data import write_data\n",
"\n",
"# The user choose the filename\n",
"filename = input(\"Write filename: \")\n",
Expand Down
168 changes: 35 additions & 133 deletions notebooks/notebook_one_day_data.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,7 @@
" \n",
" return date_input, [ts[0] for ts in timestamps]\n",
"\n",
"date, timestamps = get_unix_timestamps_for_day()\n",
"\n"
"date, timestamps = get_unix_timestamps_for_day()"
]
},
{
Expand All @@ -96,23 +95,19 @@
"sys.path.append(os.path.abspath(\"../src\"))\n",
"\n",
"# Now we can import the fucntion from the module\n",
"from my_package.fetch_data import fetch_data\n",
"from my_package.data import fetch_time_data\n",
"\n",
"# Import function to replace nordic (æøå)\n",
"from my_package.util import replace_nordic\n",
"# Import function to for input_place, replace æøå\n",
"from my_package.util import input_place\n",
"\n",
"# User choose a city they want the weather data from\n",
"city_name = input(\"Enter city name: \")\n",
"\n",
"city_name = replace_nordic(city_name)\n",
"# User input the city, for the weather\n",
"city_name = input_place()\n",
"\n",
"# Start_date is the first timestamp, end_date is the last\n",
"start_date, end_date = timestamps[0], timestamps[-1]\n",
"\n",
"city_name = replace_nordic(city_name)\n",
"\n",
"# Stores the values in the variables\n",
"weather_data, folder = fetch_data(start_date, end_date, city_name)"
"weather_data, folder = fetch_time_data(start_date, end_date, city_name)"
]
},
{
Expand All @@ -136,7 +131,7 @@
"# Gets the absolute path to the src folder\n",
"sys.path.append(os.path.abspath(\"../src\"))\n",
"\n",
"from my_package.write_data import write_data\n",
"from my_package.data import write_data\n",
"\n",
"filename = input(\"Write filename: \")\n",
"\n",
Expand All @@ -150,7 +145,7 @@
"source": [
"### Lese fra fil\n",
"\n",
"Henter opp data lagret i filen, lagd over, og skriver ut lesbart ved hjelp av pandas"
"Ved hjelp av funksjonen `extract_city_df` fjernes unødvendige kolonner, og dataen blir normalisert for lettere lesbarhet."
]
},
{
Expand All @@ -164,7 +159,7 @@
"# Reads from file using pandas\n",
"weather_data = pd.read_json(f'../data/output_stedsnavn/data_{filename}.json')\n",
"\n",
"from my_package.util import extract_city_df\n",
"from my_package.data import extract_city_df\n",
"\n",
"df = extract_city_df(weather_data)\n",
"display(df)"
Expand Down Expand Up @@ -212,10 +207,10 @@
"plt.figure(figsize=(12, 6))\n",
"\n",
"# Scatter plot for each temperature reading\n",
"plt.scatter(x_axis, temp, color='tab:green', label='Temperaturmålinger', alpha=0.6)\n",
"plt.scatter(x_axis, temp, color='tab:red', label='Temperaturmålinger', alpha=0.7)\n",
"\n",
"# Add a horizontal line for the mean temperature\n",
"plt.axhline(y=temp_mean, color='green', linestyle='--', label=f'Gj.snitt {temp_mean}°C')\n",
"plt.axhline(y=temp_mean, color='tab:red', linestyle=\"dashed\", label=f'Gj.snitt {temp_mean}°C')\n",
"\n",
"# Get the current axis and store it as ax\n",
"ax = plt.gca()\n",
Expand Down Expand Up @@ -250,8 +245,11 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"### Visualiserer nedbør\n",
"Ved hjelp av matplotlib visualiserer vi nedbør for ønsket dag."
"### Sjekker eksitensen av kolonner\n",
"\n",
"Gjennom prosjektet har vi oppdaget at kolonnene som ofte mangler er 'rain.1h' og 'snow.1h'. Derfor importerer vi funksjonen `ensure_column` som tar inn dataframen og kolonnene vi vil sjekke. Dersom kolonnene ikke eksisterer blir de lagt til og fylt med 'NaN' før dataframen returneres.\n",
"\n",
"Sjekker også for kolonnene 'wind.gust' og 'wind.speed' da de skal brukes til plotting senere."
]
},
{
Expand All @@ -260,76 +258,15 @@
"metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"import matplotlib.dates as mdates\n",
"import numpy as np\n",
"\n",
"from my_package.util import ensure_rain_column\n",
"from my_package.util import ensure_snow_column\n",
"\n",
"x_axis = df.index\n",
"\n",
"# Checks if the rain is a value, it will not be if it is no rain and then cause a KeyError\n",
"try:\n",
" rain = df['rain.1h']\n",
"\n",
"# If no rain, make the rain column and fill it with NaN\n",
"except KeyError:\n",
" df = ensure_rain_column(df)\n",
"from my_package.util import ensure_column\n",
"\n",
"# Checks if the snow is a value, it will not be if it is no rain and then cause a KeyError\n",
"try:\n",
" snow = df['snow.1h']\n",
"# The columns we want to check if exsist\n",
"columns_to_ensure = ['rain.1h', 'snow.1h', 'wind.speed', 'wind.gust']\n",
"\n",
"# If no snow, make the snow column and fill it with NaN\n",
"except KeyError:\n",
" df = ensure_snow_column(df)\n",
"# Runs the function with wanted colummns\n",
"df = ensure_column(df, columns_to_ensure)\n",
"\n",
"# Choose the width and height of the plot\n",
"plt.figure(figsize=(15, 6))\n",
"\n",
"# Check with rain, will cause NameError if the try/except over fails\n",
"try:\n",
" plt.bar(x_axis, rain, width=0.02, alpha=0.5, color='tab:blue', label='rain')\n",
"except: NameError\n",
"\n",
"# Check with snow, will cause NameError if the try/except over fails\n",
"try: \n",
" plt.bar(x_axis, snow, width=0.02, alpha=0.5, color='tab:grey', label='snow')\n",
"except: NameError\n",
"\n",
"# Get the current axsis, and store it as ax\n",
"ax = plt.gca()\n",
"\n",
"# Use the current ax, to get a tick-mark on the x_axis for each hour, and print like \"HH:MM\"\n",
"ax.xaxis.set_major_locator(mdates.HourLocator())\n",
"ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))\n",
"\n",
"# Add the label-desciption\n",
"plt.legend(loc = 'upper right')\n",
"\n",
"# Add title to the plot, with date\n",
"plt.title(f'Precipitation {city_name}, ({date}))')\n",
"\n",
"# Shows the plot\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Vise dataframe, med nye kolonner\n",
"Hvis dataframen ikke inneholdt 'rain.1h' eller 'snow.1h', skal de nå ha blitt lagt til med 'NaN' verdier."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Display df, to see if 'rain.1h' and 'snow.1h' was added with NaN values\n",
"# Display dataframe with eventual changes\n",
"display(df)"
]
},
Expand All @@ -340,7 +277,7 @@
"### Sjekk for manglende verdier\n",
"Missigno sjekker og visualiserer manglende verdier, slik at det blir lettere å se hvilke kolonner feilen ligger i. \n",
"\n",
"Vis the blir \"hull\" i en søyle, tyder the på manglende verdier."
"Hvis det blir \"hull\" i en søyle, tyder the på manglende verdier."
]
},
{
Expand All @@ -364,9 +301,9 @@
"\n",
"Under sjekker vi først om regn eller snø er i målingen, og hvis de er, bytter vi ut NaN med 0. \n",
"\n",
"Så sjekker vi om alle verdiene i en kolonne er 'NaN', isåfall så fjerner vi hele kolonnen. Grunne til at dette ikke inkluderer snø og regn, er fordi vi senere plotter disse verdiene, og da får vi ikke feil om verdien er 0, men vil få om hele kolonnen mangler.\n",
"Så sjekker vi om alle verdiene i en kolonne er 'NaN', isåfall så fjerner vi hele kolonnen. Grunnen til at dette ikke inkluderer snø og regn, er fordi vi senere plotter disse verdiene, og da får vi ikke feil om verdien er 0, men vil få om hele kolonnen mangler.\n",
"\n",
"Deretter sjekker vi andre verdier, og bytter enten 'NaN' med 0, eller med verdien før. Verdiene vi setter til 0 gjelder da snø, regn og vind, resten blir satt til verdien før."
"Deretter bruker vi interpolate dersom de skulle være NaN verdier, det er en funksjon som tar utgangspunkt i verdien før og verdien etter for å 'gjette' verdien som mangler. Vi har lagt til 'limit-direction', som gjør at den gjetter selv om man bare har en verdi på siden, som feks. første og siste verdi. "
]
},
{
Expand All @@ -375,54 +312,19 @@
"metadata": {},
"outputs": [],
"source": [
"from my_package.util import fill_rain_column\n",
"from my_package.util import fill_snow_column\n",
"from my_package.util import fill_column_0\n",
"\n",
"df = fill_rain_column(df)\n",
"# The columns we want to replace 'NaN' with 0\n",
"columns_to_0 = ['rain.1h', 'snow.1h', 'wind.gust']\n",
"\n",
"df = fill_snow_column(df)\n",
"# Runs the function with wanted columns\n",
"df = fill_column_0(df, columns_to_0)\n",
"\n",
"# Drops all the columns, if it has 'NaN' value.\n",
"df = df.dropna(axis='columns', how='all')\n",
"\n",
"# If wind_gust is stored, fill the NaN with 0\n",
"try: \n",
" df['wind.gust'] = df['wind.gust'].fillna(0)\n",
"except KeyError:\n",
" print(\"['wind.gust'], not in df\")\n",
"\n",
"# If wind_deg is stored, fill the NaN with 0\n",
"try: \n",
" df['wind.deg'] = df['wind.deg'].fillna(0)\n",
"except KeyError:\n",
" print(\"['wind.deg'], not in df\")\n",
"\n",
"# If wind_speed is stored, fill the NaN with 0\n",
"try: \n",
" df['wind.speed'] = df['wind.speed'].fillna(0)\n",
"except KeyError:\n",
" print(\"['wind.speed'], not in df\")\n",
"\n",
"# If temperature is missing, take the same as the one before\n",
"df['main.temp'] = df['main.temp'].fillna('obj.ffill()')\n",
"\n",
"# Forward fill missing values in what the temperature feels like\n",
"df['main.feels_like'] = df['main.feels_like'].fillna('obj.ffill()')\n",
"\n",
"# Forward fill missing values in the pressure\n",
"df['main.pressure'] = df['main.pressure'].fillna('obj.ffill()')\n",
"\n",
"# Forward fill missing values in the humidity\n",
"df['main.humidity'] = df['main.humidity'].fillna('obj.ffill()')\n",
"\n",
"# Forward fill missing values in the lowest temperature \n",
"df['main.temp_min'] = df['main.temp_min'].fillna('obj.ffill()')\n",
"\n",
"# Forward fill missing values in the highest temperature \n",
"df['main.temp_max'] = df['main.temp_max'].fillna('obj.ffill()')\n",
"\n",
"# Forward fill missing values of clouds\n",
"df['clouds.all'] = df['clouds.all'].fillna('obj.ffill()')\n",
"# Interpolate other missing 'NaN'-values\n",
"df = df.interpolate(method='linear', limit_direction='both')\n",
"\n",
"# Display the df, now without NaN\n",
"display(df)"
Expand Down Expand Up @@ -479,7 +381,7 @@
"# x_axis set to the index, which mean the datetime\n",
"x_axis = df.index\n",
"\n",
"# Gets the values\n",
"# Gets the values we need to for the plot\n",
"rain = df['rain.1h']\n",
"temp = df['main.temp']\n",
"snow = df['snow.1h']\n",
Expand Down Expand Up @@ -685,7 +587,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.6"
"version": "3.12.5"
}
},
"nbformat": 4,
Expand Down
Loading

0 comments on commit cf4f89b

Please sign in to comment.