diff --git a/notebooks/get_day_data_notebook.ipynb b/notebooks/get_day_data_notebook.ipynb deleted file mode 100644 index 14af281..0000000 --- a/notebooks/get_day_data_notebook.ipynb +++ /dev/null @@ -1,845 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Velg hvilken dag du vil sjekke været for\n", - "\n", - "For å kunne hente data og gjøre en analyse trenger programmet å vite hvilken dag du vil hente ut for, også skrives alle timene fra den dagen ut.\n", - "\n", - "Dataen skrives inn slik: (yyyy, mm, dd)\n", - "Her følger et eksempel: \n", - "|Hva|Hvordan|Eksempel|\n", - "|:---|:---:|:---:|\n", - "|år|yyyy|2025|\n", - "|måned|mm|03| \n", - "|dato|dd|01| \n", - "\n", - "Denne dataen skrives da inn på følgende hvis: (2025, 03, 01)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Selected date: 2025-02-21\n", - "Unix Timestamp: 1740092400 -> 2025-02-21 00:00:00\n", - "Unix Timestamp: 1740096000 -> 2025-02-21 01:00:00\n", - "Unix Timestamp: 1740099600 -> 2025-02-21 02:00:00\n", - "Unix Timestamp: 1740103200 -> 2025-02-21 03:00:00\n", - "Unix Timestamp: 1740106800 -> 2025-02-21 04:00:00\n", - "Unix Timestamp: 1740110400 -> 2025-02-21 05:00:00\n", - "Unix Timestamp: 1740114000 -> 2025-02-21 06:00:00\n", - "Unix Timestamp: 1740117600 -> 2025-02-21 07:00:00\n", - "Unix Timestamp: 1740121200 -> 2025-02-21 08:00:00\n", - "Unix Timestamp: 1740124800 -> 2025-02-21 09:00:00\n", - "Unix Timestamp: 1740128400 -> 2025-02-21 10:00:00\n", - "Unix Timestamp: 1740132000 -> 2025-02-21 11:00:00\n", - "Unix Timestamp: 1740135600 -> 2025-02-21 12:00:00\n", - "Unix Timestamp: 1740139200 -> 2025-02-21 13:00:00\n", - "Unix Timestamp: 1740142800 -> 2025-02-21 14:00:00\n", - "Unix Timestamp: 1740146400 -> 2025-02-21 15:00:00\n", - "Unix Timestamp: 1740150000 -> 2025-02-21 16:00:00\n", - "Unix Timestamp: 1740153600 -> 2025-02-21 17:00:00\n", - "Unix Timestamp: 1740157200 -> 2025-02-21 18:00:00\n", - "Unix Timestamp: 1740160800 -> 2025-02-21 19:00:00\n", - "Unix Timestamp: 1740164400 -> 2025-02-21 20:00:00\n", - "Unix Timestamp: 1740168000 -> 2025-02-21 21:00:00\n", - "Unix Timestamp: 1740171600 -> 2025-02-21 22:00:00\n", - "Unix Timestamp: 1740175200 -> 2025-02-21 23:00:00\n" - ] - } - ], - "source": [ - "import datetime\n", - "import time\n", - "\n", - "#makes a function so the star and end date is the same date, with all hours of that date\n", - "def get_unix_timestamps_for_day():\n", - " date_input = input(\"Choose a date (yyyy, mm, dd): \")\n", - " date_components = date_input.split(\",\")\n", - " year = int(date_components[0])\n", - " month = int(date_components[1])\n", - " day = int(date_components[2])\n", - "\n", - "#goes trough all hours of the day, use %Y-%m-%d etc. from pythons strftime to convert datetime into a readable string \n", - " timestamps = []\n", - " for hour in range(24):\n", - " dt = datetime.datetime(year, month, day, hour, 0)\n", - " unix_timestamp = int(time.mktime(dt.timetuple()))\n", - " timestamps.append((unix_timestamp, dt.strftime('%Y-%m-%d %H:%M:%S'))) \n", - "\n", - "#prints the date chosen\n", - " print(f\"\\nSelected date: {year}-{month:02d}-{day:02d}\")\n", - "\n", - "#prints the timestamp and the date an hour of the day after\n", - " for ts, readable in timestamps:\n", - " print(f\"Unix Timestamp: {ts} -> {readable}\")\n", - " \n", - " return [ts[0] for ts in timestamps]\n", - "\n", - "timestamps = get_unix_timestamps_for_day()\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Velg en by i Norge og få data\n", - "\n", - "Skriv inn en by du ønsker data fra, foreløpig er det begrenset til Norge\n", - "\n", - "Programmet vil deretter hente data å lagre det i en json fil" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Data fetch: ok\n" - ] - } - ], - "source": [ - "import sys\n", - "import os\n", - "\n", - "# Gets the absolute path to the src folder\n", - "sys.path.append(os.path.abspath(\"../src\"))\n", - "\n", - "# Now we can import the fucntion from the module\n", - "from my_package.fetch_data import fetch_data\n", - "\n", - "#user choose a city they want the weather data from\n", - "city_name = input(\"Enter city name: \")\n", - "start_date, end_date = timestamps[0], timestamps[-1]\n", - "weather_data,folder = fetch_data(start_date, end_date, city_name)\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Lagre data i en json-fil\n", - "\n", - "Skriv inn navn for til filen du vil lagre med dataen.\n", - "\n", - "Eks. test\n", - "Da vil filen lagres som data_**test**.json, i mappen \"../data/output_stedsnavn/data_{filnavn}.json\"" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Data has been written to /Users/hanne/Documents/anvendt prosjekt/anvendt_mappe/data/../data/output_stedsnavn/data_test_oslo1.json\n" - ] - } - ], - "source": [ - "# Gets the absolute path to the src folder\n", - "sys.path.append(os.path.abspath(\"../src\"))\n", - "\n", - "from my_package.write_data import write_data\n", - "\n", - "filename = input(\"Write filename: \")\n", - "\n", - "write_data(weather_data, folder, filename)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Lese fra fil\n", - "\n", - "Henter opp data lagret i filen, lagd over, og skriver ut lesbart ved hjelp av pandas" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " message cod city_id calctime cnt \\\n", - "0 Count: 24 200 3143244 0.010569 24 \n", - "1 Count: 24 200 3143244 0.010569 24 \n", - "2 Count: 24 200 3143244 0.010569 24 \n", - "3 Count: 24 200 3143244 0.010569 24 \n", - "4 Count: 24 200 3143244 0.010569 24 \n", - "5 Count: 24 200 3143244 0.010569 24 \n", - "6 Count: 24 200 3143244 0.010569 24 \n", - "7 Count: 24 200 3143244 0.010569 24 \n", - "8 Count: 24 200 3143244 0.010569 24 \n", - "9 Count: 24 200 3143244 0.010569 24 \n", - "10 Count: 24 200 3143244 0.010569 24 \n", - "11 Count: 24 200 3143244 0.010569 24 \n", - "12 Count: 24 200 3143244 0.010569 24 \n", - "13 Count: 24 200 3143244 0.010569 24 \n", - "14 Count: 24 200 3143244 0.010569 24 \n", - "15 Count: 24 200 3143244 0.010569 24 \n", - "16 Count: 24 200 3143244 0.010569 24 \n", - "17 Count: 24 200 3143244 0.010569 24 \n", - "18 Count: 24 200 3143244 0.010569 24 \n", - "19 Count: 24 200 3143244 0.010569 24 \n", - "20 Count: 24 200 3143244 0.010569 24 \n", - "21 Count: 24 200 3143244 0.010569 24 \n", - "22 Count: 24 200 3143244 0.010569 24 \n", - "23 Count: 24 200 3143244 0.010569 24 \n", - "\n", - " list \n", - "0 {'dt': 1740092400, 'main': {'temp': -0.97, 'fe... \n", - "1 {'dt': 1740096000, 'main': {'temp': -0.97, 'fe... \n", - "2 {'dt': 1740099600, 'main': {'temp': -0.97, 'fe... \n", - "3 {'dt': 1740103200, 'main': {'temp': -0.97, 'fe... \n", - "4 {'dt': 1740106800, 'main': {'temp': -0.97, 'fe... \n", - "5 {'dt': 1740110400, 'main': {'temp': -0.42, 'fe... \n", - "6 {'dt': 1740114000, 'main': {'temp': 0.69000000... \n", - "7 {'dt': 1740117600, 'main': {'temp': 1.25, 'fee... \n", - "8 {'dt': 1740121200, 'main': {'temp': 1.81, 'fee... \n", - "9 {'dt': 1740124800, 'main': {'temp': 1.81, 'fee... \n", - "10 {'dt': 1740128400, 'main': {'temp': 3.31, 'fee... \n", - "11 {'dt': 1740132000, 'main': {'temp': 3.31, 'fee... \n", - "12 {'dt': 1740135600, 'main': {'temp': 3.31, 'fee... \n", - "13 {'dt': 1740139200, 'main': {'temp': 3.31, 'fee... \n", - "14 {'dt': 1740142800, 'main': {'temp': 3.87, 'fee... \n", - "15 {'dt': 1740146400, 'main': {'temp': 3.87, 'fee... \n", - "16 {'dt': 1740150000, 'main': {'temp': 3.31, 'fee... \n", - "17 {'dt': 1740153600, 'main': {'temp': 3.31, 'fee... \n", - "18 {'dt': 1740157200, 'main': {'temp': 3.31, 'fee... \n", - "19 {'dt': 1740160800, 'main': {'temp': 3.31, 'fee... \n", - "20 {'dt': 1740164400, 'main': {'temp': 2.92, 'fee... \n", - "21 {'dt': 1740168000, 'main': {'temp': 2.92, 'fee... \n", - "22 {'dt': 1740171600, 'main': {'temp': 2.92, 'fee... \n", - "23 {'dt': 1740175200, 'main': {'temp': 2.92, 'fee... \n" - ] - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "weather_data = pd.read_json(f'../data/output_stedsnavn/data_{filename}.json')\n", - "\n", - "print(weather_data)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
main.tempmain.feels_likemain.pressuremain.humiditymain.temp_minmain.temp_maxwind.speedwind.degwind.gustclouds.allsnow.1hrain.1h
dt
2025-02-20 23:00:00-0.97-5.54101691-0.97-0.974.0916212.291001.94NaN
2025-02-21 00:00:00-0.97-5.23101697-0.97-0.973.6715913.071001.54NaN
2025-02-21 01:00:00-0.97-5.37101597-0.97-0.973.8516013.481002.05NaN
2025-02-21 02:00:00-0.97-5.75101597-0.97-0.974.4017612.251000.56NaN
2025-02-21 03:00:00-0.97-5.21101498-0.97-0.973.641729.68100NaNNaN
2025-02-21 04:00:00-0.42-4.16101498-0.42-0.423.181849.60100NaNNaN
2025-02-21 05:00:000.69-2.141014980.690.692.441948.92100NaNNaN
2025-02-21 06:00:001.25-1.091014981.251.252.082009.31100NaNNaN
2025-02-21 07:00:001.811.811015991.811.811.222086.28100NaNNaN
2025-02-21 08:00:001.811.811015991.811.811.231874.22100NaNNaN
2025-02-21 09:00:003.313.311016842.363.311.221804.08100NaNNaN
2025-02-21 10:00:003.313.311016842.923.311.321915.27100NaNNaN
2025-02-21 11:00:003.311.951017842.923.311.551846.56100NaNNaN
2025-02-21 12:00:003.310.991017843.313.472.4017410.09100NaNNaN
2025-02-21 13:00:003.871.741016843.473.872.311679.24100NaNNaN
2025-02-21 14:00:003.871.831016843.473.872.211759.12100NaNNaN
2025-02-21 15:00:003.311.181017882.923.312.2117910.36100NaNNaN
2025-02-21 16:00:003.311.261016882.923.312.1316910.30100NaN0.30
2025-02-21 17:00:003.311.281016882.923.312.1116510.59100NaNNaN
2025-02-21 18:00:003.311.431015882.923.311.9718612.00100NaNNaN
2025-02-21 19:00:002.920.571015992.922.922.3618313.15100NaNNaN
2025-02-21 20:00:002.920.431014992.922.922.5117312.01100NaN0.12
2025-02-21 21:00:002.920.001014992.922.923.0117512.17100NaN0.35
2025-02-21 22:00:002.92-0.241014992.922.923.3317712.50100NaN0.69
\n", - "
" - ], - "text/plain": [ - " main.temp main.feels_like main.pressure main.humidity \\\n", - "dt \n", - "2025-02-20 23:00:00 -0.97 -5.54 1016 91 \n", - "2025-02-21 00:00:00 -0.97 -5.23 1016 97 \n", - "2025-02-21 01:00:00 -0.97 -5.37 1015 97 \n", - "2025-02-21 02:00:00 -0.97 -5.75 1015 97 \n", - "2025-02-21 03:00:00 -0.97 -5.21 1014 98 \n", - "2025-02-21 04:00:00 -0.42 -4.16 1014 98 \n", - "2025-02-21 05:00:00 0.69 -2.14 1014 98 \n", - "2025-02-21 06:00:00 1.25 -1.09 1014 98 \n", - "2025-02-21 07:00:00 1.81 1.81 1015 99 \n", - "2025-02-21 08:00:00 1.81 1.81 1015 99 \n", - "2025-02-21 09:00:00 3.31 3.31 1016 84 \n", - "2025-02-21 10:00:00 3.31 3.31 1016 84 \n", - "2025-02-21 11:00:00 3.31 1.95 1017 84 \n", - "2025-02-21 12:00:00 3.31 0.99 1017 84 \n", - "2025-02-21 13:00:00 3.87 1.74 1016 84 \n", - "2025-02-21 14:00:00 3.87 1.83 1016 84 \n", - "2025-02-21 15:00:00 3.31 1.18 1017 88 \n", - "2025-02-21 16:00:00 3.31 1.26 1016 88 \n", - "2025-02-21 17:00:00 3.31 1.28 1016 88 \n", - "2025-02-21 18:00:00 3.31 1.43 1015 88 \n", - "2025-02-21 19:00:00 2.92 0.57 1015 99 \n", - "2025-02-21 20:00:00 2.92 0.43 1014 99 \n", - "2025-02-21 21:00:00 2.92 0.00 1014 99 \n", - "2025-02-21 22:00:00 2.92 -0.24 1014 99 \n", - "\n", - " main.temp_min main.temp_max wind.speed wind.deg \\\n", - "dt \n", - "2025-02-20 23:00:00 -0.97 -0.97 4.09 162 \n", - "2025-02-21 00:00:00 -0.97 -0.97 3.67 159 \n", - "2025-02-21 01:00:00 -0.97 -0.97 3.85 160 \n", - "2025-02-21 02:00:00 -0.97 -0.97 4.40 176 \n", - "2025-02-21 03:00:00 -0.97 -0.97 3.64 172 \n", - "2025-02-21 04:00:00 -0.42 -0.42 3.18 184 \n", - "2025-02-21 05:00:00 0.69 0.69 2.44 194 \n", - "2025-02-21 06:00:00 1.25 1.25 2.08 200 \n", - "2025-02-21 07:00:00 1.81 1.81 1.22 208 \n", - "2025-02-21 08:00:00 1.81 1.81 1.23 187 \n", - "2025-02-21 09:00:00 2.36 3.31 1.22 180 \n", - "2025-02-21 10:00:00 2.92 3.31 1.32 191 \n", - "2025-02-21 11:00:00 2.92 3.31 1.55 184 \n", - "2025-02-21 12:00:00 3.31 3.47 2.40 174 \n", - "2025-02-21 13:00:00 3.47 3.87 2.31 167 \n", - "2025-02-21 14:00:00 3.47 3.87 2.21 175 \n", - "2025-02-21 15:00:00 2.92 3.31 2.21 179 \n", - "2025-02-21 16:00:00 2.92 3.31 2.13 169 \n", - "2025-02-21 17:00:00 2.92 3.31 2.11 165 \n", - "2025-02-21 18:00:00 2.92 3.31 1.97 186 \n", - "2025-02-21 19:00:00 2.92 2.92 2.36 183 \n", - "2025-02-21 20:00:00 2.92 2.92 2.51 173 \n", - "2025-02-21 21:00:00 2.92 2.92 3.01 175 \n", - "2025-02-21 22:00:00 2.92 2.92 3.33 177 \n", - "\n", - " wind.gust clouds.all snow.1h rain.1h \n", - "dt \n", - "2025-02-20 23:00:00 12.29 100 1.94 NaN \n", - "2025-02-21 00:00:00 13.07 100 1.54 NaN \n", - "2025-02-21 01:00:00 13.48 100 2.05 NaN \n", - "2025-02-21 02:00:00 12.25 100 0.56 NaN \n", - "2025-02-21 03:00:00 9.68 100 NaN NaN \n", - "2025-02-21 04:00:00 9.60 100 NaN NaN \n", - "2025-02-21 05:00:00 8.92 100 NaN NaN \n", - "2025-02-21 06:00:00 9.31 100 NaN NaN \n", - "2025-02-21 07:00:00 6.28 100 NaN NaN \n", - "2025-02-21 08:00:00 4.22 100 NaN NaN \n", - "2025-02-21 09:00:00 4.08 100 NaN NaN \n", - "2025-02-21 10:00:00 5.27 100 NaN NaN \n", - "2025-02-21 11:00:00 6.56 100 NaN NaN \n", - "2025-02-21 12:00:00 10.09 100 NaN NaN \n", - "2025-02-21 13:00:00 9.24 100 NaN NaN \n", - "2025-02-21 14:00:00 9.12 100 NaN NaN \n", - "2025-02-21 15:00:00 10.36 100 NaN NaN \n", - "2025-02-21 16:00:00 10.30 100 NaN 0.30 \n", - "2025-02-21 17:00:00 10.59 100 NaN NaN \n", - "2025-02-21 18:00:00 12.00 100 NaN NaN \n", - "2025-02-21 19:00:00 13.15 100 NaN NaN \n", - "2025-02-21 20:00:00 12.01 100 NaN 0.12 \n", - "2025-02-21 21:00:00 12.17 100 NaN 0.35 \n", - "2025-02-21 22:00:00 12.50 100 NaN 0.69 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Gjennomsnitts temperatur: 1.92\n", - "max temperature: 3.87\n", - "min temperature: -0.97\n" - ] - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "weather_data = pd.read_json(f'../data/output_stedsnavn/data_{filename}.json')\n", - "\n", - "if 'list' in weather_data:\n", - " df = pd.json_normalize(weather_data['list'])\n", - "\n", - " # Delete duplicates based on the dt row, all the other values can appear more than once, but the date should only appear once\n", - " df = df.drop_duplicates(subset=['dt'])\n", - "\n", - " # The weather column dosnt have any releated information, therefor we delete it\n", - " df = df.drop(columns=\"weather\")\n", - "\n", - " # Convert 'dt' column from Unix timestamp to datetime and set it as the index\n", - " df['dt'] = pd.to_datetime(df['dt'], unit='s')\n", - " df.set_index('dt', inplace=True)\n", - " \n", - "\n", - " # Ensure the DataFrame is displayed correctly\n", - " display(df)\n", - "\n", - "# Extract main values\n", - " temp = df['main.temp']\n", - " humidity = df['main.humidity']\n", - "\n", - " # Extract wind values\n", - " w_speed = df['wind.speed']\n", - "\n", - " # Extract other variables\n", - " clouds = df['clouds.all']\n", - "\n", - " try:\n", - " rain = df['rain.1h']\n", - " except KeyError:\n", - " print(\"'Rain' is not present in the JSON file.\")\n", - "\n", - " try:\n", - " snow = df['snow.1h']\n", - " except KeyError:\n", - " print(\"'Snow' is not present in the JSON file.\")\n", - "\n", - " # Print the average temperature\n", - " print('Gjennomsnitts temperatur: ', temp.mean().round(2))\n", - "\n", - " # Display the temperature column\n", - " # display(temp)\n", - "\n", - " max_temp = df['main.temp'].max()\n", - " min_temp = df['main.temp'].min()\n", - "\n", - " print(\"max temperature:\", max_temp)\n", - " print(\"min temperature:\", min_temp)\n", - " \n", - "else:\n", - " print(\"The 'list' key is not present in the JSON file.\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/notebook_one_day_data.ipynb b/notebooks/notebook_one_day_data.ipynb new file mode 100644 index 0000000..d2972f8 --- /dev/null +++ b/notebooks/notebook_one_day_data.ipynb @@ -0,0 +1,497 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Notebook - One day data\n", + "\n", + "Denne notebooken henter data fra ønsket dag og sted, skriver til fil. Visualiserer manglende verdier, retter opp manglende verdier, og visualisere og lagrer data fra plot." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Velg hvilken dag du vil sjekke været for\n", + "\n", + "For å kunne hente data og gjøre en analyse trenger programmet å vite hvilken dag du vil hente ut for, også skrives alle timene fra den dagen ut. Programmet kan ikke hente ut data fra nåværende, eller senere datoer, altså må man velge datoer fra tidligere tidspunkt.\n", + "\n", + "Dataen skrives inn slik: (yyyy, mm, dd)\n", + "Her følger et eksempel: \n", + "|Hva|Hvordan|Eksempel|\n", + "|:---|:---:|:---:|\n", + "|år|yyyy|2025|\n", + "|måned|mm|03| \n", + "|dato|dd|01| \n", + "\n", + "Denne dataen skrives da inn på følgende hvis: (2025, 03, 01)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import datetime\n", + "import time\n", + "\n", + "# Makes a function so the start and end date is the same date, with all hours of that date\n", + "def get_unix_timestamps_for_day():\n", + " date_input = input(\"Choose a date (yyyy, mm, dd): \")\n", + " date_components = date_input.split(\",\")\n", + " year = int(date_components[0])\n", + " month = int(date_components[1])\n", + " day = int(date_components[2])\n", + "\n", + " # Goes through all hours of the day, use %Y-%m-%d etc. from pythons strftime to convert datetime into a readable string \n", + " timestamps = []\n", + " for hour in range(24):\n", + " dt = datetime.datetime(year, month, day, hour, 0)\n", + " unix_timestamp = int(time.mktime(dt.timetuple()))\n", + " timestamps.append((unix_timestamp, dt.strftime('%Y-%m-%d %H:%M:%S'))) \n", + " \n", + " # Prevents from getting data for the current day, or the future\n", + " if dt >= datetime.datetime.now():\n", + " print(\"Failed, cant use future dates\")\n", + " return None\n", + "\n", + " # Prints the date chosen\n", + " print(f\"Selected date: {year}-{month:02d}-{day:02d}\")\n", + "\n", + " # Prints the timestamp and the date an hour of the day after\n", + " for ts, readable in timestamps:\n", + " print(f\"Unix Timestamp: {ts} -> {readable}\")\n", + " \n", + " return date_input, [ts[0] for ts in timestamps]\n", + "\n", + "date, timestamps = get_unix_timestamps_for_day()\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Velg en by i Norge og få data\n", + "\n", + "Skriv inn en by du ønsker data fra, foreløpig er det begrenset til Norge\n", + "\n", + "Programmet vil deretter hente data å lagre det i en json fil" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import os\n", + "\n", + "# Gets the absolute path to the src folder\n", + "sys.path.append(os.path.abspath(\"../src\"))\n", + "\n", + "# Now we can import the fucntion from the module\n", + "from my_package.fetch_data import fetch_data\n", + "\n", + "# User choose a city they want the weather data from\n", + "city_name = input(\"Enter city name: \")\n", + "\n", + "# Start_date is the first timestamp, end_date is the last\n", + "start_date, end_date = timestamps[0], timestamps[-1]\n", + "\n", + "# Stores the values in the variables\n", + "weather_data, folder = fetch_data(start_date, end_date, city_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Lagre data i en json-fil\n", + "\n", + "Skriv inn navn for til filen du vil lagre med dataen.\n", + "\n", + "Eks. test\n", + "Da vil filen lagres som data_**test**.json, i mappen \"../data/output_stedsnavn/data_{filnavn}.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Gets the absolute path to the src folder\n", + "sys.path.append(os.path.abspath(\"../src\"))\n", + "\n", + "from my_package.write_data import write_data\n", + "\n", + "filename = input(\"Write filename: \")\n", + "\n", + "# Writes the data, with the chosen name\n", + "write_data(weather_data, folder, filename)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Lese fra fil\n", + "\n", + "Henter opp data lagret i filen, lagd over, og skriver ut lesbart ved hjelp av pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "# Reads from file using pandas\n", + "weather_data = pd.read_json(f'../data/output_stedsnavn/data_{filename}.json')\n", + "\n", + "# Checks if 'list' in weather, then proceed because it is the right data\n", + "if 'list' in weather_data:\n", + " # Normalize the json for better readability\n", + " df = pd.json_normalize(weather_data['list'])\n", + "\n", + " # Delete duplicates based on the dt row, all the other values can appear more than once, but the date should only appear once\n", + " df = df.drop_duplicates(subset=['dt'])\n", + "\n", + " # The weather column dosnt have any releated information, therefor we delete it\n", + " df = df.drop(columns=\"weather\")\n", + "\n", + " # Convert 'dt' column from Unix timestamp to datetime and set it as the index\n", + " df['dt'] = pd.to_datetime(df['dt'], unit='s')\n", + " df.set_index('dt', inplace=True)\n", + "\n", + " # Ensure the DataFrame is displayed correctly \n", + " display(df)\n", + " \n", + "else:\n", + " print(\"The 'list' key is not present in the JSON file.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Viser temperaturen\n", + "Regner ut gjennomsnittst-temperatur ved hjelp av innebygde funksjoner. Finner også høyeste og laveste målte temperatur." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Stores the temperature values\n", + "temp = df['main.temp']\n", + "\n", + "temp_mean = temp.mean().round(2)\n", + "\n", + "# Print the average temperature\n", + "print(f'Mean temperatur: {temp_mean}')\n", + "\n", + "# Find the highest and lowest temperatures\n", + "max_temp = df['main.temp'].max().round(2)\n", + "min_temp = df['main.temp'].min().round(2)\n", + "\n", + "print(\"Highest temperature:\", max_temp)\n", + "print(\"Lowest temperature:\", min_temp)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Visualiserer nedbør\n", + "Ved hjelp av matplotlib visualiserer vi nedbør for ønsket dag." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import matplotlib.dates as mdates\n", + "import numpy as np\n", + "\n", + "x_axis = df.index\n", + "\n", + "# Checks if the rain is a value, it will not be if it is no rain and then cause a KeyError\n", + "try:\n", + " rain = df['rain.1h']\n", + "\n", + "# If no rain, make the rain column and fill it with NaN\n", + "except KeyError:\n", + " print(\"'Rain' is not present in the JSON file.\")\n", + " df['rain.1h'] = np.nan\n", + "\n", + "# Checks if the snow is a value, it will not be if it is no rain and then cause a KeyError\n", + "try:\n", + " snow = df['snow.1h']\n", + "\n", + "# If no snow, make the snow column and fill it with NaN\n", + "except KeyError:\n", + " print(\"'Snow' is not present in the JSON file.\")\n", + " df['snow.1h'] = np.nan\n", + "\n", + "# Choose the width and height of the plot\n", + "plt.figure(figsize=(15, 6))\n", + "\n", + "# Check with rain, will cause NameError if the try/except over fails\n", + "try:\n", + " plt.bar(x_axis, rain, width=0.02, alpha=0.5, color='tab:blue', label='rain')\n", + "except: NameError\n", + "\n", + "# Check with snow, will cause NameError if the try/except over fails\n", + "try: \n", + " plt.bar(x_axis, snow, width=0.02, alpha=0.5, color='tab:grey', label='snow')\n", + "except: NameError\n", + "\n", + "# Get the current axsis, and store it as ax\n", + "ax = plt.gca()\n", + "\n", + "# Use the current ax, to get a tick-mark on the x_axis for each hour, and print like \"HH:MM\"\n", + "ax.xaxis.set_major_locator(mdates.HourLocator())\n", + "ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))\n", + "\n", + "# Add the label-desciption\n", + "plt.legend()\n", + "\n", + "# Shows the plot\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Vise dataframe, med nye kolonner\n", + "Hvis dataframen ikke inneholdt 'rain.1h' eller 'snow.1h', skal de nå ha blitt lagt til med 'NaN' verdier." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Display df, to see if 'rain.1h' and 'snow.1h' was added with NaN values\n", + "display(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Sjekk for manglende verdier\n", + "Missigno sjekker og visualiserer manglende verdier, slik at det blir lettere å se hvilke kolonner feilen ligger i. \n", + "\n", + "Vis the blir \"hull\" i en søyle, tyder the på manglende verdier." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import missingno as msno\n", + "\n", + "# Checks for and display missing values\n", + "msno.matrix(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Endre manglende verdier\n", + "I de fleste tilfeller virker dataene å være tilnærmet \"perfekte\", men de inkluderer bare snø og regn dersom det er snø eller regn. Derfor vil vi fa NaN verdier i de målingene det ikke har regnet/snødd. \n", + "\n", + "Under sjekker vi først om regn eller snø er i målingen, og hvis den er, bytter vi ut NaN med 0." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# If rain is stored, fill the NaN with 0\n", + "try: \n", + " df['rain.1h'] = df['rain.1h'].fillna(0)\n", + "except KeyError:\n", + " print([\"'rain.1h', not in df\"])\n", + "\n", + "# If snow is stored, fill the NaN with 0\n", + "try: \n", + " df['snow.1h'] = df['snow.1h'].fillna(0)\n", + "except KeyError:\n", + " print(\"['snow.1h'], not in df\")\n", + "\n", + "# Display the df, now without NaN (atleast for rain and snow)\n", + "display(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Visualisere endring av data\n", + "Har lagt inn en ny missigno visualisering, for å se at de manglende dataene \"forsvinner\" når vi kjører cellen over. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import missingno as msno\n", + "\n", + "# Visulaize the same data again, but now it should be no missing values (atleast for rain and snow)\n", + "msno.matrix(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Visualisere data i en graf\n", + "Ved hjelp av Matplotlib har vi visualiert ønsket data, og ved hjelp av subplot, en modul i matplotlib, kan vi plotte flere verdier i samme graf, og få \"to y-akse\" på samme x-akse. \n", + "\n", + "Temperatur og nedbør får plass i samme graf, hvor man leser temperatur verdiene på venstre side, og nedbørsverdiene på høyre side.\n", + "\n", + "I grafen under, men på samme x-akse, finner vi informasjon om vind, både vindhastighet og vindkast." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import matplotlib.dates as mdates\n", + "import os\n", + "\n", + "# Where the figure should be saved when exported\n", + "output_folder = \"../data/output_fig\"\n", + "\n", + "# Creates the folder if it does not exist\n", + "os.makedirs(output_folder, exist_ok=True)\n", + "\n", + "# x_axis set to the index, which mean the datetime\n", + "x_axis = df.index\n", + "\n", + "# Gets the values\n", + "rain = df['rain.1h']\n", + "temp = df['main.temp']\n", + "snow = df['snow.1h']\n", + "wind_gust = df['wind.gust']\n", + "wind_speed = df['wind.speed']\n", + "\n", + "# Two vertically stacked axis, (2 rows, 1 column), width and height of the figure, and the axis share the same x_axis\n", + "fig, (ax1, ax3) = plt.subplots(2, 1,figsize=(15, 8), sharex=True)\n", + "\n", + "\n", + "# Set the title for the diagram, above the first axis, with city_name and input_date\n", + "ax1.set_title(f'Weather data for {city_name} ({date}) ')\n", + "\n", + "# Plot temperature on the primary y-axis\n", + "ax1.plot(x_axis, temp, color='tab:red', label='Temperature (°C)')\n", + "\n", + "# Design the y-axis for temperatur\n", + "ax1.set_ylabel('Temperature (°C)', color='tab:red')\n", + "ax1.tick_params(axis='y', labelcolor='tab:red')\n", + "\n", + "# Plot Precipitation as bars on the secondary y-axis\n", + "ax2 = ax1.twinx()\n", + "\n", + "# Add rain\n", + "ax2.bar(x_axis, rain, color='tab:blue', alpha=0.5, width=0.02, label='Rain (mm)')\n", + "\n", + "# Add snow\n", + "ax2.bar(x_axis, snow, color='tab:grey', alpha=0.5, width=0.02, label='Snow (mm)')\n", + "\n", + "# Design the y-axis for precipiation\n", + "ax2.set_ylabel(\"Precipitation (mm)\", color='tab:blue')\n", + "ax2.tick_params(axis='y', labelcolor='tab:blue')\n", + "\n", + "\n", + "# Format the x-axis to show all hours, in the format \"HH:MM\"\n", + "ax1.xaxis.set_major_locator(mdates.HourLocator()) \n", + "ax1.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))\n", + "\n", + "# Add label-description for both axis\n", + "ax1.legend(loc='upper left')\n", + "ax2.legend(loc='upper right')\n", + "\n", + "# Add grid, but only vertically\n", + "ax1.grid(axis = 'x')\n", + "\n", + "\n", + "# Plot the wind at the second x-axis (the axis below)\n", + "ax3.plot(x_axis, wind_gust, color='tab:purple', label='Wind_gust')\n", + "ax3.plot(x_axis, wind_speed, color='tab:purple', linestyle='dashed', label='Wind_speed')\n", + "ax3.set_ylabel('Wind (m/s)')\n", + "\n", + "# Add x_label visible for both x-axis\n", + "ax3.set_xlabel('Datetime')\n", + "\n", + "# Add label-description\n", + "ax3.legend(loc='upper right')\n", + "\n", + "# Format the x-axis to show all hours, in the format \"HH:MM\"\n", + "ax3.xaxis.set_major_locator(mdates.HourLocator())\n", + "ax3.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))\n", + "\n", + "# Add grid, but only vertically\n", + "ax3.grid(axis = 'x')\n", + "\n", + "# Adjust layout\n", + "plt.tight_layout()\n", + "\n", + "# Save the plot to the data/output_fig folder\n", + "plot_path = os.path.join(output_folder, f\"weather_data_plot{city_name}.png\")\n", + "plt.savefig(plot_path) # Save the plot as a PNG file\n", + "\n", + "\n", + "# Show the plot\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}