diff --git a/aggregateweather.py b/aggregateweather.py new file mode 100755 index 0000000..18173c8 --- /dev/null +++ b/aggregateweather.py @@ -0,0 +1,172 @@ +#!/usr/bin/python3 + +from datetime import datetime +from eto import ETo, datasets +from io import StringIO +import argparse +import configparser +import mysql.connector +import pandas as pd +import requests +import sys + +def usage(): + print( "Usage: " + __file__ + "[--config path_to_file.ini] [--station smhi_station_number ] --db database] [--host dbhost] [--user dbuser] [--password dbpassword]\n" + "Default configfile is weather.ini, any parameter can be overwritten on the command line") +parser = argparse.ArgumentParser() +parser.add_argument('--config') +parser.add_argument('--db') +parser.add_argument('--host') +parser.add_argument('--user') +parser.add_argument('--password') +parser.add_argument('--station') +args = parser.parse_args() +config_file = "weather.ini" +if args.config: + config_file = args.config +config = configparser.ConfigParser() +config.read(config_file) +db = config['MySQL']['db'] +host = config['MySQL']['host'] +user = config['MySQL']['user'] +password = config['MySQL']['password'] + +station = config['SMHI']['station'] + + +# Defaults +z_msl = 48.854 +lat = 59.178503 +lon = 17.909265 +TZ_lon = lon +freq = 'D' + +if args.station: + station = args.station +if not station: + station = "97100" +if args.db: + db = args.db +if args.host: + host = args.host +if not host: + host = "localhost" +if args.user: + user = args.usr +if args.password: + password = args.password + +if not (db and host and user and password): + usage() + sys.exit(1) + +pd.options.mode.chained_assignment = None +station_data = requests.get(url='https://opendata-download-metobs.smhi.se/api/version/1.0/parameter/1/station/{}.json'.format(station)).json() +newest_to = 0 +for i in station_data['position']: + if i['to'] > newest_to: + z_msl = i['height'] + lat = i['latitude'] + lon = i['longitude'] + TZ_lon = lon + + +mydb = mysql.connector.connect( + auth_plugin='mysql_native_password', + database=db, + host=host, + passwd=password, + user=user +) + +cursor = mydb.cursor(); + +date_select = ( + "SELECT DISTINCT `date` " + "FROM weather " +) + +cursor.execute(date_select) +dates = cursor.fetchall() +csv = "date,T_max,T_min,T_mean,RH_max,RH_min,RH_mean,Rainfall\n" +for i in dates: + working_date = i[0].strftime('%Y-%m-%d') + day_select = ( + 'SELECT * FROM weather ' + 'WHERE date = "{}"'.format(working_date) + ) + cursor.execute(day_select) + day = cursor.fetchall() + sum_rain = 0 + T_max = -9999 + T_min = 9999 + RH_max = -9999 + RH_min = 9999 + sum_temp = 0 + sum_rel_hum = 0 + sum_windspeed = 0 + counter = 0 + for j in day: + counter += 1 + # +----------------+---------------+------+-----+---------+----------------+ + #| Field | Type | Null | Key | Default | Extra | + #+----------------+---------------+------+-----+---------+----------------+ + #| observation_id | int | NO | PRI | NULL | auto_increment | + #| date | date | YES | MUL | NULL | | + #| time | time | YES | | NULL | | + #| rainfall | float | YES | | NULL | | + #| rel_hum | decimal(10,0) | YES | | NULL | | + #| temp | float | YES | | NULL | | + #| windspeed | float | YES | | NULL | | + #| winddir | int | YES | | NULL | | + #+----------------+---------------+------+-----+---------+----------------+ + observation_id = 0 + date = 1 + time = 2 + rainfall = 3 + rel_hum = 4 + temp = 5 + windspeed = 6 + winddir = 7 + sum_rain += j[rainfall] + sum_temp += j[temp] + sum_rel_hum += j[rel_hum] + sum_windspeed += j[windspeed] + if T_max < j[temp]: + T_max = j[temp] + if T_min > j[temp]: + T_min = j[temp] + if RH_max < j[rel_hum]: + RH_max = j[rel_hum] + if RH_min > j[rel_hum]: + RH_min = j[rel_hum] + T_mean = sum_temp / counter + RH_mean = sum_rel_hum / counter + csv += working_date + "," + str(T_max) + "," + str(T_min) + "," + str(T_mean) + "," + str(RH_max) + "," + str(RH_min) +"," + str(RH_mean) + "," + str(sum_rain) + "\n" +DATA = StringIO(csv) +tsdata = pd.read_csv(DATA, parse_dates=True, infer_datetime_format=True, index_col='date') +et1 = ETo() +et1.param_est(tsdata, freq, z_msl, lat, lon, TZ_lon) +et1.ts_param.head() +eto1 = et1.eto_hargreaves() + +upsert = ( + "REPLACE INTO aggregated_weather " + "(Date, T_max, T_min, T_mean, RH_max, RH_min, RH_mean, Rainfall, ETo, station) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" +) + + + +for key, value in eto1.items(): + aggdate = key.strftime('%Y-%m-%d') + data = ( + aggdate,float(tsdata.loc[aggdate, 'T_max']),float(tsdata.loc[aggdate, 'T_min']),float(tsdata.loc[aggdate, 'T_mean']),float(tsdata.loc[aggdate, 'RH_max']),float(tsdata.loc[aggdate, 'RH_min']),float(tsdata.loc[aggdate, 'RH_mean']),float(tsdata.loc[aggdate, 'Rainfall']),float(value),int(station) + ) + cursor.execute(upsert, data) + + + +mydb.commit() +mydb.close() + diff --git a/getweather.py b/getweather.py new file mode 100755 index 0000000..554ed87 --- /dev/null +++ b/getweather.py @@ -0,0 +1,111 @@ +#!/usr/bin/python3 + +from datetime import datetime +import argparse +import configparser +import mysql.connector +import requests +import sys + +def usage(): + print( "Usage: " + __file__ + "[--config path_to_file.ini] [--station smhi_station_number] [--period latest-hour|latest-day|latest-months] [--db database] [--host dbhost] [--user dbuser] [--password dbpassword]\n" + "Default configfile is weather.ini, any parameter can be overwritten on the command line") +parser = argparse.ArgumentParser() +parser.add_argument('--config') +parser.add_argument('--station') +parser.add_argument('--period') +parser.add_argument('--db') +parser.add_argument('--host') +parser.add_argument('--user') +parser.add_argument('--password') +args = parser.parse_args() +config_file = "weather.ini" +if args.config: + config_file = args.config +config = configparser.ConfigParser() +config.read(config_file) +db = config['MySQL']['db'] +host = config['MySQL']['host'] +user = config['MySQL']['user'] +password = config['MySQL']['password'] +station = config['SMHI']['station'] +period = config['SMHI']['period'] + +if args.station: + station = args.station +if not station: + station = "97100" +if args.period: + period = args.period +if not period: + period = "latest-day" + +if period not in ["latest-hour", "latest-day", "latest-months"]: + period = "latest-day" + +if args.db: + db = args.db +if args.host: + host = args.host +if not host: + host = "localhost" +if args.user: + user = args.usr +if args.password: + password = args.password + +if not (db and host and user and password): + usage() + sys.exit(1) + +mydb = mysql.connector.connect( + auth_plugin='mysql_native_password', + database=db, + host=host, + passwd=password, + user=user +) + +cursor = mydb.cursor(); + +upsert = ( + "REPLACE INTO weather " + "(date, time, rainfall, rel_hum, temp, winddir, windspeed, station) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s)" +) +# Metrics mapping +rainfall = "7" +rel_hum = "6" +temp = "1" +winddir = "3" +windspeed = "4" +results = {} +for metric in [rainfall,rel_hum,temp,winddir,windspeed]: + results[metric] = requests.get(url='https://opendata-download-metobs.smhi.se/api/version/1.0/parameter/{}/station/{}/period/{}/data.json'.format(metric,station,period)) + +rainfall_arr = sorted(results[rainfall].json()['value'], key = lambda i: i['date']) +rel_hum_arr = sorted(results[rel_hum].json()['value'], key = lambda i: i['date']) +temp_arr = sorted(results[temp].json()['value'], key = lambda i: i['date']) +winddir_arr = sorted(results[winddir].json()['value'], key = lambda i: i['date']) +windspeed_arr = sorted(results[windspeed].json()['value'], key = lambda i: i['date']) +i = 0 +for k in rainfall_arr: + ts = int(k['date'] / 1000) + date = datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d') + time = datetime.utcfromtimestamp(ts).strftime('%H:%M:%S') + rain = k['value'] + hum = rel_hum_arr[i]['value'] + te = temp_arr[i]['value'] + widi = winddir_arr[i]['value'] + wisp = windspeed_arr[i]['value'] + + data = ( + date, time, rain, hum, te, widi, wisp, station + ) + cursor.execute(upsert, data) + i = i + 1 + +mydb.commit() +mydb.close() + + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7b5004f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +mysql-connector-python +eto diff --git a/weather.ini b/weather.ini new file mode 100644 index 0000000..e9a2267 --- /dev/null +++ b/weather.ini @@ -0,0 +1,10 @@ +[MySQL] +db = weather +host = localhost +user = your_user_name +password = secret_password + +[SMHI] +station = 97100 +period = latest-months + diff --git a/weather.sql b/weather.sql new file mode 100644 index 0000000..75ca1b0 --- /dev/null +++ b/weather.sql @@ -0,0 +1,27 @@ +CREATE TABLE `weather` ( + `observation_id` int NOT NULL AUTO_INCREMENT, + `date` date DEFAULT NULL, + `time` time DEFAULT NULL, + `rainfall` float DEFAULT NULL, + `rel_hum` decimal(10,0) DEFAULT NULL, + `temp` float DEFAULT NULL, + `windspeed` float DEFAULT NULL, + `winddir` int DEFAULT NULL, + PRIMARY KEY (`observation_id`), + UNIQUE KEY `date_time_idx` (`date`,`time`) +) + +CREATE TABLE `aggregated_weather` ( + `Date_id` int NOT NULL AUTO_INCREMENT, + `Date` date DEFAULT NULL, + `T_max` float DEFAULT NULL, + `T_min` float DEFAULT NULL, + `T_mean` float DEFAULT NULL, + `RH_max` float DEFAULT NULL, + `RH_min` float DEFAULT NULL, + `RH_mean` float DEFAULT NULL, + `Rainfall` float DEFAULT NULL, + `ETo` float DEFAULT NULL, + PRIMARY KEY (`Date_id`), + UNIQUE KEY `date_idx` (`Date`) +)