weather/aggregateweather.py

#!/usr/bin/python3

from datetime import datetime
from eto import ETo, datasets
from io import StringIO
import argparse
import configparser
import mysql.connector
import pandas as pd
import requests
import sys

def usage():
    print(  "Usage: " + __file__ + "[--config path_to_file.ini] [--station smhi_station_number ] --db database] [--host dbhost] [--user dbuser] [--password dbpassword]\n"
            "Default configfile is weather.ini, any parameter can be overwritten on the command line")
parser = argparse.ArgumentParser()
parser.add_argument('--config')
parser.add_argument('--db')
parser.add_argument('--host')
parser.add_argument('--user')
parser.add_argument('--password')
parser.add_argument('--station')
args = parser.parse_args()
config_file = "weather.ini"
if args.config:
    config_file = args.config
config = configparser.ConfigParser()
config.read(config_file)
db = config['MySQL']['db']
host = config['MySQL']['host']
user = config['MySQL']['user']
password = config['MySQL']['password']

station = config['SMHI']['station']


# Defaults
z_msl = 48.854
lat = 59.178503
lon = 17.909265
TZ_lon = lon
freq = 'D'

if args.station:
    station = args.station
if not station:
    station = "97100"
if args.db:
    db = args.db
if args.host:
    host = args.host
if not host:
    host = "localhost"
if args.user:
    user = args.usr
if args.password:
    password = args.password

if not (db and host and user and password):
    usage()
    sys.exit(1)

pd.options.mode.chained_assignment = None
station_data = requests.get(url='https://opendata-download-metobs.smhi.se/api/version/1.0/parameter/1/station/{}.json'.format(station)).json()
newest_to = 0
for i in station_data['position']:
    if i['to'] > newest_to:
        z_msl = i['height']
        lat = i['latitude']
        lon = i['longitude']
        TZ_lon = lon
    

mydb = mysql.connector.connect(
  auth_plugin='mysql_native_password',
  database=db,
  host=host,
  passwd=password,
  user=user
)

cursor = mydb.cursor();

date_select = (
    "SELECT DISTINCT `date` "
    "FROM weather "
)

cursor.execute(date_select)
dates = cursor.fetchall()
csv = "date,T_max,T_min,T_mean,RH_max,RH_min,RH_mean,Rainfall\n"
for i in dates:
    working_date = i[0].strftime('%Y-%m-%d')
    day_select = (
        'SELECT * FROM weather '
        'WHERE date = "{}"'.format(working_date)
    )
    cursor.execute(day_select)
    day = cursor.fetchall()
    sum_rain = 0
    T_max = -9999 
    T_min = 9999
    RH_max = -9999
    RH_min = 9999
    sum_temp = 0
    sum_rel_hum = 0
    sum_windspeed = 0
    counter = 0
    for j in day:
        counter += 1
        # +----------------+---------------+------+-----+---------+----------------+
        #| Field          | Type          | Null | Key | Default | Extra          |
        #+----------------+---------------+------+-----+---------+----------------+
        #| observation_id | int           | NO   | PRI | NULL    | auto_increment |
        #| date           | date          | YES  | MUL | NULL    |                |
        #| time           | time          | YES  |     | NULL    |                |
        #| rainfall       | float         | YES  |     | NULL    |                |
        #| rel_hum        | decimal(10,0) | YES  |     | NULL    |                |
        #| temp           | float         | YES  |     | NULL    |                |
        #| windspeed      | float         | YES  |     | NULL    |                |
        #| winddir        | int           | YES  |     | NULL    |                |
        #+----------------+---------------+------+-----+---------+----------------+
        observation_id = 0
        date = 1
        time = 2
        rainfall = 3
        rel_hum = 4
        temp = 5
        windspeed = 6
        winddir = 7
        sum_rain += j[rainfall]
        sum_temp += j[temp]
        sum_rel_hum += j[rel_hum]
        sum_windspeed += j[windspeed]
        if T_max <  j[temp]:
            T_max = j[temp]
        if T_min >  j[temp]:
            T_min = j[temp]
        if RH_max <  j[rel_hum]:
            RH_max = j[rel_hum]
        if RH_min >  j[rel_hum]:
            RH_min = j[rel_hum]
    T_mean = sum_temp / counter
    RH_mean = sum_rel_hum / counter
    csv += working_date + "," + str(T_max) + "," + str(T_min) + "," + str(T_mean) + "," + str(RH_max) + "," + str(RH_min) +"," + str(RH_mean) + "," + str(sum_rain) + "\n"
DATA = StringIO(csv)
tsdata =  pd.read_csv(DATA, parse_dates=True, infer_datetime_format=True, index_col='date')
et1 = ETo()
et1.param_est(tsdata, freq, z_msl, lat, lon, TZ_lon)
et1.ts_param.head()
eto1 = et1.eto_hargreaves()

upsert = (
    "REPLACE INTO aggregated_weather "
    "(Date, T_max, T_min, T_mean, RH_max, RH_min, RH_mean, Rainfall, ETo, station) "
    "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
)



for key, value in eto1.items():
    aggdate = key.strftime('%Y-%m-%d')
    data = (
    aggdate,float(tsdata.loc[aggdate, 'T_max']),float(tsdata.loc[aggdate, 'T_min']),float(tsdata.loc[aggdate, 'T_mean']),float(tsdata.loc[aggdate, 'RH_max']),float(tsdata.loc[aggdate, 'RH_min']),float(tsdata.loc[aggdate, 'RH_mean']),float(tsdata.loc[aggdate, 'Rainfall']),float(value),int(station)
    )
    cursor.execute(upsert, data)



mydb.commit()
mydb.close()