micke
/
weather
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Collection of data and aggregation to mysql works

Browse Source
master
Micke Nordin 5 years ago
parent b59e6fb4cb
commit 62b65dc2d8
5 changed files with 322 additions and 0 deletions
Show Stats Download Patch File Download Diff File

172
aggregateweather.py
Unescape View File

@ -0,0 +1,172 @@
#!/usr/bin/python3
from datetime import datetime
from eto import ETo, datasets
from io import StringIO
import argparse
import configparser
import mysql.connector
import pandas as pd
import requests
import sys
def usage():
print( "Usage: " + __file__ + "[--config path_to_file.ini] [--station smhi_station_number ] --db database] [--host dbhost] [--user dbuser] [--password dbpassword]\n"
"Default configfile is weather.ini, any parameter can be overwritten on the command line")
parser = argparse.ArgumentParser()
parser.add_argument('--config')
parser.add_argument('--db')
parser.add_argument('--host')
parser.add_argument('--user')
parser.add_argument('--password')
parser.add_argument('--station')
args = parser.parse_args()
config_file = "weather.ini"
if args.config:
config_file = args.config
config = configparser.ConfigParser()
config.read(config_file)
db = config['MySQL']['db']
host = config['MySQL']['host']
user = config['MySQL']['user']
password = config['MySQL']['password']
station = config['SMHI']['station']
# Defaults
z_msl = 48.854
lat = 59.178503
lon = 17.909265
TZ_lon = lon
freq = 'D'
if args.station:
station = args.station
if not station:
station = "97100"
if args.db:
db = args.db
if args.host:
host = args.host
if not host:
host = "localhost"
if args.user:
user = args.usr
if args.password:
password = args.password
if not (db and host and user and password):
usage()
sys.exit(1)
pd.options.mode.chained_assignment = None
station_data = requests.get(url='https://opendata-download-metobs.smhi.se/api/version/1.0/parameter/1/station/{}.json'.format(station)).json()
newest_to = 0
for i in station_data['position']:
if i['to'] > newest_to:
z_msl = i['height']
lat = i['latitude']
lon = i['longitude']
TZ_lon = lon
mydb = mysql.connector.connect(
auth_plugin='mysql_native_password',
database=db,
host=host,
passwd=password,
user=user
)
cursor = mydb.cursor();
date_select = (
"SELECT DISTINCT `date` "
"FROM weather "
)
cursor.execute(date_select)
dates = cursor.fetchall()
csv = "date,T_max,T_min,T_mean,RH_max,RH_min,RH_mean,Rainfall\n"
for i in dates:
working_date = i[0].strftime('%Y-%m-%d')
day_select = (
'SELECT * FROM weather '
'WHERE date = "{}"'.format(working_date)
)
cursor.execute(day_select)
day = cursor.fetchall()
sum_rain = 0
T_max = -9999
T_min = 9999
RH_max = -9999
RH_min = 9999
sum_temp = 0
sum_rel_hum = 0
sum_windspeed = 0
counter = 0
for j in day:
counter += 1
# +----------------+---------------+------+-----+---------+----------------+
#| Field | Type | Null | Key | Default | Extra |
#+----------------+---------------+------+-----+---------+----------------+
#| observation_id | int | NO | PRI | NULL | auto_increment |
#| date | date | YES | MUL | NULL | |
#| time | time | YES | | NULL | |
#| rainfall | float | YES | | NULL | |
#| rel_hum | decimal(10,0) | YES | | NULL | |
#| temp | float | YES | | NULL | |
#| windspeed | float | YES | | NULL | |
#| winddir | int | YES | | NULL | |
#+----------------+---------------+------+-----+---------+----------------+
observation_id = 0
date = 1
time = 2
rainfall = 3
rel_hum = 4
temp = 5
windspeed = 6
winddir = 7
sum_rain += j[rainfall]
sum_temp += j[temp]
sum_rel_hum += j[rel_hum]
sum_windspeed += j[windspeed]
if T_max < j[temp]:
T_max = j[temp]
if T_min > j[temp]:
T_min = j[temp]
if RH_max < j[rel_hum]:
RH_max = j[rel_hum]
if RH_min > j[rel_hum]:
RH_min = j[rel_hum]
T_mean = sum_temp / counter
RH_mean = sum_rel_hum / counter
csv += working_date + "," + str(T_max) + "," + str(T_min) + "," + str(T_mean) + "," + str(RH_max) + "," + str(RH_min) +"," + str(RH_mean) + "," + str(sum_rain) + "\n"
DATA = StringIO(csv)
tsdata = pd.read_csv(DATA, parse_dates=True, infer_datetime_format=True, index_col='date')
et1 = ETo()
et1.param_est(tsdata, freq, z_msl, lat, lon, TZ_lon)
et1.ts_param.head()
eto1 = et1.eto_hargreaves()
upsert = (
"REPLACE INTO aggregated_weather "
"(Date, T_max, T_min, T_mean, RH_max, RH_min, RH_mean, Rainfall, ETo, station) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
)
for key, value in eto1.items():
aggdate = key.strftime('%Y-%m-%d')
data = (
aggdate,float(tsdata.loc[aggdate, 'T_max']),float(tsdata.loc[aggdate, 'T_min']),float(tsdata.loc[aggdate, 'T_mean']),float(tsdata.loc[aggdate, 'RH_max']),float(tsdata.loc[aggdate, 'RH_min']),float(tsdata.loc[aggdate, 'RH_mean']),float(tsdata.loc[aggdate, 'Rainfall']),float(value),int(station)
)
cursor.execute(upsert, data)
mydb.commit()
mydb.close()

111
getweather.py
Unescape View File

@ -0,0 +1,111 @@
#!/usr/bin/python3
from datetime import datetime
import argparse
import configparser
import mysql.connector
import requests
import sys
def usage():
print( "Usage: " + __file__ + "[--config path_to_file.ini] [--station smhi_station_number] [--period latest-hour|latest-day|latest-months] [--db database] [--host dbhost] [--user dbuser] [--password dbpassword]\n"
"Default configfile is weather.ini, any parameter can be overwritten on the command line")
parser = argparse.ArgumentParser()
parser.add_argument('--config')
parser.add_argument('--station')
parser.add_argument('--period')
parser.add_argument('--db')
parser.add_argument('--host')
parser.add_argument('--user')
parser.add_argument('--password')
args = parser.parse_args()
config_file = "weather.ini"
if args.config:
config_file = args.config
config = configparser.ConfigParser()
config.read(config_file)
db = config['MySQL']['db']
host = config['MySQL']['host']
user = config['MySQL']['user']
password = config['MySQL']['password']
station = config['SMHI']['station']
period = config['SMHI']['period']
if args.station:
station = args.station
if not station:
station = "97100"
if args.period:
period = args.period
if not period:
period = "latest-day"
if period not in ["latest-hour", "latest-day", "latest-months"]:
period = "latest-day"
if args.db:
db = args.db
if args.host:
host = args.host
if not host:
host = "localhost"
if args.user:
user = args.usr
if args.password:
password = args.password
if not (db and host and user and password):
usage()
sys.exit(1)
mydb = mysql.connector.connect(
auth_plugin='mysql_native_password',
database=db,
host=host,
passwd=password,
user=user
)
cursor = mydb.cursor();
upsert = (
"REPLACE INTO weather "
"(date, time, rainfall, rel_hum, temp, winddir, windspeed, station) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"
)
# Metrics mapping
rainfall = "7"
rel_hum = "6"
temp = "1"
winddir = "3"
windspeed = "4"
results = {}
for metric in [rainfall,rel_hum,temp,winddir,windspeed]:
results[metric] = requests.get(url='https://opendata-download-metobs.smhi.se/api/version/1.0/parameter/{}/station/{}/period/{}/data.json'.format(metric,station,period))
rainfall_arr = sorted(results[rainfall].json()['value'], key = lambda i: i['date'])
rel_hum_arr = sorted(results[rel_hum].json()['value'], key = lambda i: i['date'])
temp_arr = sorted(results[temp].json()['value'], key = lambda i: i['date'])
winddir_arr = sorted(results[winddir].json()['value'], key = lambda i: i['date'])
windspeed_arr = sorted(results[windspeed].json()['value'], key = lambda i: i['date'])
i = 0
for k in rainfall_arr:
ts = int(k['date'] / 1000)
date = datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d')
time = datetime.utcfromtimestamp(ts).strftime('%H:%M:%S')
rain = k['value']
hum = rel_hum_arr[i]['value']
te = temp_arr[i]['value']
widi = winddir_arr[i]['value']
wisp = windspeed_arr[i]['value']
data = (
date, time, rain, hum, te, widi, wisp, station
)
cursor.execute(upsert, data)
i = i + 1
mydb.commit()
mydb.close()

2
requirements.txt
Unescape View File

@ -0,0 +1,2 @@
mysql-connector-python
eto

10
weather.ini
Unescape View File

@ -0,0 +1,10 @@
[MySQL]
db = weather
host = localhost
user = your_user_name
password = secret_password
[SMHI]
station = 97100
period = latest-months

27
weather.sql
Unescape View File

@ -0,0 +1,27 @@
CREATE TABLE `weather` (
`observation_id` int NOT NULL AUTO_INCREMENT,
`date` date DEFAULT NULL,
`time` time DEFAULT NULL,
`rainfall` float DEFAULT NULL,
`rel_hum` decimal(10,0) DEFAULT NULL,
`temp` float DEFAULT NULL,
`windspeed` float DEFAULT NULL,
`winddir` int DEFAULT NULL,
PRIMARY KEY (`observation_id`),
UNIQUE KEY `date_time_idx` (`date`,`time`)
)
CREATE TABLE `aggregated_weather` (
`Date_id` int NOT NULL AUTO_INCREMENT,
`Date` date DEFAULT NULL,
`T_max` float DEFAULT NULL,
`T_min` float DEFAULT NULL,
`T_mean` float DEFAULT NULL,
`RH_max` float DEFAULT NULL,
`RH_min` float DEFAULT NULL,
`RH_mean` float DEFAULT NULL,
`Rainfall` float DEFAULT NULL,
`ETo` float DEFAULT NULL,
PRIMARY KEY (`Date_id`),
UNIQUE KEY `date_idx` (`Date`)
)
Write Preview
Loading…
Cancel
Save