Browse Source

Collection of data and aggregation to mysql works

master
Micke Nordin 1 year ago
parent
commit
62b65dc2d8
5 changed files with 322 additions and 0 deletions
  1. 172
    0
      aggregateweather.py
  2. 111
    0
      getweather.py
  3. 2
    0
      requirements.txt
  4. 10
    0
      weather.ini
  5. 27
    0
      weather.sql

+ 172
- 0
aggregateweather.py View File

@@ -0,0 +1,172 @@
1
+#!/usr/bin/python3
2
+
3
+from datetime import datetime
4
+from eto import ETo, datasets
5
+from io import StringIO
6
+import argparse
7
+import configparser
8
+import mysql.connector
9
+import pandas as pd
10
+import requests
11
+import sys
12
+
13
+def usage():
14
+    print(  "Usage: " + __file__ + "[--config path_to_file.ini] [--station smhi_station_number ] --db database] [--host dbhost] [--user dbuser] [--password dbpassword]\n"
15
+            "Default configfile is weather.ini, any parameter can be overwritten on the command line")
16
+parser = argparse.ArgumentParser()
17
+parser.add_argument('--config')
18
+parser.add_argument('--db')
19
+parser.add_argument('--host')
20
+parser.add_argument('--user')
21
+parser.add_argument('--password')
22
+parser.add_argument('--station')
23
+args = parser.parse_args()
24
+config_file = "weather.ini"
25
+if args.config:
26
+    config_file = args.config
27
+config = configparser.ConfigParser()
28
+config.read(config_file)
29
+db = config['MySQL']['db']
30
+host = config['MySQL']['host']
31
+user = config['MySQL']['user']
32
+password = config['MySQL']['password']
33
+
34
+station = config['SMHI']['station']
35
+
36
+
37
+# Defaults
38
+z_msl = 48.854
39
+lat = 59.178503
40
+lon = 17.909265
41
+TZ_lon = lon
42
+freq = 'D'
43
+
44
+if args.station:
45
+    station = args.station
46
+if not station:
47
+    station = "97100"
48
+if args.db:
49
+    db = args.db
50
+if args.host:
51
+    host = args.host
52
+if not host:
53
+    host = "localhost"
54
+if args.user:
55
+    user = args.usr
56
+if args.password:
57
+    password = args.password
58
+
59
+if not (db and host and user and password):
60
+    usage()
61
+    sys.exit(1)
62
+
63
+pd.options.mode.chained_assignment = None
64
+station_data = requests.get(url='https://opendata-download-metobs.smhi.se/api/version/1.0/parameter/1/station/{}.json'.format(station)).json()
65
+newest_to = 0
66
+for i in station_data['position']:
67
+    if i['to'] > newest_to:
68
+        z_msl = i['height']
69
+        lat = i['latitude']
70
+        lon = i['longitude']
71
+        TZ_lon = lon
72
+    
73
+
74
+mydb = mysql.connector.connect(
75
+  auth_plugin='mysql_native_password',
76
+  database=db,
77
+  host=host,
78
+  passwd=password,
79
+  user=user
80
+)
81
+
82
+cursor = mydb.cursor();
83
+
84
+date_select = (
85
+    "SELECT DISTINCT `date` "
86
+    "FROM weather "
87
+)
88
+
89
+cursor.execute(date_select)
90
+dates = cursor.fetchall()
91
+csv = "date,T_max,T_min,T_mean,RH_max,RH_min,RH_mean,Rainfall\n"
92
+for i in dates:
93
+    working_date = i[0].strftime('%Y-%m-%d')
94
+    day_select = (
95
+        'SELECT * FROM weather '
96
+        'WHERE date = "{}"'.format(working_date)
97
+    )
98
+    cursor.execute(day_select)
99
+    day = cursor.fetchall()
100
+    sum_rain = 0
101
+    T_max = -9999 
102
+    T_min = 9999
103
+    RH_max = -9999
104
+    RH_min = 9999
105
+    sum_temp = 0
106
+    sum_rel_hum = 0
107
+    sum_windspeed = 0
108
+    counter = 0
109
+    for j in day:
110
+        counter += 1
111
+        # +----------------+---------------+------+-----+---------+----------------+
112
+        #| Field          | Type          | Null | Key | Default | Extra          |
113
+        #+----------------+---------------+------+-----+---------+----------------+
114
+        #| observation_id | int           | NO   | PRI | NULL    | auto_increment |
115
+        #| date           | date          | YES  | MUL | NULL    |                |
116
+        #| time           | time          | YES  |     | NULL    |                |
117
+        #| rainfall       | float         | YES  |     | NULL    |                |
118
+        #| rel_hum        | decimal(10,0) | YES  |     | NULL    |                |
119
+        #| temp           | float         | YES  |     | NULL    |                |
120
+        #| windspeed      | float         | YES  |     | NULL    |                |
121
+        #| winddir        | int           | YES  |     | NULL    |                |
122
+        #+----------------+---------------+------+-----+---------+----------------+
123
+        observation_id = 0
124
+        date = 1
125
+        time = 2
126
+        rainfall = 3
127
+        rel_hum = 4
128
+        temp = 5
129
+        windspeed = 6
130
+        winddir = 7
131
+        sum_rain += j[rainfall]
132
+        sum_temp += j[temp]
133
+        sum_rel_hum += j[rel_hum]
134
+        sum_windspeed += j[windspeed]
135
+        if T_max <  j[temp]:
136
+            T_max = j[temp]
137
+        if T_min >  j[temp]:
138
+            T_min = j[temp]
139
+        if RH_max <  j[rel_hum]:
140
+            RH_max = j[rel_hum]
141
+        if RH_min >  j[rel_hum]:
142
+            RH_min = j[rel_hum]
143
+    T_mean = sum_temp / counter
144
+    RH_mean = sum_rel_hum / counter
145
+    csv += working_date + "," + str(T_max) + "," + str(T_min) + "," + str(T_mean) + "," + str(RH_max) + "," + str(RH_min) +"," + str(RH_mean) + "," + str(sum_rain) + "\n"
146
+DATA = StringIO(csv)
147
+tsdata =  pd.read_csv(DATA, parse_dates=True, infer_datetime_format=True, index_col='date')
148
+et1 = ETo()
149
+et1.param_est(tsdata, freq, z_msl, lat, lon, TZ_lon)
150
+et1.ts_param.head()
151
+eto1 = et1.eto_hargreaves()
152
+
153
+upsert = (
154
+    "REPLACE INTO aggregated_weather "
155
+    "(Date, T_max, T_min, T_mean, RH_max, RH_min, RH_mean, Rainfall, ETo, station) "
156
+    "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
157
+)
158
+
159
+
160
+
161
+for key, value in eto1.items():
162
+    aggdate = key.strftime('%Y-%m-%d')
163
+    data = (
164
+    aggdate,float(tsdata.loc[aggdate, 'T_max']),float(tsdata.loc[aggdate, 'T_min']),float(tsdata.loc[aggdate, 'T_mean']),float(tsdata.loc[aggdate, 'RH_max']),float(tsdata.loc[aggdate, 'RH_min']),float(tsdata.loc[aggdate, 'RH_mean']),float(tsdata.loc[aggdate, 'Rainfall']),float(value),int(station)
165
+    )
166
+    cursor.execute(upsert, data)
167
+
168
+
169
+
170
+mydb.commit()
171
+mydb.close()
172
+

+ 111
- 0
getweather.py View File

@@ -0,0 +1,111 @@
1
+#!/usr/bin/python3
2
+
3
+from datetime import datetime
4
+import argparse
5
+import configparser
6
+import mysql.connector 
7
+import requests
8
+import sys
9
+
10
+def usage():
11
+    print(  "Usage: " + __file__ + "[--config path_to_file.ini] [--station smhi_station_number] [--period latest-hour|latest-day|latest-months] [--db database] [--host dbhost] [--user dbuser] [--password dbpassword]\n"
12
+            "Default configfile is weather.ini, any parameter can be overwritten on the command line")
13
+parser = argparse.ArgumentParser()
14
+parser.add_argument('--config')
15
+parser.add_argument('--station')
16
+parser.add_argument('--period')
17
+parser.add_argument('--db')
18
+parser.add_argument('--host')
19
+parser.add_argument('--user')
20
+parser.add_argument('--password')
21
+args = parser.parse_args()
22
+config_file = "weather.ini"
23
+if args.config:
24
+    config_file = args.config
25
+config = configparser.ConfigParser()
26
+config.read(config_file)
27
+db = config['MySQL']['db']
28
+host = config['MySQL']['host']
29
+user = config['MySQL']['user']
30
+password = config['MySQL']['password']
31
+station = config['SMHI']['station']
32
+period = config['SMHI']['period']
33
+
34
+if args.station:
35
+    station = args.station
36
+if not station:
37
+    station = "97100"
38
+if args.period:
39
+    period = args.period
40
+if not period:
41
+    period = "latest-day"
42
+
43
+if period not in ["latest-hour", "latest-day", "latest-months"]:
44
+    period = "latest-day"
45
+
46
+if args.db:
47
+    db = args.db
48
+if args.host:
49
+    host = args.host
50
+if not host:
51
+    host = "localhost"
52
+if args.user:
53
+    user = args.usr
54
+if args.password:
55
+    password = args.password
56
+
57
+if not (db and host and user and password):
58
+    usage()
59
+    sys.exit(1)
60
+
61
+mydb = mysql.connector.connect(
62
+  auth_plugin='mysql_native_password',
63
+  database=db,
64
+  host=host,
65
+  passwd=password,
66
+  user=user
67
+)
68
+
69
+cursor = mydb.cursor();
70
+
71
+upsert = (
72
+    "REPLACE INTO weather "
73
+    "(date, time, rainfall, rel_hum, temp, winddir, windspeed, station) "
74
+    "VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"
75
+)
76
+# Metrics mapping
77
+rainfall = "7"
78
+rel_hum = "6"
79
+temp = "1"
80
+winddir = "3"
81
+windspeed = "4"
82
+results = {}
83
+for metric in [rainfall,rel_hum,temp,winddir,windspeed]:
84
+    results[metric] = requests.get(url='https://opendata-download-metobs.smhi.se/api/version/1.0/parameter/{}/station/{}/period/{}/data.json'.format(metric,station,period))
85
+
86
+rainfall_arr = sorted(results[rainfall].json()['value'], key = lambda i: i['date'])
87
+rel_hum_arr = sorted(results[rel_hum].json()['value'], key = lambda i: i['date'])
88
+temp_arr = sorted(results[temp].json()['value'], key = lambda i: i['date'])
89
+winddir_arr = sorted(results[winddir].json()['value'], key = lambda i: i['date'])
90
+windspeed_arr = sorted(results[windspeed].json()['value'], key = lambda i: i['date'])
91
+i = 0
92
+for k in rainfall_arr:
93
+    ts = int(k['date'] / 1000)
94
+    date = datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d')
95
+    time = datetime.utcfromtimestamp(ts).strftime('%H:%M:%S')
96
+    rain = k['value']
97
+    hum = rel_hum_arr[i]['value']
98
+    te = temp_arr[i]['value']
99
+    widi = winddir_arr[i]['value']
100
+    wisp = windspeed_arr[i]['value']
101
+
102
+    data = (
103
+        date, time, rain, hum, te, widi, wisp, station
104
+    )
105
+    cursor.execute(upsert, data)
106
+    i = i + 1
107
+
108
+mydb.commit()
109
+mydb.close()
110
+
111
+

+ 2
- 0
requirements.txt View File

@@ -0,0 +1,2 @@
1
+mysql-connector-python
2
+eto

+ 10
- 0
weather.ini View File

@@ -0,0 +1,10 @@
1
+[MySQL]
2
+db = weather
3
+host = localhost
4
+user = your_user_name
5
+password = secret_password
6
+
7
+[SMHI]
8
+station = 97100
9
+period = latest-months
10
+

+ 27
- 0
weather.sql View File

@@ -0,0 +1,27 @@
1
+CREATE TABLE `weather` (
2
+  `observation_id` int NOT NULL AUTO_INCREMENT,
3
+  `date` date DEFAULT NULL,
4
+  `time` time DEFAULT NULL,
5
+  `rainfall` float DEFAULT NULL,
6
+  `rel_hum` decimal(10,0) DEFAULT NULL,
7
+  `temp` float DEFAULT NULL,
8
+  `windspeed` float DEFAULT NULL,
9
+  `winddir` int DEFAULT NULL,
10
+  PRIMARY KEY (`observation_id`),
11
+  UNIQUE KEY `date_time_idx` (`date`,`time`)
12
+)
13
+
14
+CREATE TABLE `aggregated_weather` (
15
+  `Date_id` int NOT NULL AUTO_INCREMENT,
16
+  `Date` date DEFAULT NULL,
17
+  `T_max` float DEFAULT NULL,
18
+  `T_min` float DEFAULT NULL,
19
+  `T_mean` float DEFAULT NULL,
20
+  `RH_max` float DEFAULT NULL,
21
+  `RH_min` float DEFAULT NULL,
22
+  `RH_mean` float DEFAULT NULL,
23
+  `Rainfall` float DEFAULT NULL,
24
+  `ETo` float DEFAULT NULL,
25
+  PRIMARY KEY (`Date_id`),
26
+  UNIQUE KEY `date_idx` (`Date`)
27
+)

Loading…
Cancel
Save