emhass 0.11.2__py3-none-any.whl → 0.11.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- emhass/command_line.py +702 -373
- emhass/data/associations.csv +1 -1
- emhass/forecast.py +671 -346
- emhass/machine_learning_forecaster.py +204 -105
- emhass/machine_learning_regressor.py +26 -7
- emhass/optimization.py +1017 -471
- emhass/retrieve_hass.py +226 -79
- emhass/static/data/param_definitions.json +5 -4
- emhass/utils.py +689 -455
- emhass/web_server.py +339 -225
- {emhass-0.11.2.dist-info → emhass-0.11.3.dist-info}/METADATA +17 -8
- {emhass-0.11.2.dist-info → emhass-0.11.3.dist-info}/RECORD +16 -16
- {emhass-0.11.2.dist-info → emhass-0.11.3.dist-info}/WHEEL +1 -1
- {emhass-0.11.2.dist-info → emhass-0.11.3.dist-info}/LICENSE +0 -0
- {emhass-0.11.2.dist-info → emhass-0.11.3.dist-info}/entry_points.txt +0 -0
- {emhass-0.11.2.dist-info → emhass-0.11.3.dist-info}/top_level.txt +0 -0
emhass/forecast.py
CHANGED
@@ -1,29 +1,29 @@
|
|
1
1
|
#!/usr/bin/env python3
|
2
2
|
# -*- coding: utf-8 -*-
|
3
3
|
|
4
|
-
import
|
5
|
-
import os
|
6
|
-
import pickle
|
4
|
+
import bz2
|
7
5
|
import copy
|
8
|
-
import logging
|
9
6
|
import json
|
10
|
-
|
11
|
-
import
|
7
|
+
import logging
|
8
|
+
import os
|
9
|
+
import pickle
|
12
10
|
import pickle as cPickle
|
13
|
-
import pandas as pd
|
14
|
-
import numpy as np
|
15
11
|
from datetime import datetime, timedelta
|
16
|
-
from
|
17
|
-
|
12
|
+
from typing import Optional
|
13
|
+
|
14
|
+
import numpy as np
|
15
|
+
import pandas as pd
|
18
16
|
import pvlib
|
19
|
-
from
|
17
|
+
from bs4 import BeautifulSoup
|
18
|
+
from pvlib.irradiance import disc
|
20
19
|
from pvlib.location import Location
|
21
20
|
from pvlib.modelchain import ModelChain
|
21
|
+
from pvlib.pvsystem import PVSystem
|
22
22
|
from pvlib.temperature import TEMPERATURE_MODEL_PARAMETERS
|
23
|
-
from
|
23
|
+
from requests import get
|
24
24
|
|
25
|
-
from emhass.retrieve_hass import RetrieveHass
|
26
25
|
from emhass.machine_learning_forecaster import MLForecaster
|
26
|
+
from emhass.retrieve_hass import RetrieveHass
|
27
27
|
from emhass.utils import get_days_list, set_df_index_freq
|
28
28
|
|
29
29
|
|
@@ -98,13 +98,20 @@ class Forecast(object):
|
|
98
98
|
|
99
99
|
"""
|
100
100
|
|
101
|
-
def __init__(
|
102
|
-
|
103
|
-
|
104
|
-
|
101
|
+
def __init__(
|
102
|
+
self,
|
103
|
+
retrieve_hass_conf: dict,
|
104
|
+
optim_conf: dict,
|
105
|
+
plant_conf: dict,
|
106
|
+
params: str,
|
107
|
+
emhass_conf: dict,
|
108
|
+
logger: logging.Logger,
|
109
|
+
opt_time_delta: Optional[int] = 24,
|
110
|
+
get_data_from_file: Optional[bool] = False,
|
111
|
+
) -> None:
|
105
112
|
"""
|
106
113
|
Define constructor for the forecast class.
|
107
|
-
|
114
|
+
|
108
115
|
:param retrieve_hass_conf: Dictionary containing the needed configuration
|
109
116
|
data from the configuration file, specific to retrieve data from HASS
|
110
117
|
:type retrieve_hass_conf: dict
|
@@ -120,10 +127,10 @@ class Forecast(object):
|
|
120
127
|
:type emhass_conf: dict
|
121
128
|
:param logger: The passed logger object
|
122
129
|
:type logger: logging object
|
123
|
-
:param opt_time_delta: The time delta in hours used to generate forecasts,
|
130
|
+
:param opt_time_delta: The time delta in hours used to generate forecasts,
|
124
131
|
a value of 24 will generate 24 hours of forecast data, defaults to 24
|
125
132
|
:type opt_time_delta: int, optional
|
126
|
-
:param get_data_from_file: Select if data should be retrieved from a
|
133
|
+
:param get_data_from_file: Select if data should be retrieved from a
|
127
134
|
previously saved pickle useful for testing or directly from connection to
|
128
135
|
hass database
|
129
136
|
:type get_data_from_file: bool, optional
|
@@ -132,47 +139,71 @@ class Forecast(object):
|
|
132
139
|
self.retrieve_hass_conf = retrieve_hass_conf
|
133
140
|
self.optim_conf = optim_conf
|
134
141
|
self.plant_conf = plant_conf
|
135
|
-
self.freq = self.retrieve_hass_conf[
|
136
|
-
self.time_zone = self.retrieve_hass_conf[
|
137
|
-
self.method_ts_round = self.retrieve_hass_conf[
|
138
|
-
self.timeStep = self.freq.seconds/3600
|
142
|
+
self.freq = self.retrieve_hass_conf["optimization_time_step"]
|
143
|
+
self.time_zone = self.retrieve_hass_conf["time_zone"]
|
144
|
+
self.method_ts_round = self.retrieve_hass_conf["method_ts_round"]
|
145
|
+
self.timeStep = self.freq.seconds / 3600 # in hours
|
139
146
|
self.time_delta = pd.to_timedelta(opt_time_delta, "hours")
|
140
|
-
self.var_PV = self.retrieve_hass_conf[
|
141
|
-
self.var_load = self.retrieve_hass_conf[
|
142
|
-
self.var_load_new = self.var_load+
|
143
|
-
self.lat = self.retrieve_hass_conf[
|
144
|
-
self.lon = self.retrieve_hass_conf[
|
147
|
+
self.var_PV = self.retrieve_hass_conf["sensor_power_photovoltaics"]
|
148
|
+
self.var_load = self.retrieve_hass_conf["sensor_power_load_no_var_loads"]
|
149
|
+
self.var_load_new = self.var_load + "_positive"
|
150
|
+
self.lat = self.retrieve_hass_conf["Latitude"]
|
151
|
+
self.lon = self.retrieve_hass_conf["Longitude"]
|
145
152
|
self.emhass_conf = emhass_conf
|
146
153
|
self.logger = logger
|
147
154
|
self.get_data_from_file = get_data_from_file
|
148
|
-
self.var_load_cost =
|
149
|
-
self.var_prod_price =
|
155
|
+
self.var_load_cost = "unit_load_cost"
|
156
|
+
self.var_prod_price = "unit_prod_price"
|
150
157
|
if (params == None) or (params == "null"):
|
151
158
|
self.params = {}
|
152
159
|
elif type(params) is dict:
|
153
160
|
self.params = params
|
154
161
|
else:
|
155
162
|
self.params = json.loads(params)
|
156
|
-
if self.method_ts_round ==
|
157
|
-
self.start_forecast = pd.Timestamp(
|
158
|
-
|
159
|
-
|
160
|
-
elif self.method_ts_round ==
|
161
|
-
self.start_forecast =
|
163
|
+
if self.method_ts_round == "nearest":
|
164
|
+
self.start_forecast = pd.Timestamp(
|
165
|
+
datetime.now(), tz=self.time_zone
|
166
|
+
).replace(microsecond=0)
|
167
|
+
elif self.method_ts_round == "first":
|
168
|
+
self.start_forecast = (
|
169
|
+
pd.Timestamp(datetime.now(), tz=self.time_zone)
|
170
|
+
.replace(microsecond=0)
|
171
|
+
.floor(freq=self.freq)
|
172
|
+
)
|
173
|
+
elif self.method_ts_round == "last":
|
174
|
+
self.start_forecast = (
|
175
|
+
pd.Timestamp(datetime.now(), tz=self.time_zone)
|
176
|
+
.replace(microsecond=0)
|
177
|
+
.ceil(freq=self.freq)
|
178
|
+
)
|
162
179
|
else:
|
163
180
|
self.logger.error("Wrong method_ts_round passed parameter")
|
164
|
-
self.end_forecast = (
|
165
|
-
|
166
|
-
|
167
|
-
|
181
|
+
self.end_forecast = (
|
182
|
+
self.start_forecast + self.optim_conf["delta_forecast_daily"]
|
183
|
+
).replace(microsecond=0)
|
184
|
+
self.forecast_dates = (
|
185
|
+
pd.date_range(
|
186
|
+
start=self.start_forecast,
|
187
|
+
end=self.end_forecast - self.freq,
|
188
|
+
freq=self.freq,
|
189
|
+
tz=self.time_zone,
|
190
|
+
)
|
191
|
+
.tz_convert("utc")
|
192
|
+
.round(self.freq, ambiguous="infer", nonexistent="shift_forward")
|
193
|
+
.tz_convert(self.time_zone)
|
194
|
+
)
|
168
195
|
if params is not None:
|
169
|
-
if
|
170
|
-
if self.params[
|
171
|
-
self.forecast_dates = self.forecast_dates[
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
196
|
+
if "prediction_horizon" in list(self.params["passed_data"].keys()):
|
197
|
+
if self.params["passed_data"]["prediction_horizon"] is not None:
|
198
|
+
self.forecast_dates = self.forecast_dates[
|
199
|
+
0 : self.params["passed_data"]["prediction_horizon"]
|
200
|
+
]
|
201
|
+
|
202
|
+
def get_weather_forecast(
|
203
|
+
self,
|
204
|
+
method: Optional[str] = "scrapper",
|
205
|
+
csv_path: Optional[str] = "data_weather_forecast.csv",
|
206
|
+
) -> pd.DataFrame:
|
176
207
|
r"""
|
177
208
|
Get and generate weather forecast data.
|
178
209
|
|
@@ -183,230 +214,360 @@ class Forecast(object):
|
|
183
214
|
:rtype: pd.DataFrame
|
184
215
|
|
185
216
|
"""
|
186
|
-
csv_path
|
187
|
-
w_forecast_cache_path = os.path.abspath(
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
217
|
+
csv_path = self.emhass_conf["data_path"] / csv_path
|
218
|
+
w_forecast_cache_path = os.path.abspath(
|
219
|
+
self.emhass_conf["data_path"] / "weather_forecast_data.pkl"
|
220
|
+
)
|
221
|
+
|
222
|
+
self.logger.info("Retrieving weather forecast data using method = " + method)
|
223
|
+
self.weather_forecast_method = (
|
224
|
+
method # Saving this attribute for later use to identify csv method usage
|
225
|
+
)
|
226
|
+
if method == "scrapper":
|
227
|
+
freq_scrap = pd.to_timedelta(
|
228
|
+
60, "minutes"
|
229
|
+
) # The scrapping time step is 60min on clearoutside
|
230
|
+
forecast_dates_scrap = (
|
231
|
+
pd.date_range(
|
232
|
+
start=self.start_forecast,
|
233
|
+
end=self.end_forecast - freq_scrap,
|
234
|
+
freq=freq_scrap,
|
235
|
+
tz=self.time_zone,
|
236
|
+
)
|
237
|
+
.tz_convert("utc")
|
238
|
+
.round(freq_scrap, ambiguous="infer", nonexistent="shift_forward")
|
239
|
+
.tz_convert(self.time_zone)
|
240
|
+
)
|
196
241
|
# Using the clearoutside webpage
|
197
|
-
response = get(
|
198
|
-
|
242
|
+
response = get(
|
243
|
+
"https://clearoutside.com/forecast/"
|
244
|
+
+ str(round(self.lat, 2))
|
245
|
+
+ "/"
|
246
|
+
+ str(round(self.lon, 2))
|
247
|
+
+ "?desktop=true"
|
248
|
+
)
|
249
|
+
"""import bz2 # Uncomment to save a serialized data for tests
|
199
250
|
import _pickle as cPickle
|
200
251
|
with bz2.BZ2File("data/test_response_scrapper_get_method.pbz2", "w") as f:
|
201
|
-
cPickle.dump(response.content, f)
|
202
|
-
soup = BeautifulSoup(response.content,
|
203
|
-
table = soup.find_all(id=
|
204
|
-
list_names = table.find_all(class_=
|
205
|
-
list_tables = table.find_all(
|
206
|
-
selected_cols = [0, 1, 2, 3, 10, 12, 15]
|
252
|
+
cPickle.dump(response.content, f)"""
|
253
|
+
soup = BeautifulSoup(response.content, "html.parser")
|
254
|
+
table = soup.find_all(id="day_0")[0]
|
255
|
+
list_names = table.find_all(class_="fc_detail_label")
|
256
|
+
list_tables = table.find_all("ul")[1:]
|
257
|
+
selected_cols = [0, 1, 2, 3, 10, 12, 15] # Selected variables
|
207
258
|
col_names = [list_names[i].get_text() for i in selected_cols]
|
208
259
|
list_tables = [list_tables[i] for i in selected_cols]
|
209
260
|
# Building the raw DF container
|
210
|
-
raw_data = pd.DataFrame(
|
261
|
+
raw_data = pd.DataFrame(
|
262
|
+
index=range(len(forecast_dates_scrap)), columns=col_names, dtype=float
|
263
|
+
)
|
211
264
|
for count_col, col in enumerate(col_names):
|
212
|
-
list_rows = list_tables[count_col].find_all(
|
265
|
+
list_rows = list_tables[count_col].find_all("li")
|
213
266
|
for count_row, row in enumerate(list_rows):
|
214
267
|
raw_data.loc[count_row, col] = float(row.get_text())
|
215
268
|
# Treating index
|
216
269
|
raw_data.set_index(forecast_dates_scrap, inplace=True)
|
217
|
-
raw_data = raw_data[~raw_data.index.duplicated(keep=
|
270
|
+
raw_data = raw_data[~raw_data.index.duplicated(keep="first")]
|
218
271
|
raw_data = raw_data.reindex(self.forecast_dates)
|
219
|
-
raw_data.interpolate(
|
220
|
-
|
272
|
+
raw_data.interpolate(
|
273
|
+
method="linear",
|
274
|
+
axis=0,
|
275
|
+
limit=None,
|
276
|
+
limit_direction="both",
|
277
|
+
inplace=True,
|
278
|
+
)
|
221
279
|
# Converting the cloud cover into Global Horizontal Irradiance with a PVLib method
|
222
|
-
ghi_est = self.cloud_cover_to_irradiance(
|
280
|
+
ghi_est = self.cloud_cover_to_irradiance(
|
281
|
+
raw_data["Total Clouds (% Sky Obscured)"]
|
282
|
+
)
|
223
283
|
data = ghi_est
|
224
|
-
data[
|
225
|
-
data[
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
284
|
+
data["temp_air"] = raw_data["Temperature (°C)"]
|
285
|
+
data["wind_speed"] = (
|
286
|
+
raw_data["Wind Speed/Direction (mph)"] * 1.60934
|
287
|
+
) # conversion to km/h
|
288
|
+
data["relative_humidity"] = raw_data["Relative Humidity (%)"]
|
289
|
+
data["precipitable_water"] = pvlib.atmosphere.gueymard94_pw(
|
290
|
+
data["temp_air"], data["relative_humidity"]
|
291
|
+
)
|
292
|
+
elif method == "solcast": # using Solcast API
|
230
293
|
# Check if weather_forecast_cache is true or if forecast_data file does not exist
|
231
294
|
if not os.path.isfile(w_forecast_cache_path):
|
232
295
|
# Check if weather_forecast_cache_only is true, if so produce error for not finding cache file
|
233
|
-
if not self.params["passed_data"].get(
|
296
|
+
if not self.params["passed_data"].get(
|
297
|
+
"weather_forecast_cache_only", False
|
298
|
+
):
|
234
299
|
# Retrieve data from the Solcast API
|
235
|
-
if
|
236
|
-
self.logger.error(
|
300
|
+
if "solcast_api_key" not in self.retrieve_hass_conf:
|
301
|
+
self.logger.error(
|
302
|
+
"The solcast_api_key parameter was not defined"
|
303
|
+
)
|
237
304
|
return False
|
238
|
-
if
|
239
|
-
self.logger.error(
|
305
|
+
if "solcast_rooftop_id" not in self.retrieve_hass_conf:
|
306
|
+
self.logger.error(
|
307
|
+
"The solcast_rooftop_id parameter was not defined"
|
308
|
+
)
|
240
309
|
return False
|
241
310
|
headers = {
|
242
|
-
|
243
|
-
"Authorization": "Bearer "
|
311
|
+
"User-Agent": "EMHASS",
|
312
|
+
"Authorization": "Bearer "
|
313
|
+
+ self.retrieve_hass_conf["solcast_api_key"],
|
244
314
|
"content-type": "application/json",
|
245
|
-
|
246
|
-
days_solcast = int(
|
247
|
-
|
248
|
-
|
315
|
+
}
|
316
|
+
days_solcast = int(
|
317
|
+
len(self.forecast_dates) * self.freq.seconds / 3600
|
318
|
+
)
|
319
|
+
# If weather_forecast_cache, set request days as twice as long to avoid length issues (add a buffer)
|
320
|
+
if self.params["passed_data"].get("weather_forecast_cache", False):
|
249
321
|
days_solcast = min((days_solcast * 2), 336)
|
250
|
-
url =
|
322
|
+
url = (
|
323
|
+
"https://api.solcast.com.au/rooftop_sites/"
|
324
|
+
+ self.retrieve_hass_conf["solcast_rooftop_id"]
|
325
|
+
+ "/forecasts?hours="
|
326
|
+
+ str(days_solcast)
|
327
|
+
)
|
251
328
|
response = get(url, headers=headers)
|
252
|
-
|
329
|
+
"""import bz2 # Uncomment to save a serialized data for tests
|
253
330
|
import _pickle as cPickle
|
254
331
|
with bz2.BZ2File("data/test_response_solcast_get_method.pbz2", "w") as f:
|
255
|
-
cPickle.dump(response, f)
|
332
|
+
cPickle.dump(response, f)"""
|
256
333
|
# Verify the request passed
|
257
334
|
if int(response.status_code) == 200:
|
258
335
|
data = response.json()
|
259
|
-
elif
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
self.logger.error(
|
264
|
-
|
265
|
-
|
336
|
+
elif (
|
337
|
+
int(response.status_code) == 402
|
338
|
+
or int(response.status_code) == 429
|
339
|
+
):
|
340
|
+
self.logger.error(
|
341
|
+
"Solcast error: May have exceeded your subscription limit."
|
342
|
+
)
|
343
|
+
return False
|
344
|
+
elif (
|
345
|
+
int(response.status_code) >= 400
|
346
|
+
or int(response.status_code) >= 202
|
347
|
+
):
|
348
|
+
self.logger.error(
|
349
|
+
"Solcast error: There was a issue with the solcast request, check solcast API key and rooftop ID."
|
350
|
+
)
|
351
|
+
self.logger.error(
|
352
|
+
"Solcast error: Check that your subscription is valid and your network can connect to Solcast."
|
353
|
+
)
|
354
|
+
return False
|
266
355
|
data_list = []
|
267
|
-
for elm in data[
|
268
|
-
data_list.append(
|
356
|
+
for elm in data["forecasts"]:
|
357
|
+
data_list.append(
|
358
|
+
elm["pv_estimate"] * 1000
|
359
|
+
) # Converting kW to W
|
269
360
|
# Check if the retrieved data has the correct length
|
270
361
|
if len(data_list) < len(self.forecast_dates):
|
271
|
-
self.logger.error(
|
362
|
+
self.logger.error(
|
363
|
+
"Not enough data retried from Solcast service, try increasing the time step or use MPC."
|
364
|
+
)
|
272
365
|
else:
|
273
366
|
# If runtime weather_forecast_cache is true save forecast result to file as cache
|
274
|
-
if self.params["passed_data"].get(
|
367
|
+
if self.params["passed_data"].get(
|
368
|
+
"weather_forecast_cache", False
|
369
|
+
):
|
275
370
|
# Add x2 forecast periods for cached results. This adds a extra delta_forecast amount of days for a buffer
|
276
|
-
cached_forecast_dates =
|
277
|
-
|
278
|
-
|
371
|
+
cached_forecast_dates = self.forecast_dates.union(
|
372
|
+
pd.date_range(
|
373
|
+
self.forecast_dates[-1],
|
374
|
+
periods=(len(self.forecast_dates) + 1),
|
375
|
+
freq=self.freq,
|
376
|
+
)[1:]
|
377
|
+
)
|
378
|
+
cache_data_list = data_list[0 : len(cached_forecast_dates)]
|
379
|
+
cache_data_dict = {
|
380
|
+
"ts": cached_forecast_dates,
|
381
|
+
"yhat": cache_data_list,
|
382
|
+
}
|
279
383
|
data_cache = pd.DataFrame.from_dict(cache_data_dict)
|
280
|
-
data_cache.set_index(
|
281
|
-
with open(w_forecast_cache_path, "wb") as file:
|
384
|
+
data_cache.set_index("ts", inplace=True)
|
385
|
+
with open(w_forecast_cache_path, "wb") as file:
|
282
386
|
cPickle.dump(data_cache, file)
|
283
387
|
if not os.path.isfile(w_forecast_cache_path):
|
284
|
-
self.logger.warning(
|
388
|
+
self.logger.warning(
|
389
|
+
"Solcast forecast data could not be saved to file."
|
390
|
+
)
|
285
391
|
else:
|
286
|
-
self.logger.info(
|
287
|
-
|
288
|
-
|
289
|
-
|
392
|
+
self.logger.info(
|
393
|
+
"Saved the Solcast results to cache, for later reference."
|
394
|
+
)
|
395
|
+
# Trim request results to forecast_dates
|
396
|
+
data_list = data_list[0 : len(self.forecast_dates)]
|
397
|
+
data_dict = {"ts": self.forecast_dates, "yhat": data_list}
|
290
398
|
# Define DataFrame
|
291
399
|
data = pd.DataFrame.from_dict(data_dict)
|
292
400
|
# Define index
|
293
|
-
data.set_index(
|
401
|
+
data.set_index("ts", inplace=True)
|
294
402
|
# Else, notify user to update cache
|
295
403
|
else:
|
296
404
|
self.logger.error("Unable to obtain Solcast cache file.")
|
297
|
-
self.logger.error(
|
298
|
-
|
405
|
+
self.logger.error(
|
406
|
+
"Try running optimization again with 'weather_forecast_cache_only': false"
|
407
|
+
)
|
408
|
+
self.logger.error(
|
409
|
+
"Optionally, obtain new Solcast cache with runtime parameter 'weather_forecast_cache': true in an optimization, or run the `weather-forecast-cache` action, to pull new data from Solcast and cache."
|
410
|
+
)
|
299
411
|
return False
|
300
412
|
# Else, open stored weather_forecast_data.pkl file for previous forecast data (cached data)
|
301
413
|
else:
|
302
414
|
with open(w_forecast_cache_path, "rb") as file:
|
303
415
|
data = cPickle.load(file)
|
304
|
-
if not isinstance(data, pd.DataFrame) or len(data) < len(
|
305
|
-
self.
|
306
|
-
|
307
|
-
self.logger.
|
416
|
+
if not isinstance(data, pd.DataFrame) or len(data) < len(
|
417
|
+
self.forecast_dates
|
418
|
+
):
|
419
|
+
self.logger.error(
|
420
|
+
"There has been a error obtaining cached Solcast forecast data."
|
421
|
+
)
|
422
|
+
self.logger.error(
|
423
|
+
"Try running optimization again with 'weather_forecast_cache': true, or run action `weather-forecast-cache`, to pull new data from Solcast and cache."
|
424
|
+
)
|
425
|
+
self.logger.warning(
|
426
|
+
"Removing old Solcast cache file. Next optimization will pull data from Solcast, unless 'weather_forecast_cache_only': true"
|
427
|
+
)
|
308
428
|
os.remove(w_forecast_cache_path)
|
309
429
|
return False
|
310
430
|
# Filter cached forecast data to match current forecast_dates start-end range (reduce forecast Dataframe size to appropriate length)
|
311
|
-
if
|
312
|
-
|
313
|
-
self.
|
431
|
+
if (
|
432
|
+
self.forecast_dates[0] in data.index
|
433
|
+
and self.forecast_dates[-1] in data.index
|
434
|
+
):
|
435
|
+
data = data.loc[
|
436
|
+
self.forecast_dates[0] : self.forecast_dates[-1]
|
437
|
+
]
|
438
|
+
self.logger.info(
|
439
|
+
"Retrieved Solcast data from the previously saved cache."
|
440
|
+
)
|
314
441
|
else:
|
315
|
-
self.logger.error(
|
316
|
-
|
317
|
-
|
442
|
+
self.logger.error(
|
443
|
+
"Unable to obtain cached Solcast forecast data within the requested timeframe range."
|
444
|
+
)
|
445
|
+
self.logger.error(
|
446
|
+
"Try running optimization again (not using cache). Optionally, add runtime parameter 'weather_forecast_cache': true to pull new data from Solcast and cache."
|
447
|
+
)
|
448
|
+
self.logger.warning(
|
449
|
+
"Removing old Solcast cache file. Next optimization will pull data from Solcast, unless 'weather_forecast_cache_only': true"
|
450
|
+
)
|
318
451
|
os.remove(w_forecast_cache_path)
|
319
|
-
return False
|
320
|
-
elif method ==
|
452
|
+
return False
|
453
|
+
elif method == "solar.forecast": # using the solar.forecast API
|
321
454
|
# Retrieve data from the solar.forecast API
|
322
|
-
if
|
323
|
-
self.logger.warning(
|
324
|
-
|
325
|
-
|
326
|
-
self.
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
self.
|
332
|
-
|
333
|
-
|
334
|
-
|
455
|
+
if "solar_forecast_kwp" not in self.retrieve_hass_conf:
|
456
|
+
self.logger.warning(
|
457
|
+
"The solar_forecast_kwp parameter was not defined, using dummy values for testing"
|
458
|
+
)
|
459
|
+
self.retrieve_hass_conf["solar_forecast_kwp"] = 5
|
460
|
+
if self.retrieve_hass_conf["solar_forecast_kwp"] == 0:
|
461
|
+
self.logger.warning(
|
462
|
+
"The solar_forecast_kwp parameter is set to zero, setting to default 5"
|
463
|
+
)
|
464
|
+
self.retrieve_hass_conf["solar_forecast_kwp"] = 5
|
465
|
+
if self.optim_conf["delta_forecast_daily"].days > 1:
|
466
|
+
self.logger.warning(
|
467
|
+
"The free public tier for solar.forecast only provides one day forecasts"
|
468
|
+
)
|
469
|
+
self.logger.warning(
|
470
|
+
"Continuing with just the first day of data, the other days are filled with 0.0."
|
471
|
+
)
|
472
|
+
self.logger.warning(
|
473
|
+
"Use the other available methods for delta_forecast_daily > 1"
|
474
|
+
)
|
475
|
+
headers = {"Accept": "application/json"}
|
335
476
|
data = pd.DataFrame()
|
336
|
-
for i in range(len(self.plant_conf[
|
337
|
-
url =
|
338
|
-
"
|
339
|
-
|
477
|
+
for i in range(len(self.plant_conf["pv_module_model"])):
|
478
|
+
url = (
|
479
|
+
"https://api.forecast.solar/estimate/"
|
480
|
+
+ str(round(self.lat, 2))
|
481
|
+
+ "/"
|
482
|
+
+ str(round(self.lon, 2))
|
483
|
+
+ "/"
|
484
|
+
+ str(self.plant_conf["surface_tilt"][i])
|
485
|
+
+ "/"
|
486
|
+
+ str(self.plant_conf["surface_azimuth"][i] - 180)
|
487
|
+
+ "/"
|
488
|
+
+ str(self.retrieve_hass_conf["solar_forecast_kwp"])
|
489
|
+
)
|
340
490
|
response = get(url, headers=headers)
|
341
|
-
|
491
|
+
"""import bz2 # Uncomment to save a serialized data for tests
|
342
492
|
import _pickle as cPickle
|
343
493
|
with bz2.BZ2File("data/test_response_solarforecast_get_method.pbz2", "w") as f:
|
344
|
-
cPickle.dump(response.json(), f)
|
494
|
+
cPickle.dump(response.json(), f)"""
|
345
495
|
data_raw = response.json()
|
346
|
-
data_dict = {
|
496
|
+
data_dict = {
|
497
|
+
"ts": list(data_raw["result"]["watts"].keys()),
|
498
|
+
"yhat": list(data_raw["result"]["watts"].values()),
|
499
|
+
}
|
347
500
|
# Form the final DataFrame
|
348
501
|
data_tmp = pd.DataFrame.from_dict(data_dict)
|
349
|
-
data_tmp.set_index(
|
502
|
+
data_tmp.set_index("ts", inplace=True)
|
350
503
|
data_tmp.index = pd.to_datetime(data_tmp.index)
|
351
504
|
data_tmp = data_tmp.tz_localize(self.forecast_dates.tz)
|
352
505
|
data_tmp = data_tmp.reindex(index=self.forecast_dates)
|
353
|
-
mask_up_data_df =
|
354
|
-
|
355
|
-
|
356
|
-
|
506
|
+
mask_up_data_df = (
|
507
|
+
data_tmp.copy(deep=True).fillna(method="ffill").isnull()
|
508
|
+
)
|
509
|
+
mask_down_data_df = (
|
510
|
+
data_tmp.copy(deep=True).fillna(method="bfill").isnull()
|
511
|
+
)
|
512
|
+
data_tmp.loc[data_tmp.index[mask_up_data_df["yhat"] == True], :] = 0.0
|
513
|
+
data_tmp.loc[data_tmp.index[mask_down_data_df["yhat"] == True], :] = 0.0
|
357
514
|
data_tmp.interpolate(inplace=True, limit=1)
|
358
515
|
data_tmp = data_tmp.fillna(0.0)
|
359
516
|
if len(data) == 0:
|
360
517
|
data = copy.deepcopy(data_tmp)
|
361
518
|
else:
|
362
519
|
data = data + data_tmp
|
363
|
-
elif method ==
|
520
|
+
elif method == "csv": # reading from a csv file
|
364
521
|
weather_csv_file_path = csv_path
|
365
522
|
# Loading the csv file, we will consider that this is the PV power in W
|
366
|
-
data = pd.read_csv(weather_csv_file_path, header=None, names=[
|
367
|
-
# Check if the passed data has the correct length
|
523
|
+
data = pd.read_csv(weather_csv_file_path, header=None, names=["ts", "yhat"])
|
524
|
+
# Check if the passed data has the correct length
|
368
525
|
if len(data) < len(self.forecast_dates):
|
369
526
|
self.logger.error("Passed data from CSV is not long enough")
|
370
527
|
else:
|
371
528
|
# Ensure correct length
|
372
|
-
data = data.loc[data.index[0:len(self.forecast_dates)]
|
529
|
+
data = data.loc[data.index[0 : len(self.forecast_dates)], :]
|
373
530
|
# Define index
|
374
531
|
data.index = self.forecast_dates
|
375
|
-
data.drop(
|
532
|
+
data.drop("ts", axis=1, inplace=True)
|
376
533
|
data = data.copy().loc[self.forecast_dates]
|
377
|
-
elif method ==
|
534
|
+
elif method == "list": # reading a list of values
|
378
535
|
# Loading data from passed list
|
379
|
-
data_list = self.params[
|
536
|
+
data_list = self.params["passed_data"]["pv_power_forecast"]
|
380
537
|
# Check if the passed data has the correct length
|
381
|
-
if
|
538
|
+
if (
|
539
|
+
len(data_list) < len(self.forecast_dates)
|
540
|
+
and self.params["passed_data"]["prediction_horizon"] is None
|
541
|
+
):
|
382
542
|
self.logger.error("Passed data from passed list is not long enough")
|
383
543
|
else:
|
384
544
|
# Ensure correct length
|
385
|
-
data_list = data_list[0:len(self.forecast_dates)]
|
545
|
+
data_list = data_list[0 : len(self.forecast_dates)]
|
386
546
|
# Define DataFrame
|
387
|
-
data_dict = {
|
547
|
+
data_dict = {"ts": self.forecast_dates, "yhat": data_list}
|
388
548
|
data = pd.DataFrame.from_dict(data_dict)
|
389
549
|
# Define index
|
390
|
-
data.set_index(
|
550
|
+
data.set_index("ts", inplace=True)
|
391
551
|
else:
|
392
552
|
self.logger.error("Method %r is not valid", method)
|
393
553
|
data = None
|
394
554
|
return data
|
395
|
-
|
396
|
-
def cloud_cover_to_irradiance(
|
397
|
-
|
555
|
+
|
556
|
+
def cloud_cover_to_irradiance(
|
557
|
+
self, cloud_cover: pd.Series, offset: Optional[int] = 35
|
558
|
+
) -> pd.DataFrame:
|
398
559
|
"""
|
399
560
|
Estimates irradiance from cloud cover in the following steps.
|
400
|
-
|
561
|
+
|
401
562
|
1. Determine clear sky GHI using Ineichen model and
|
402
563
|
climatological turbidity.
|
403
|
-
|
564
|
+
|
404
565
|
2. Estimate cloudy sky GHI using a function of cloud_cover
|
405
|
-
|
566
|
+
|
406
567
|
3. Estimate cloudy sky DNI using the DISC model.
|
407
|
-
|
568
|
+
|
408
569
|
4. Calculate DHI from DNI and GHI.
|
409
|
-
|
570
|
+
|
410
571
|
(This function was copied and modified from PVLib)
|
411
572
|
|
412
573
|
:param cloud_cover: Cloud cover in %.
|
@@ -418,21 +579,27 @@ class Forecast(object):
|
|
418
579
|
"""
|
419
580
|
location = Location(latitude=self.lat, longitude=self.lon)
|
420
581
|
solpos = location.get_solarposition(cloud_cover.index)
|
421
|
-
cs = location.get_clearsky(
|
422
|
-
|
582
|
+
cs = location.get_clearsky(
|
583
|
+
cloud_cover.index, model="ineichen", solar_position=solpos
|
584
|
+
)
|
423
585
|
# Using only the linear method
|
424
|
-
offset = offset / 100.
|
425
|
-
cloud_cover_unit = copy.deepcopy(cloud_cover) / 100.
|
426
|
-
ghi = (offset + (1 - offset) * (1 - cloud_cover_unit)) * cs[
|
586
|
+
offset = offset / 100.0
|
587
|
+
cloud_cover_unit = copy.deepcopy(cloud_cover) / 100.0
|
588
|
+
ghi = (offset + (1 - offset) * (1 - cloud_cover_unit)) * cs["ghi"]
|
427
589
|
# Using disc model
|
428
|
-
dni = disc(ghi, solpos[
|
429
|
-
dhi = ghi - dni * np.cos(np.radians(solpos[
|
430
|
-
irrads = pd.DataFrame({
|
590
|
+
dni = disc(ghi, solpos["zenith"], cloud_cover.index)["dni"]
|
591
|
+
dhi = ghi - dni * np.cos(np.radians(solpos["zenith"]))
|
592
|
+
irrads = pd.DataFrame({"ghi": ghi, "dni": dni, "dhi": dhi}).fillna(0)
|
431
593
|
return irrads
|
432
|
-
|
594
|
+
|
433
595
|
@staticmethod
|
434
|
-
def get_mix_forecast(
|
435
|
-
|
596
|
+
def get_mix_forecast(
|
597
|
+
df_now: pd.DataFrame,
|
598
|
+
df_forecast: pd.DataFrame,
|
599
|
+
alpha: float,
|
600
|
+
beta: float,
|
601
|
+
col: str,
|
602
|
+
) -> pd.DataFrame:
|
436
603
|
"""A simple correction method for forecasted data using the current real values of a variable.
|
437
604
|
|
438
605
|
:param df_now: The DataFrame containing the current/real values
|
@@ -448,13 +615,16 @@ class Forecast(object):
|
|
448
615
|
:return: The output DataFrame with the corrected values
|
449
616
|
:rtype: pd.DataFrame
|
450
617
|
"""
|
451
|
-
first_fcst = alpha*df_forecast.iloc[0] + beta*df_now[col].iloc[-1]
|
618
|
+
first_fcst = alpha * df_forecast.iloc[0] + beta * df_now[col].iloc[-1]
|
452
619
|
df_forecast.iloc[0] = first_fcst
|
453
620
|
return df_forecast
|
454
|
-
|
455
|
-
def get_power_from_weather(
|
456
|
-
|
457
|
-
|
621
|
+
|
622
|
+
def get_power_from_weather(
|
623
|
+
self,
|
624
|
+
df_weather: pd.DataFrame,
|
625
|
+
set_mix_forecast: Optional[bool] = False,
|
626
|
+
df_now: Optional[pd.DataFrame] = pd.DataFrame(),
|
627
|
+
) -> pd.Series:
|
458
628
|
r"""
|
459
629
|
Convert wheater forecast data into electrical power.
|
460
630
|
|
@@ -471,36 +641,55 @@ class Forecast(object):
|
|
471
641
|
|
472
642
|
"""
|
473
643
|
# If using csv method we consider that yhat is the PV power in W
|
474
|
-
if
|
644
|
+
if (
|
645
|
+
"solar_forecast_kwp" in self.retrieve_hass_conf.keys()
|
646
|
+
and self.retrieve_hass_conf["solar_forecast_kwp"] == 0
|
647
|
+
):
|
475
648
|
P_PV_forecast = pd.Series(0, index=df_weather.index)
|
476
649
|
else:
|
477
|
-
if
|
478
|
-
self.weather_forecast_method ==
|
479
|
-
|
650
|
+
if (
|
651
|
+
self.weather_forecast_method == "solcast"
|
652
|
+
or self.weather_forecast_method == "solar.forecast"
|
653
|
+
or self.weather_forecast_method == "csv"
|
654
|
+
or self.weather_forecast_method == "list"
|
655
|
+
):
|
656
|
+
P_PV_forecast = df_weather["yhat"]
|
480
657
|
P_PV_forecast.name = None
|
481
|
-
else:
|
658
|
+
else: # We will transform the weather data into electrical power
|
482
659
|
# Transform to power (Watts)
|
483
660
|
# Setting the main parameters of the PV plant
|
484
661
|
location = Location(latitude=self.lat, longitude=self.lon)
|
485
|
-
temp_params = TEMPERATURE_MODEL_PARAMETERS[
|
486
|
-
|
662
|
+
temp_params = TEMPERATURE_MODEL_PARAMETERS["sapm"][
|
663
|
+
"close_mount_glass_glass"
|
664
|
+
]
|
665
|
+
cec_modules = bz2.BZ2File(
|
666
|
+
self.emhass_conf["root_path"] / "data" / "cec_modules.pbz2", "rb"
|
667
|
+
)
|
487
668
|
cec_modules = cPickle.load(cec_modules)
|
488
|
-
cec_inverters = bz2.BZ2File(
|
669
|
+
cec_inverters = bz2.BZ2File(
|
670
|
+
self.emhass_conf["root_path"] / "data" / "cec_inverters.pbz2", "rb"
|
671
|
+
)
|
489
672
|
cec_inverters = cPickle.load(cec_inverters)
|
490
|
-
if type(self.plant_conf[
|
673
|
+
if type(self.plant_conf["pv_module_model"]) == list:
|
491
674
|
P_PV_forecast = pd.Series(0, index=df_weather.index)
|
492
|
-
for i in range(len(self.plant_conf[
|
675
|
+
for i in range(len(self.plant_conf["pv_module_model"])):
|
493
676
|
# Selecting correct module and inverter
|
494
|
-
module = cec_modules[self.plant_conf[
|
495
|
-
inverter = cec_inverters[
|
677
|
+
module = cec_modules[self.plant_conf["pv_module_model"][i]]
|
678
|
+
inverter = cec_inverters[
|
679
|
+
self.plant_conf["pv_inverter_model"][i]
|
680
|
+
]
|
496
681
|
# Building the PV system in PVLib
|
497
|
-
system = PVSystem(
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
682
|
+
system = PVSystem(
|
683
|
+
surface_tilt=self.plant_conf["surface_tilt"][i],
|
684
|
+
surface_azimuth=self.plant_conf["surface_azimuth"][i],
|
685
|
+
module_parameters=module,
|
686
|
+
inverter_parameters=inverter,
|
687
|
+
temperature_model_parameters=temp_params,
|
688
|
+
modules_per_string=self.plant_conf["modules_per_string"][i],
|
689
|
+
strings_per_inverter=self.plant_conf[
|
690
|
+
"strings_per_inverter"
|
691
|
+
][i],
|
692
|
+
)
|
504
693
|
mc = ModelChain(system, location, aoi_model="physical")
|
505
694
|
# Run the model on the weather DF indexes
|
506
695
|
mc.run_model(df_weather)
|
@@ -508,16 +697,18 @@ class Forecast(object):
|
|
508
697
|
P_PV_forecast = P_PV_forecast + mc.results.ac
|
509
698
|
else:
|
510
699
|
# Selecting correct module and inverter
|
511
|
-
module = cec_modules[self.plant_conf[
|
512
|
-
inverter = cec_inverters[self.plant_conf[
|
700
|
+
module = cec_modules[self.plant_conf["pv_module_model"]]
|
701
|
+
inverter = cec_inverters[self.plant_conf["pv_inverter_model"]]
|
513
702
|
# Building the PV system in PVLib
|
514
|
-
system = PVSystem(
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
703
|
+
system = PVSystem(
|
704
|
+
surface_tilt=self.plant_conf["surface_tilt"],
|
705
|
+
surface_azimuth=self.plant_conf["surface_azimuth"],
|
706
|
+
module_parameters=module,
|
707
|
+
inverter_parameters=inverter,
|
708
|
+
temperature_model_parameters=temp_params,
|
709
|
+
modules_per_string=self.plant_conf["modules_per_string"],
|
710
|
+
strings_per_inverter=self.plant_conf["strings_per_inverter"],
|
711
|
+
)
|
521
712
|
mc = ModelChain(system, location, aoi_model="physical")
|
522
713
|
# Run the model on the weather DF indexes
|
523
714
|
mc.run_model(df_weather)
|
@@ -525,47 +716,80 @@ class Forecast(object):
|
|
525
716
|
P_PV_forecast = mc.results.ac
|
526
717
|
if set_mix_forecast:
|
527
718
|
P_PV_forecast = Forecast.get_mix_forecast(
|
528
|
-
df_now,
|
529
|
-
|
719
|
+
df_now,
|
720
|
+
P_PV_forecast,
|
721
|
+
self.params["passed_data"]["alpha"],
|
722
|
+
self.params["passed_data"]["beta"],
|
723
|
+
self.var_PV,
|
724
|
+
)
|
530
725
|
return P_PV_forecast
|
531
|
-
|
726
|
+
|
532
727
|
def get_forecast_days_csv(self, timedelta_days: Optional[int] = 1) -> pd.date_range:
|
533
728
|
r"""
|
534
729
|
Get the date range vector of forecast dates that will be used when loading a CSV file.
|
535
|
-
|
730
|
+
|
536
731
|
:return: The forecast dates vector
|
537
732
|
:rtype: pd.date_range
|
538
733
|
|
539
734
|
"""
|
540
|
-
start_forecast_csv = pd.Timestamp(datetime.now(), tz=self.time_zone).replace(
|
541
|
-
|
542
|
-
|
543
|
-
|
544
|
-
start_forecast_csv = pd.Timestamp(
|
545
|
-
|
546
|
-
|
735
|
+
start_forecast_csv = pd.Timestamp(datetime.now(), tz=self.time_zone).replace(
|
736
|
+
microsecond=0
|
737
|
+
)
|
738
|
+
if self.method_ts_round == "nearest":
|
739
|
+
start_forecast_csv = pd.Timestamp(
|
740
|
+
datetime.now(), tz=self.time_zone
|
741
|
+
).replace(microsecond=0)
|
742
|
+
elif self.method_ts_round == "first":
|
743
|
+
start_forecast_csv = (
|
744
|
+
pd.Timestamp(datetime.now(), tz=self.time_zone)
|
745
|
+
.replace(microsecond=0)
|
746
|
+
.floor(freq=self.freq)
|
747
|
+
)
|
748
|
+
elif self.method_ts_round == "last":
|
749
|
+
start_forecast_csv = (
|
750
|
+
pd.Timestamp(datetime.now(), tz=self.time_zone)
|
751
|
+
.replace(microsecond=0)
|
752
|
+
.ceil(freq=self.freq)
|
753
|
+
)
|
547
754
|
else:
|
548
755
|
self.logger.error("Wrong method_ts_round passed parameter")
|
549
|
-
end_forecast_csv = (
|
550
|
-
|
551
|
-
|
552
|
-
|
756
|
+
end_forecast_csv = (
|
757
|
+
start_forecast_csv + self.optim_conf["delta_forecast_daily"]
|
758
|
+
).replace(microsecond=0)
|
759
|
+
forecast_dates_csv = (
|
760
|
+
pd.date_range(
|
761
|
+
start=start_forecast_csv,
|
762
|
+
end=end_forecast_csv + timedelta(days=timedelta_days) - self.freq,
|
763
|
+
freq=self.freq,
|
764
|
+
tz=self.time_zone,
|
765
|
+
)
|
766
|
+
.tz_convert("utc")
|
767
|
+
.round(self.freq, ambiguous="infer", nonexistent="shift_forward")
|
768
|
+
.tz_convert(self.time_zone)
|
769
|
+
)
|
553
770
|
if self.params is not None:
|
554
|
-
if
|
555
|
-
if self.params[
|
556
|
-
forecast_dates_csv = forecast_dates_csv[
|
771
|
+
if "prediction_horizon" in list(self.params["passed_data"].keys()):
|
772
|
+
if self.params["passed_data"]["prediction_horizon"] is not None:
|
773
|
+
forecast_dates_csv = forecast_dates_csv[
|
774
|
+
0 : self.params["passed_data"]["prediction_horizon"]
|
775
|
+
]
|
557
776
|
return forecast_dates_csv
|
558
|
-
|
559
|
-
def get_forecast_out_from_csv_or_list(
|
560
|
-
|
561
|
-
|
777
|
+
|
778
|
+
def get_forecast_out_from_csv_or_list(
|
779
|
+
self,
|
780
|
+
df_final: pd.DataFrame,
|
781
|
+
forecast_dates_csv: pd.date_range,
|
782
|
+
csv_path: str,
|
783
|
+
data_list: Optional[list] = None,
|
784
|
+
list_and_perfect: Optional[bool] = False,
|
785
|
+
) -> pd.DataFrame:
|
562
786
|
r"""
|
563
|
-
Get the forecast data as a DataFrame from a CSV file.
|
564
|
-
|
565
|
-
The data contained in the CSV file should be a 24h forecast with the same frequency as
|
566
|
-
the main 'optimization_time_step' parameter in the configuration file. The timestamp will not be used and
|
787
|
+
Get the forecast data as a DataFrame from a CSV file.
|
788
|
+
|
789
|
+
The data contained in the CSV file should be a 24h forecast with the same frequency as
|
790
|
+
the main 'optimization_time_step' parameter in the configuration file. The timestamp will not be used and
|
567
791
|
a new DateTimeIndex is generated to fit the timestamp index of the input data in 'df_final'.
|
568
|
-
|
792
|
+
|
569
793
|
:param df_final: The DataFrame containing the input data.
|
570
794
|
:type df_final: pd.DataFrame
|
571
795
|
:param forecast_dates_csv: The forecast dates vector
|
@@ -577,10 +801,10 @@ class Forecast(object):
|
|
577
801
|
|
578
802
|
"""
|
579
803
|
if csv_path is None:
|
580
|
-
data_dict = {
|
804
|
+
data_dict = {"ts": forecast_dates_csv, "yhat": data_list}
|
581
805
|
df_csv = pd.DataFrame.from_dict(data_dict)
|
582
806
|
df_csv.index = forecast_dates_csv
|
583
|
-
df_csv.drop([
|
807
|
+
df_csv.drop(["ts"], axis=1, inplace=True)
|
584
808
|
df_csv = set_df_index_freq(df_csv)
|
585
809
|
if list_and_perfect:
|
586
810
|
days_list = df_final.index.day.unique().tolist()
|
@@ -588,11 +812,11 @@ class Forecast(object):
|
|
588
812
|
days_list = df_csv.index.day.unique().tolist()
|
589
813
|
else:
|
590
814
|
if not os.path.exists(csv_path):
|
591
|
-
csv_path = self.emhass_conf[
|
592
|
-
load_csv_file_path = csv_path
|
593
|
-
df_csv = pd.read_csv(load_csv_file_path, header=None, names=[
|
815
|
+
csv_path = self.emhass_conf["data_path"] / csv_path
|
816
|
+
load_csv_file_path = csv_path
|
817
|
+
df_csv = pd.read_csv(load_csv_file_path, header=None, names=["ts", "yhat"])
|
594
818
|
df_csv.index = forecast_dates_csv
|
595
|
-
df_csv.drop([
|
819
|
+
df_csv.drop(["ts"], axis=1, inplace=True)
|
596
820
|
df_csv = set_df_index_freq(df_csv)
|
597
821
|
days_list = df_final.index.day.unique().tolist()
|
598
822
|
forecast_out = pd.DataFrame()
|
@@ -606,47 +830,73 @@ class Forecast(object):
|
|
606
830
|
df_tmp = copy.deepcopy(df_final)
|
607
831
|
first_elm_index = [i for i, x in enumerate(df_tmp.index.day == day) if x][0]
|
608
832
|
last_elm_index = [i for i, x in enumerate(df_tmp.index.day == day) if x][-1]
|
609
|
-
fcst_index = pd.date_range(
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
833
|
+
fcst_index = pd.date_range(
|
834
|
+
start=df_tmp.index[first_elm_index],
|
835
|
+
end=df_tmp.index[last_elm_index],
|
836
|
+
freq=df_tmp.index.freq,
|
837
|
+
)
|
838
|
+
first_hour = (
|
839
|
+
str(df_tmp.index[first_elm_index].hour)
|
840
|
+
+ ":"
|
841
|
+
+ str(df_tmp.index[first_elm_index].minute)
|
842
|
+
)
|
843
|
+
last_hour = (
|
844
|
+
str(df_tmp.index[last_elm_index].hour)
|
845
|
+
+ ":"
|
846
|
+
+ str(df_tmp.index[last_elm_index].minute)
|
847
|
+
)
|
614
848
|
if len(forecast_out) == 0:
|
615
849
|
if csv_path is None:
|
616
850
|
if list_and_perfect:
|
617
851
|
forecast_out = pd.DataFrame(
|
618
852
|
df_csv.between_time(first_hour, last_hour).values,
|
619
|
-
index=fcst_index
|
853
|
+
index=fcst_index,
|
854
|
+
)
|
620
855
|
else:
|
621
856
|
forecast_out = pd.DataFrame(
|
622
|
-
df_csv.loc[fcst_index
|
623
|
-
|
857
|
+
df_csv.loc[fcst_index, :]
|
858
|
+
.between_time(first_hour, last_hour)
|
859
|
+
.values,
|
860
|
+
index=fcst_index,
|
861
|
+
)
|
624
862
|
else:
|
625
863
|
forecast_out = pd.DataFrame(
|
626
864
|
df_csv.between_time(first_hour, last_hour).values,
|
627
|
-
index=fcst_index
|
865
|
+
index=fcst_index,
|
866
|
+
)
|
628
867
|
else:
|
629
868
|
if csv_path is None:
|
630
869
|
if list_and_perfect:
|
631
870
|
forecast_tp = pd.DataFrame(
|
632
871
|
df_csv.between_time(first_hour, last_hour).values,
|
633
|
-
index=fcst_index
|
872
|
+
index=fcst_index,
|
873
|
+
)
|
634
874
|
else:
|
635
875
|
forecast_tp = pd.DataFrame(
|
636
|
-
df_csv.loc[fcst_index
|
637
|
-
|
876
|
+
df_csv.loc[fcst_index, :]
|
877
|
+
.between_time(first_hour, last_hour)
|
878
|
+
.values,
|
879
|
+
index=fcst_index,
|
880
|
+
)
|
638
881
|
else:
|
639
882
|
forecast_tp = pd.DataFrame(
|
640
883
|
df_csv.between_time(first_hour, last_hour).values,
|
641
|
-
index=fcst_index
|
884
|
+
index=fcst_index,
|
885
|
+
)
|
642
886
|
forecast_out = pd.concat([forecast_out, forecast_tp], axis=0)
|
643
887
|
return forecast_out
|
644
|
-
|
645
|
-
def get_load_forecast(
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
888
|
+
|
889
|
+
def get_load_forecast(
|
890
|
+
self,
|
891
|
+
days_min_load_forecast: Optional[int] = 3,
|
892
|
+
method: Optional[str] = "naive",
|
893
|
+
csv_path: Optional[str] = "data_load_forecast.csv",
|
894
|
+
set_mix_forecast: Optional[bool] = False,
|
895
|
+
df_now: Optional[pd.DataFrame] = pd.DataFrame(),
|
896
|
+
use_last_window: Optional[bool] = True,
|
897
|
+
mlf: Optional[MLForecaster] = None,
|
898
|
+
debug: Optional[bool] = False,
|
899
|
+
) -> pd.Series:
|
650
900
|
r"""
|
651
901
|
Get and generate the load forecast data.
|
652
902
|
|
@@ -681,118 +931,165 @@ class Forecast(object):
|
|
681
931
|
:rtype: pd.DataFrame
|
682
932
|
|
683
933
|
"""
|
684
|
-
csv_path
|
685
|
-
|
686
|
-
if
|
687
|
-
|
934
|
+
csv_path = self.emhass_conf["data_path"] / csv_path
|
935
|
+
|
936
|
+
if (
|
937
|
+
method == "naive" or method == "mlforecaster"
|
938
|
+
): # retrieving needed data for these methods
|
939
|
+
self.logger.info(
|
940
|
+
"Retrieving data from hass for load forecast using method = " + method
|
941
|
+
)
|
688
942
|
var_list = [self.var_load]
|
689
943
|
var_replace_zero = None
|
690
944
|
var_interp = [self.var_load]
|
691
945
|
time_zone_load_foreacast = None
|
692
946
|
# We will need to retrieve a new set of load data according to the days_min_load_forecast parameter
|
693
|
-
rh = RetrieveHass(
|
694
|
-
|
947
|
+
rh = RetrieveHass(
|
948
|
+
self.retrieve_hass_conf["hass_url"],
|
949
|
+
self.retrieve_hass_conf["long_lived_token"],
|
950
|
+
self.freq,
|
951
|
+
time_zone_load_foreacast,
|
952
|
+
self.params,
|
953
|
+
self.emhass_conf,
|
954
|
+
self.logger,
|
955
|
+
)
|
695
956
|
if self.get_data_from_file:
|
696
|
-
filename_path = self.emhass_conf[
|
697
|
-
with open(filename_path,
|
957
|
+
filename_path = self.emhass_conf["data_path"] / "test_df_final.pkl"
|
958
|
+
with open(filename_path, "rb") as inp:
|
698
959
|
rh.df_final, days_list, var_list = pickle.load(inp)
|
699
960
|
self.var_load = var_list[0]
|
700
|
-
self.retrieve_hass_conf[
|
961
|
+
self.retrieve_hass_conf["sensor_power_load_no_var_loads"] = (
|
962
|
+
self.var_load
|
963
|
+
)
|
701
964
|
var_interp = [var_list[0]]
|
702
965
|
self.var_list = [var_list[0]]
|
703
|
-
self.var_load_new = self.var_load+
|
966
|
+
self.var_load_new = self.var_load + "_positive"
|
704
967
|
else:
|
705
|
-
days_list = get_days_list(days_min_load_forecast)
|
968
|
+
days_list = get_days_list(days_min_load_forecast)
|
706
969
|
if not rh.get_data(days_list, var_list):
|
707
970
|
return False
|
708
|
-
if
|
709
|
-
self.retrieve_hass_conf[
|
710
|
-
|
711
|
-
|
971
|
+
if not rh.prepare_data(
|
972
|
+
self.retrieve_hass_conf["sensor_power_load_no_var_loads"],
|
973
|
+
load_negative=self.retrieve_hass_conf["load_negative"],
|
974
|
+
set_zero_min=self.retrieve_hass_conf["set_zero_min"],
|
975
|
+
var_replace_zero=var_replace_zero,
|
976
|
+
var_interp=var_interp,
|
977
|
+
):
|
712
978
|
return False
|
713
979
|
df = rh.df_final.copy()[[self.var_load_new]]
|
714
|
-
if method ==
|
715
|
-
mask_forecast_out = (
|
980
|
+
if method == "naive": # using a naive approach
|
981
|
+
mask_forecast_out = (
|
982
|
+
df.index > days_list[-1] - self.optim_conf["delta_forecast_daily"]
|
983
|
+
)
|
716
984
|
forecast_out = df.copy().loc[mask_forecast_out]
|
717
|
-
forecast_out = forecast_out.rename(columns={self.var_load_new:
|
985
|
+
forecast_out = forecast_out.rename(columns={self.var_load_new: "yhat"})
|
718
986
|
# Force forecast_out length to avoid mismatches
|
719
|
-
forecast_out = forecast_out.iloc[0:len(self.forecast_dates)]
|
987
|
+
forecast_out = forecast_out.iloc[0 : len(self.forecast_dates)]
|
720
988
|
forecast_out.index = self.forecast_dates
|
721
|
-
elif
|
989
|
+
elif (
|
990
|
+
method == "mlforecaster"
|
991
|
+
): # using a custom forecast model with machine learning
|
722
992
|
# Load model
|
723
|
-
model_type = self.params[
|
724
|
-
filename = model_type+
|
725
|
-
filename_path = self.emhass_conf[
|
993
|
+
model_type = self.params["passed_data"]["model_type"]
|
994
|
+
filename = model_type + "_mlf.pkl"
|
995
|
+
filename_path = self.emhass_conf["data_path"] / filename
|
726
996
|
if not debug:
|
727
997
|
if filename_path.is_file():
|
728
|
-
with open(filename_path,
|
998
|
+
with open(filename_path, "rb") as inp:
|
729
999
|
mlf = pickle.load(inp)
|
730
1000
|
else:
|
731
|
-
self.logger.error(
|
1001
|
+
self.logger.error(
|
1002
|
+
"The ML forecaster file was not found, please run a model fit method before this predict method"
|
1003
|
+
)
|
732
1004
|
return False
|
733
1005
|
# Make predictions
|
734
1006
|
if use_last_window:
|
735
1007
|
data_last_window = copy.deepcopy(df)
|
736
|
-
data_last_window = data_last_window.rename(
|
1008
|
+
data_last_window = data_last_window.rename(
|
1009
|
+
columns={self.var_load_new: self.var_load}
|
1010
|
+
)
|
737
1011
|
else:
|
738
1012
|
data_last_window = None
|
739
1013
|
forecast_out = mlf.predict(data_last_window)
|
740
1014
|
# Force forecast length to avoid mismatches
|
741
|
-
self.logger.debug(
|
742
|
-
|
1015
|
+
self.logger.debug(
|
1016
|
+
"Number of ML predict forcast data generated (lags_opt): "
|
1017
|
+
+ str(len(forecast_out.index))
|
1018
|
+
)
|
1019
|
+
self.logger.debug(
|
1020
|
+
"Number of forcast dates obtained: " + str(len(self.forecast_dates))
|
1021
|
+
)
|
743
1022
|
if len(self.forecast_dates) < len(forecast_out.index):
|
744
|
-
forecast_out = forecast_out.iloc[0:len(self.forecast_dates)]
|
1023
|
+
forecast_out = forecast_out.iloc[0 : len(self.forecast_dates)]
|
745
1024
|
# To be removed once bug is fixed
|
746
1025
|
elif len(self.forecast_dates) > len(forecast_out.index):
|
747
|
-
self.logger.error(
|
1026
|
+
self.logger.error(
|
1027
|
+
"Unable to obtain: "
|
1028
|
+
+ str(len(self.forecast_dates))
|
1029
|
+
+ " lags_opt values from sensor: power load no var loads, check optimization_time_step/freq and historic_days_to_retrieve/days_to_retrieve parameters"
|
1030
|
+
)
|
748
1031
|
return False
|
749
1032
|
# Define DataFrame
|
750
|
-
data_dict = {
|
1033
|
+
data_dict = {
|
1034
|
+
"ts": self.forecast_dates,
|
1035
|
+
"yhat": forecast_out.values.tolist(),
|
1036
|
+
}
|
751
1037
|
data = pd.DataFrame.from_dict(data_dict)
|
752
1038
|
# Define index
|
753
|
-
data.set_index(
|
1039
|
+
data.set_index("ts", inplace=True)
|
754
1040
|
forecast_out = data.copy().loc[self.forecast_dates]
|
755
|
-
elif method ==
|
1041
|
+
elif method == "csv": # reading from a csv file
|
756
1042
|
load_csv_file_path = csv_path
|
757
|
-
df_csv = pd.read_csv(load_csv_file_path, header=None, names=[
|
1043
|
+
df_csv = pd.read_csv(load_csv_file_path, header=None, names=["ts", "yhat"])
|
758
1044
|
if len(df_csv) < len(self.forecast_dates):
|
759
1045
|
self.logger.error("Passed data from CSV is not long enough")
|
760
1046
|
else:
|
761
1047
|
# Ensure correct length
|
762
|
-
df_csv = df_csv.loc[df_csv.index[0:len(self.forecast_dates)]
|
1048
|
+
df_csv = df_csv.loc[df_csv.index[0 : len(self.forecast_dates)], :]
|
763
1049
|
# Define index
|
764
1050
|
df_csv.index = self.forecast_dates
|
765
|
-
df_csv.drop([
|
1051
|
+
df_csv.drop(["ts"], axis=1, inplace=True)
|
766
1052
|
forecast_out = df_csv.copy().loc[self.forecast_dates]
|
767
|
-
elif method ==
|
1053
|
+
elif method == "list": # reading a list of values
|
768
1054
|
# Loading data from passed list
|
769
|
-
data_list = self.params[
|
1055
|
+
data_list = self.params["passed_data"]["load_power_forecast"]
|
770
1056
|
# Check if the passed data has the correct length
|
771
|
-
if
|
1057
|
+
if (
|
1058
|
+
len(data_list) < len(self.forecast_dates)
|
1059
|
+
and self.params["passed_data"]["prediction_horizon"] is None
|
1060
|
+
):
|
772
1061
|
self.logger.error("Passed data from passed list is not long enough")
|
773
1062
|
return False
|
774
1063
|
else:
|
775
1064
|
# Ensure correct length
|
776
|
-
data_list = data_list[0:len(self.forecast_dates)]
|
1065
|
+
data_list = data_list[0 : len(self.forecast_dates)]
|
777
1066
|
# Define DataFrame
|
778
|
-
data_dict = {
|
1067
|
+
data_dict = {"ts": self.forecast_dates, "yhat": data_list}
|
779
1068
|
data = pd.DataFrame.from_dict(data_dict)
|
780
1069
|
# Define index
|
781
|
-
data.set_index(
|
1070
|
+
data.set_index("ts", inplace=True)
|
782
1071
|
forecast_out = data.copy().loc[self.forecast_dates]
|
783
1072
|
else:
|
784
1073
|
self.logger.error("Passed method is not valid")
|
785
1074
|
return False
|
786
|
-
P_Load_forecast = copy.deepcopy(forecast_out[
|
1075
|
+
P_Load_forecast = copy.deepcopy(forecast_out["yhat"])
|
787
1076
|
if set_mix_forecast:
|
788
1077
|
P_Load_forecast = Forecast.get_mix_forecast(
|
789
|
-
df_now,
|
790
|
-
|
1078
|
+
df_now,
|
1079
|
+
P_Load_forecast,
|
1080
|
+
self.params["passed_data"]["alpha"],
|
1081
|
+
self.params["passed_data"]["beta"],
|
1082
|
+
self.var_load_new,
|
1083
|
+
)
|
791
1084
|
return P_Load_forecast
|
792
|
-
|
793
|
-
def get_load_cost_forecast(
|
794
|
-
|
795
|
-
|
1085
|
+
|
1086
|
+
def get_load_cost_forecast(
|
1087
|
+
self,
|
1088
|
+
df_final: pd.DataFrame,
|
1089
|
+
method: Optional[str] = "hp_hc_periods",
|
1090
|
+
csv_path: Optional[str] = "data_load_cost_forecast.csv",
|
1091
|
+
list_and_perfect: Optional[bool] = False,
|
1092
|
+
) -> pd.DataFrame:
|
796
1093
|
r"""
|
797
1094
|
Get the unit cost for the load consumption based on multiple tariff \
|
798
1095
|
periods. This is the cost of the energy from the utility in a vector \
|
@@ -812,45 +1109,62 @@ class Forecast(object):
|
|
812
1109
|
:rtype: pd.DataFrame
|
813
1110
|
|
814
1111
|
"""
|
815
|
-
csv_path
|
816
|
-
if method ==
|
817
|
-
df_final[self.var_load_cost] = self.optim_conf[
|
1112
|
+
csv_path = self.emhass_conf["data_path"] / csv_path
|
1113
|
+
if method == "hp_hc_periods":
|
1114
|
+
df_final[self.var_load_cost] = self.optim_conf["load_offpeak_hours_cost"]
|
818
1115
|
list_df_hp = []
|
819
|
-
for key, period_hp in self.optim_conf[
|
820
|
-
list_df_hp.append(
|
821
|
-
|
1116
|
+
for key, period_hp in self.optim_conf["load_peak_hour_periods"].items():
|
1117
|
+
list_df_hp.append(
|
1118
|
+
df_final[self.var_load_cost].between_time(
|
1119
|
+
period_hp[0]["start"], period_hp[1]["end"]
|
1120
|
+
)
|
1121
|
+
)
|
822
1122
|
for df_hp in list_df_hp:
|
823
|
-
df_final.loc[df_hp.index, self.var_load_cost] = self.optim_conf[
|
824
|
-
|
1123
|
+
df_final.loc[df_hp.index, self.var_load_cost] = self.optim_conf[
|
1124
|
+
"load_peak_hours_cost"
|
1125
|
+
]
|
1126
|
+
elif method == "csv":
|
825
1127
|
forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0)
|
826
1128
|
forecast_out = self.get_forecast_out_from_csv_or_list(
|
827
|
-
df_final, forecast_dates_csv, csv_path
|
1129
|
+
df_final, forecast_dates_csv, csv_path
|
1130
|
+
)
|
828
1131
|
df_final[self.var_load_cost] = forecast_out
|
829
|
-
elif method ==
|
1132
|
+
elif method == "list": # reading a list of values
|
830
1133
|
# Loading data from passed list
|
831
|
-
data_list = self.params[
|
1134
|
+
data_list = self.params["passed_data"]["load_cost_forecast"]
|
832
1135
|
# Check if the passed data has the correct length
|
833
|
-
if
|
1136
|
+
if (
|
1137
|
+
len(data_list) < len(self.forecast_dates)
|
1138
|
+
and self.params["passed_data"]["prediction_horizon"] is None
|
1139
|
+
):
|
834
1140
|
self.logger.error("Passed data from passed list is not long enough")
|
835
1141
|
return False
|
836
1142
|
else:
|
837
1143
|
# Ensure correct length
|
838
|
-
data_list = data_list[0:len(self.forecast_dates)]
|
1144
|
+
data_list = data_list[0 : len(self.forecast_dates)]
|
839
1145
|
# Define the correct dates
|
840
1146
|
forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0)
|
841
1147
|
forecast_out = self.get_forecast_out_from_csv_or_list(
|
842
|
-
df_final,
|
1148
|
+
df_final,
|
1149
|
+
forecast_dates_csv,
|
1150
|
+
None,
|
1151
|
+
data_list=data_list,
|
1152
|
+
list_and_perfect=list_and_perfect,
|
1153
|
+
)
|
843
1154
|
# Fill the final DF
|
844
1155
|
df_final[self.var_load_cost] = forecast_out
|
845
1156
|
else:
|
846
1157
|
self.logger.error("Passed method is not valid")
|
847
1158
|
return False
|
848
1159
|
return df_final
|
849
|
-
|
850
|
-
def get_prod_price_forecast(self, df_final: pd.DataFrame, method: Optional[str] = 'constant',
|
851
|
-
csv_path: Optional[str] = "data_prod_price_forecast.csv",
|
852
|
-
list_and_perfect: Optional[bool] = False) -> pd.DataFrame:
|
853
1160
|
|
1161
|
+
def get_prod_price_forecast(
|
1162
|
+
self,
|
1163
|
+
df_final: pd.DataFrame,
|
1164
|
+
method: Optional[str] = "constant",
|
1165
|
+
csv_path: Optional[str] = "data_prod_price_forecast.csv",
|
1166
|
+
list_and_perfect: Optional[bool] = False,
|
1167
|
+
) -> pd.DataFrame:
|
854
1168
|
r"""
|
855
1169
|
Get the unit power production price for the energy injected to the grid.\
|
856
1170
|
This is the price of the energy injected to the utility in a vector \
|
@@ -871,31 +1185,42 @@ class Forecast(object):
|
|
871
1185
|
:rtype: pd.DataFrame
|
872
1186
|
|
873
1187
|
"""
|
874
|
-
csv_path
|
875
|
-
if method ==
|
876
|
-
df_final[self.var_prod_price] = self.optim_conf[
|
877
|
-
|
1188
|
+
csv_path = self.emhass_conf["data_path"] / csv_path
|
1189
|
+
if method == "constant":
|
1190
|
+
df_final[self.var_prod_price] = self.optim_conf[
|
1191
|
+
"photovoltaic_production_sell_price"
|
1192
|
+
]
|
1193
|
+
elif method == "csv":
|
878
1194
|
forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0)
|
879
1195
|
forecast_out = self.get_forecast_out_from_csv_or_list(
|
880
|
-
df_final, forecast_dates_csv, csv_path
|
1196
|
+
df_final, forecast_dates_csv, csv_path
|
1197
|
+
)
|
881
1198
|
df_final[self.var_prod_price] = forecast_out
|
882
|
-
elif method ==
|
1199
|
+
elif method == "list": # reading a list of values
|
883
1200
|
# Loading data from passed list
|
884
|
-
data_list = self.params[
|
1201
|
+
data_list = self.params["passed_data"]["prod_price_forecast"]
|
885
1202
|
# Check if the passed data has the correct length
|
886
|
-
if
|
1203
|
+
if (
|
1204
|
+
len(data_list) < len(self.forecast_dates)
|
1205
|
+
and self.params["passed_data"]["prediction_horizon"] is None
|
1206
|
+
):
|
887
1207
|
self.logger.error("Passed data from passed list is not long enough")
|
888
1208
|
return False
|
889
1209
|
else:
|
890
1210
|
# Ensure correct length
|
891
|
-
data_list = data_list[0:len(self.forecast_dates)]
|
1211
|
+
data_list = data_list[0 : len(self.forecast_dates)]
|
892
1212
|
# Define the correct dates
|
893
1213
|
forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0)
|
894
1214
|
forecast_out = self.get_forecast_out_from_csv_or_list(
|
895
|
-
df_final,
|
1215
|
+
df_final,
|
1216
|
+
forecast_dates_csv,
|
1217
|
+
None,
|
1218
|
+
data_list=data_list,
|
1219
|
+
list_and_perfect=list_and_perfect,
|
1220
|
+
)
|
896
1221
|
# Fill the final DF
|
897
1222
|
df_final[self.var_prod_price] = forecast_out
|
898
1223
|
else:
|
899
1224
|
self.logger.error("Passed method is not valid")
|
900
1225
|
return False
|
901
|
-
return df_final
|
1226
|
+
return df_final
|