emhass 0.8.5__py3-none-any.whl → 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- emhass/command_line.py +705 -272
- emhass/forecast.py +114 -45
- emhass/machine_learning_forecaster.py +4 -4
- emhass/machine_learning_regressor.py +290 -0
- emhass/optimization.py +4 -3
- emhass/retrieve_hass.py +235 -103
- emhass/static/advanced.html +3 -0
- emhass/static/script.js +2 -0
- emhass/utils.py +605 -305
- emhass/web_server.py +48 -26
- {emhass-0.8.5.dist-info → emhass-0.9.0.dist-info}/METADATA +19 -5
- emhass-0.9.0.dist-info/RECORD +26 -0
- emhass-0.8.5.dist-info/RECORD +0 -25
- {emhass-0.8.5.dist-info → emhass-0.9.0.dist-info}/LICENSE +0 -0
- {emhass-0.8.5.dist-info → emhass-0.9.0.dist-info}/WHEEL +0 -0
- {emhass-0.8.5.dist-info → emhass-0.9.0.dist-info}/entry_points.txt +0 -0
- {emhass-0.8.5.dist-info → emhass-0.9.0.dist-info}/top_level.txt +0 -0
emhass/utils.py
CHANGED
|
@@ -2,10 +2,19 @@
|
|
|
2
2
|
# -*- coding: utf-8 -*-
|
|
3
3
|
|
|
4
4
|
from typing import Tuple, Optional
|
|
5
|
-
import numpy as np, pandas as pd
|
|
6
|
-
import yaml, pytz, logging, pathlib, json, copy
|
|
7
5
|
from datetime import datetime, timedelta, timezone
|
|
6
|
+
import logging
|
|
7
|
+
import pathlib
|
|
8
|
+
import json
|
|
9
|
+
import copy
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
import yaml
|
|
13
|
+
import pytz
|
|
14
|
+
|
|
15
|
+
|
|
8
16
|
import plotly.express as px
|
|
17
|
+
|
|
9
18
|
pd.options.plotting.backend = "plotly"
|
|
10
19
|
|
|
11
20
|
from emhass.machine_learning_forecaster import MLForecaster
|
|
@@ -14,13 +23,13 @@ from emhass.machine_learning_forecaster import MLForecaster
|
|
|
14
23
|
def get_root(file: str, num_parent: Optional[int] = 3) -> str:
|
|
15
24
|
"""
|
|
16
25
|
Get the root absolute path of the working directory.
|
|
17
|
-
|
|
26
|
+
|
|
18
27
|
:param file: The passed file path with __file__
|
|
19
28
|
:return: The root path
|
|
20
29
|
:param num_parent: The number of parents levels up to desired root folder
|
|
21
30
|
:type num_parent: int, optional
|
|
22
31
|
:rtype: str
|
|
23
|
-
|
|
32
|
+
|
|
24
33
|
"""
|
|
25
34
|
if num_parent == 3:
|
|
26
35
|
root = pathlib.Path(file).resolve().parent.parent.parent
|
|
@@ -32,27 +41,27 @@ def get_root(file: str, num_parent: Optional[int] = 3) -> str:
|
|
|
32
41
|
raise ValueError("num_parent value not valid, must be between 1 and 3")
|
|
33
42
|
return root
|
|
34
43
|
|
|
35
|
-
def get_logger(fun_name: str,
|
|
44
|
+
def get_logger(fun_name: str, emhass_conf: dict, save_to_file: Optional[bool] = True,
|
|
36
45
|
logging_level: Optional[str] = "DEBUG") -> Tuple[logging.Logger, logging.StreamHandler]:
|
|
37
46
|
"""
|
|
38
47
|
Create a simple logger object.
|
|
39
|
-
|
|
48
|
+
|
|
40
49
|
:param fun_name: The Python function object name where the logger will be used
|
|
41
50
|
:type fun_name: str
|
|
42
|
-
:param
|
|
43
|
-
:type
|
|
51
|
+
:param emhass_conf: Dictionary containing the needed emhass paths
|
|
52
|
+
:type emhass_conf: dict
|
|
44
53
|
:param save_to_file: Write log to a file, defaults to True
|
|
45
54
|
:type save_to_file: bool, optional
|
|
46
55
|
:return: The logger object and the handler
|
|
47
56
|
:rtype: object
|
|
48
|
-
|
|
57
|
+
|
|
49
58
|
"""
|
|
50
|
-
|
|
59
|
+
# create logger object
|
|
51
60
|
logger = logging.getLogger(fun_name)
|
|
52
61
|
logger.propagate = True
|
|
53
62
|
logger.fileSetting = save_to_file
|
|
54
63
|
if save_to_file:
|
|
55
|
-
ch = logging.FileHandler(
|
|
64
|
+
ch = logging.FileHandler(emhass_conf['data_path'] / 'logger_emhass.log')
|
|
56
65
|
else:
|
|
57
66
|
ch = logging.StreamHandler()
|
|
58
67
|
if logging_level == "DEBUG":
|
|
@@ -70,14 +79,18 @@ def get_logger(fun_name: str, config_path: str, save_to_file: Optional[bool] = T
|
|
|
70
79
|
else:
|
|
71
80
|
logger.setLevel(logging.DEBUG)
|
|
72
81
|
ch.setLevel(logging.DEBUG)
|
|
73
|
-
formatter = logging.Formatter(
|
|
82
|
+
formatter = logging.Formatter(
|
|
83
|
+
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
84
|
+
)
|
|
74
85
|
ch.setFormatter(formatter)
|
|
75
86
|
logger.addHandler(ch)
|
|
76
87
|
|
|
77
88
|
return logger, ch
|
|
78
89
|
|
|
79
|
-
|
|
80
|
-
|
|
90
|
+
|
|
91
|
+
def get_forecast_dates(
|
|
92
|
+
freq: int, delta_forecast: int, timedelta_days: Optional[int] = 0
|
|
93
|
+
) -> pd.core.indexes.datetimes.DatetimeIndex:
|
|
81
94
|
"""
|
|
82
95
|
Get the date_range list of the needed future dates using the delta_forecast parameter.
|
|
83
96
|
|
|
@@ -89,7 +102,7 @@ def get_forecast_dates(freq: int, delta_forecast: int,
|
|
|
89
102
|
:type timedelta_days: Optional[int], optional
|
|
90
103
|
:return: A list of future forecast dates.
|
|
91
104
|
:rtype: pd.core.indexes.datetimes.DatetimeIndex
|
|
92
|
-
|
|
105
|
+
|
|
93
106
|
"""
|
|
94
107
|
freq = pd.to_timedelta(freq, "minutes")
|
|
95
108
|
start_forecast = pd.Timestamp(datetime.now()).replace(hour=0, minute=0, second=0, microsecond=0)
|
|
@@ -99,11 +112,19 @@ def get_forecast_dates(freq: int, delta_forecast: int,
|
|
|
99
112
|
freq=freq).round(freq, ambiguous='infer', nonexistent='shift_forward')
|
|
100
113
|
return forecast_dates
|
|
101
114
|
|
|
102
|
-
|
|
103
|
-
|
|
115
|
+
|
|
116
|
+
def treat_runtimeparams(
|
|
117
|
+
runtimeparams: str,
|
|
118
|
+
params: str,
|
|
119
|
+
retrieve_hass_conf: dict,
|
|
120
|
+
optim_conf: dict,
|
|
121
|
+
plant_conf: dict,
|
|
122
|
+
set_type: str,
|
|
123
|
+
logger: logging.Logger,
|
|
124
|
+
) -> Tuple[str, dict]:
|
|
104
125
|
"""
|
|
105
|
-
Treat the passed optimization runtime parameters.
|
|
106
|
-
|
|
126
|
+
Treat the passed optimization runtime parameters.
|
|
127
|
+
|
|
107
128
|
:param runtimeparams: Json string containing the runtime parameters dict.
|
|
108
129
|
:type runtimeparams: str
|
|
109
130
|
:param params: Configuration parameters passed from data/options.json
|
|
@@ -120,93 +141,167 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
|
|
|
120
141
|
:type logger: logging.Logger
|
|
121
142
|
:return: Returning the params and optimization parameter container.
|
|
122
143
|
:rtype: Tuple[str, dict]
|
|
123
|
-
|
|
144
|
+
|
|
124
145
|
"""
|
|
125
|
-
if (params != None) and (params !=
|
|
146
|
+
if (params != None) and (params != "null"):
|
|
126
147
|
params = json.loads(params)
|
|
127
148
|
else:
|
|
128
149
|
params = {}
|
|
129
150
|
# Some default data needed
|
|
130
151
|
custom_deferrable_forecast_id = []
|
|
131
|
-
for k in range(optim_conf[
|
|
132
|
-
custom_deferrable_forecast_id.append(
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
152
|
+
for k in range(optim_conf["num_def_loads"]):
|
|
153
|
+
custom_deferrable_forecast_id.append(
|
|
154
|
+
{
|
|
155
|
+
"entity_id": "sensor.p_deferrable{}".format(k),
|
|
156
|
+
"unit_of_measurement": "W",
|
|
157
|
+
"friendly_name": "Deferrable Load {}".format(k),
|
|
158
|
+
}
|
|
159
|
+
)
|
|
160
|
+
default_passed_dict = {
|
|
161
|
+
"custom_pv_forecast_id": {
|
|
162
|
+
"entity_id": "sensor.p_pv_forecast",
|
|
163
|
+
"unit_of_measurement": "W",
|
|
164
|
+
"friendly_name": "PV Power Forecast",
|
|
165
|
+
},
|
|
166
|
+
"custom_load_forecast_id": {
|
|
167
|
+
"entity_id": "sensor.p_load_forecast",
|
|
168
|
+
"unit_of_measurement": "W",
|
|
169
|
+
"friendly_name": "Load Power Forecast",
|
|
170
|
+
},
|
|
171
|
+
"custom_batt_forecast_id": {
|
|
172
|
+
"entity_id": "sensor.p_batt_forecast",
|
|
173
|
+
"unit_of_measurement": "W",
|
|
174
|
+
"friendly_name": "Battery Power Forecast",
|
|
175
|
+
},
|
|
176
|
+
"custom_batt_soc_forecast_id": {
|
|
177
|
+
"entity_id": "sensor.soc_batt_forecast",
|
|
178
|
+
"unit_of_measurement": "%",
|
|
179
|
+
"friendly_name": "Battery SOC Forecast",
|
|
180
|
+
},
|
|
181
|
+
"custom_grid_forecast_id": {
|
|
182
|
+
"entity_id": "sensor.p_grid_forecast",
|
|
183
|
+
"unit_of_measurement": "W",
|
|
184
|
+
"friendly_name": "Grid Power Forecast",
|
|
185
|
+
},
|
|
186
|
+
"custom_cost_fun_id": {
|
|
187
|
+
"entity_id": "sensor.total_cost_fun_value",
|
|
188
|
+
"unit_of_measurement": "",
|
|
189
|
+
"friendly_name": "Total cost function value",
|
|
190
|
+
},
|
|
191
|
+
"custom_optim_status_id": {
|
|
192
|
+
"entity_id": "sensor.optim_status",
|
|
193
|
+
"unit_of_measurement": "",
|
|
194
|
+
"friendly_name": "EMHASS optimization status",
|
|
195
|
+
},
|
|
196
|
+
"custom_unit_load_cost_id": {
|
|
197
|
+
"entity_id": "sensor.unit_load_cost",
|
|
198
|
+
"unit_of_measurement": "€/kWh",
|
|
199
|
+
"friendly_name": "Unit Load Cost",
|
|
200
|
+
},
|
|
201
|
+
"custom_unit_prod_price_id": {
|
|
202
|
+
"entity_id": "sensor.unit_prod_price",
|
|
203
|
+
"unit_of_measurement": "€/kWh",
|
|
204
|
+
"friendly_name": "Unit Prod Price",
|
|
205
|
+
},
|
|
206
|
+
"custom_deferrable_forecast_id": custom_deferrable_forecast_id,
|
|
207
|
+
"publish_prefix": "",
|
|
208
|
+
}
|
|
209
|
+
if "passed_data" in params.keys():
|
|
149
210
|
for key, value in default_passed_dict.items():
|
|
150
|
-
params[
|
|
211
|
+
params["passed_data"][key] = value
|
|
151
212
|
else:
|
|
152
|
-
params[
|
|
213
|
+
params["passed_data"] = default_passed_dict
|
|
153
214
|
if runtimeparams is not None:
|
|
154
215
|
runtimeparams = json.loads(runtimeparams)
|
|
155
|
-
freq = int(retrieve_hass_conf[
|
|
156
|
-
delta_forecast = int(optim_conf[
|
|
216
|
+
freq = int(retrieve_hass_conf["freq"].seconds / 60.0)
|
|
217
|
+
delta_forecast = int(optim_conf["delta_forecast"].days)
|
|
157
218
|
forecast_dates = get_forecast_dates(freq, delta_forecast)
|
|
219
|
+
if set_type == "regressor-model-fit":
|
|
220
|
+
if "csv_file" in runtimeparams:
|
|
221
|
+
csv_file = runtimeparams["csv_file"]
|
|
222
|
+
params["passed_data"]["csv_file"] = csv_file
|
|
223
|
+
if "features" in runtimeparams:
|
|
224
|
+
features = runtimeparams["features"]
|
|
225
|
+
params["passed_data"]["features"] = features
|
|
226
|
+
if "target" in runtimeparams:
|
|
227
|
+
target = runtimeparams["target"]
|
|
228
|
+
params["passed_data"]["target"] = target
|
|
229
|
+
if "timestamp" not in runtimeparams:
|
|
230
|
+
params["passed_data"]["timestamp"] = None
|
|
231
|
+
else:
|
|
232
|
+
timestamp = runtimeparams["timestamp"]
|
|
233
|
+
params["passed_data"]["timestamp"] = timestamp
|
|
234
|
+
if "date_features" not in runtimeparams:
|
|
235
|
+
params["passed_data"]["date_features"] = []
|
|
236
|
+
else:
|
|
237
|
+
date_features = runtimeparams["date_features"]
|
|
238
|
+
params["passed_data"]["date_features"] = date_features
|
|
239
|
+
if set_type == "regressor-model-predict":
|
|
240
|
+
if "new_values" in runtimeparams:
|
|
241
|
+
new_values = runtimeparams["new_values"]
|
|
242
|
+
params["passed_data"]["new_values"] = new_values
|
|
243
|
+
if "csv_file" in runtimeparams:
|
|
244
|
+
csv_file = runtimeparams["csv_file"]
|
|
245
|
+
params["passed_data"]["csv_file"] = csv_file
|
|
246
|
+
if "features" in runtimeparams:
|
|
247
|
+
features = runtimeparams["features"]
|
|
248
|
+
params["passed_data"]["features"] = features
|
|
249
|
+
if "target" in runtimeparams:
|
|
250
|
+
target = runtimeparams["target"]
|
|
251
|
+
params["passed_data"]["target"] = target
|
|
252
|
+
|
|
158
253
|
# Treating special data passed for MPC control case
|
|
159
|
-
if set_type ==
|
|
160
|
-
if
|
|
161
|
-
prediction_horizon = 10
|
|
254
|
+
if set_type == "naive-mpc-optim":
|
|
255
|
+
if "prediction_horizon" not in runtimeparams.keys():
|
|
256
|
+
prediction_horizon = 10 # 10 time steps by default
|
|
162
257
|
else:
|
|
163
|
-
prediction_horizon = runtimeparams[
|
|
164
|
-
params[
|
|
165
|
-
if
|
|
166
|
-
soc_init = plant_conf[
|
|
258
|
+
prediction_horizon = runtimeparams["prediction_horizon"]
|
|
259
|
+
params["passed_data"]["prediction_horizon"] = prediction_horizon
|
|
260
|
+
if "soc_init" not in runtimeparams.keys():
|
|
261
|
+
soc_init = plant_conf["SOCtarget"]
|
|
167
262
|
else:
|
|
168
|
-
soc_init = runtimeparams[
|
|
169
|
-
params[
|
|
170
|
-
if
|
|
171
|
-
soc_final = plant_conf[
|
|
263
|
+
soc_init = runtimeparams["soc_init"]
|
|
264
|
+
params["passed_data"]["soc_init"] = soc_init
|
|
265
|
+
if "soc_final" not in runtimeparams.keys():
|
|
266
|
+
soc_final = plant_conf["SOCtarget"]
|
|
172
267
|
else:
|
|
173
|
-
soc_final = runtimeparams[
|
|
174
|
-
params[
|
|
175
|
-
if
|
|
176
|
-
def_total_hours = optim_conf[
|
|
268
|
+
soc_final = runtimeparams["soc_final"]
|
|
269
|
+
params["passed_data"]["soc_final"] = soc_final
|
|
270
|
+
if "def_total_hours" not in runtimeparams.keys():
|
|
271
|
+
def_total_hours = optim_conf["def_total_hours"]
|
|
177
272
|
else:
|
|
178
|
-
def_total_hours = runtimeparams[
|
|
179
|
-
params[
|
|
180
|
-
if
|
|
181
|
-
def_start_timestep = optim_conf[
|
|
273
|
+
def_total_hours = runtimeparams["def_total_hours"]
|
|
274
|
+
params["passed_data"]["def_total_hours"] = def_total_hours
|
|
275
|
+
if "def_start_timestep" not in runtimeparams.keys():
|
|
276
|
+
def_start_timestep = optim_conf["def_start_timestep"]
|
|
182
277
|
else:
|
|
183
|
-
def_start_timestep = runtimeparams[
|
|
184
|
-
params[
|
|
185
|
-
if
|
|
186
|
-
def_end_timestep = optim_conf[
|
|
278
|
+
def_start_timestep = runtimeparams["def_start_timestep"]
|
|
279
|
+
params["passed_data"]["def_start_timestep"] = def_start_timestep
|
|
280
|
+
if "def_end_timestep" not in runtimeparams.keys():
|
|
281
|
+
def_end_timestep = optim_conf["def_end_timestep"]
|
|
187
282
|
else:
|
|
188
|
-
def_end_timestep = runtimeparams[
|
|
189
|
-
params[
|
|
190
|
-
if
|
|
283
|
+
def_end_timestep = runtimeparams["def_end_timestep"]
|
|
284
|
+
params["passed_data"]["def_end_timestep"] = def_end_timestep
|
|
285
|
+
if "alpha" not in runtimeparams.keys():
|
|
191
286
|
alpha = 0.5
|
|
192
287
|
else:
|
|
193
|
-
alpha = runtimeparams[
|
|
194
|
-
params[
|
|
195
|
-
if
|
|
288
|
+
alpha = runtimeparams["alpha"]
|
|
289
|
+
params["passed_data"]["alpha"] = alpha
|
|
290
|
+
if "beta" not in runtimeparams.keys():
|
|
196
291
|
beta = 0.5
|
|
197
292
|
else:
|
|
198
|
-
beta = runtimeparams[
|
|
199
|
-
params[
|
|
293
|
+
beta = runtimeparams["beta"]
|
|
294
|
+
params["passed_data"]["beta"] = beta
|
|
200
295
|
forecast_dates = copy.deepcopy(forecast_dates)[0:prediction_horizon]
|
|
201
296
|
else:
|
|
202
|
-
params[
|
|
203
|
-
params[
|
|
204
|
-
params[
|
|
205
|
-
params[
|
|
206
|
-
params[
|
|
207
|
-
params[
|
|
208
|
-
params[
|
|
209
|
-
params[
|
|
297
|
+
params["passed_data"]["prediction_horizon"] = None
|
|
298
|
+
params["passed_data"]["soc_init"] = None
|
|
299
|
+
params["passed_data"]["soc_final"] = None
|
|
300
|
+
params["passed_data"]["def_total_hours"] = None
|
|
301
|
+
params["passed_data"]["def_start_timestep"] = None
|
|
302
|
+
params["passed_data"]["def_end_timestep"] = None
|
|
303
|
+
params["passed_data"]["alpha"] = None
|
|
304
|
+
params["passed_data"]["beta"] = None
|
|
210
305
|
# Treat passed forecast data lists
|
|
211
306
|
list_forecast_key = ['pv_power_forecast', 'load_power_forecast', 'load_cost_forecast', 'prod_price_forecast']
|
|
212
307
|
forecast_methods = ['weather_forecast_method', 'load_forecast_method', 'load_cost_forecast_method', 'prod_price_forecast_method']
|
|
@@ -226,130 +321,195 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
|
|
|
226
321
|
else:
|
|
227
322
|
params['passed_data'][forecast_key] = None
|
|
228
323
|
# Treat passed data for forecast model fit/predict/tune at runtime
|
|
229
|
-
if
|
|
324
|
+
if "days_to_retrieve" not in runtimeparams.keys():
|
|
230
325
|
days_to_retrieve = 9
|
|
231
326
|
else:
|
|
232
|
-
days_to_retrieve = runtimeparams[
|
|
233
|
-
params[
|
|
234
|
-
if
|
|
327
|
+
days_to_retrieve = runtimeparams["days_to_retrieve"]
|
|
328
|
+
params["passed_data"]["days_to_retrieve"] = days_to_retrieve
|
|
329
|
+
if "model_type" not in runtimeparams.keys():
|
|
235
330
|
model_type = "load_forecast"
|
|
236
331
|
else:
|
|
237
|
-
model_type = runtimeparams[
|
|
238
|
-
params[
|
|
239
|
-
if
|
|
332
|
+
model_type = runtimeparams["model_type"]
|
|
333
|
+
params["passed_data"]["model_type"] = model_type
|
|
334
|
+
if "var_model" not in runtimeparams.keys():
|
|
240
335
|
var_model = "sensor.power_load_no_var_loads"
|
|
241
336
|
else:
|
|
242
|
-
var_model = runtimeparams[
|
|
243
|
-
params[
|
|
244
|
-
if
|
|
337
|
+
var_model = runtimeparams["var_model"]
|
|
338
|
+
params["passed_data"]["var_model"] = var_model
|
|
339
|
+
if "sklearn_model" not in runtimeparams.keys():
|
|
245
340
|
sklearn_model = "KNeighborsRegressor"
|
|
246
341
|
else:
|
|
247
|
-
sklearn_model = runtimeparams[
|
|
248
|
-
params[
|
|
249
|
-
if
|
|
342
|
+
sklearn_model = runtimeparams["sklearn_model"]
|
|
343
|
+
params["passed_data"]["sklearn_model"] = sklearn_model
|
|
344
|
+
if "regression_model" not in runtimeparams.keys():
|
|
345
|
+
regression_model = "AdaBoostRegression"
|
|
346
|
+
else:
|
|
347
|
+
regression_model = runtimeparams["regression_model"]
|
|
348
|
+
params["passed_data"]["regression_model"] = regression_model
|
|
349
|
+
if "num_lags" not in runtimeparams.keys():
|
|
250
350
|
num_lags = 48
|
|
251
351
|
else:
|
|
252
|
-
num_lags = runtimeparams[
|
|
253
|
-
params[
|
|
254
|
-
if
|
|
255
|
-
split_date_delta =
|
|
352
|
+
num_lags = runtimeparams["num_lags"]
|
|
353
|
+
params["passed_data"]["num_lags"] = num_lags
|
|
354
|
+
if "split_date_delta" not in runtimeparams.keys():
|
|
355
|
+
split_date_delta = "48h"
|
|
256
356
|
else:
|
|
257
|
-
split_date_delta = runtimeparams[
|
|
258
|
-
params[
|
|
259
|
-
if
|
|
357
|
+
split_date_delta = runtimeparams["split_date_delta"]
|
|
358
|
+
params["passed_data"]["split_date_delta"] = split_date_delta
|
|
359
|
+
if "perform_backtest" not in runtimeparams.keys():
|
|
260
360
|
perform_backtest = False
|
|
261
361
|
else:
|
|
262
|
-
perform_backtest = eval(str(runtimeparams[
|
|
263
|
-
params[
|
|
264
|
-
if
|
|
362
|
+
perform_backtest = eval(str(runtimeparams["perform_backtest"]).capitalize())
|
|
363
|
+
params["passed_data"]["perform_backtest"] = perform_backtest
|
|
364
|
+
if "model_predict_publish" not in runtimeparams.keys():
|
|
265
365
|
model_predict_publish = False
|
|
266
366
|
else:
|
|
267
|
-
model_predict_publish = eval(
|
|
268
|
-
|
|
269
|
-
|
|
367
|
+
model_predict_publish = eval(
|
|
368
|
+
str(runtimeparams["model_predict_publish"]).capitalize()
|
|
369
|
+
)
|
|
370
|
+
params["passed_data"]["model_predict_publish"] = model_predict_publish
|
|
371
|
+
if "model_predict_entity_id" not in runtimeparams.keys():
|
|
270
372
|
model_predict_entity_id = "sensor.p_load_forecast_custom_model"
|
|
271
373
|
else:
|
|
272
|
-
model_predict_entity_id = runtimeparams[
|
|
273
|
-
params[
|
|
274
|
-
if
|
|
374
|
+
model_predict_entity_id = runtimeparams["model_predict_entity_id"]
|
|
375
|
+
params["passed_data"]["model_predict_entity_id"] = model_predict_entity_id
|
|
376
|
+
if "model_predict_unit_of_measurement" not in runtimeparams.keys():
|
|
275
377
|
model_predict_unit_of_measurement = "W"
|
|
276
378
|
else:
|
|
277
|
-
model_predict_unit_of_measurement = runtimeparams[
|
|
278
|
-
|
|
279
|
-
|
|
379
|
+
model_predict_unit_of_measurement = runtimeparams[
|
|
380
|
+
"model_predict_unit_of_measurement"
|
|
381
|
+
]
|
|
382
|
+
params["passed_data"][
|
|
383
|
+
"model_predict_unit_of_measurement"
|
|
384
|
+
] = model_predict_unit_of_measurement
|
|
385
|
+
if "model_predict_friendly_name" not in runtimeparams.keys():
|
|
280
386
|
model_predict_friendly_name = "Load Power Forecast custom ML model"
|
|
281
387
|
else:
|
|
282
|
-
model_predict_friendly_name = runtimeparams[
|
|
283
|
-
params[
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
if
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
optim_conf[
|
|
308
|
-
if
|
|
309
|
-
optim_conf[
|
|
310
|
-
if
|
|
311
|
-
optim_conf[
|
|
388
|
+
model_predict_friendly_name = runtimeparams["model_predict_friendly_name"]
|
|
389
|
+
params["passed_data"][
|
|
390
|
+
"model_predict_friendly_name"
|
|
391
|
+
] = model_predict_friendly_name
|
|
392
|
+
if "mlr_predict_entity_id" not in runtimeparams.keys():
|
|
393
|
+
mlr_predict_entity_id = "sensor.mlr_predict"
|
|
394
|
+
else:
|
|
395
|
+
mlr_predict_entity_id = runtimeparams["mlr_predict_entity_id"]
|
|
396
|
+
params["passed_data"]["mlr_predict_entity_id"] = mlr_predict_entity_id
|
|
397
|
+
if "mlr_predict_unit_of_measurement" not in runtimeparams.keys():
|
|
398
|
+
mlr_predict_unit_of_measurement = None
|
|
399
|
+
else:
|
|
400
|
+
mlr_predict_unit_of_measurement = runtimeparams[
|
|
401
|
+
"mlr_predict_unit_of_measurement"
|
|
402
|
+
]
|
|
403
|
+
params["passed_data"][
|
|
404
|
+
"mlr_predict_unit_of_measurement"
|
|
405
|
+
] = mlr_predict_unit_of_measurement
|
|
406
|
+
if "mlr_predict_friendly_name" not in runtimeparams.keys():
|
|
407
|
+
mlr_predict_friendly_name = "mlr predictor"
|
|
408
|
+
else:
|
|
409
|
+
mlr_predict_friendly_name = runtimeparams["mlr_predict_friendly_name"]
|
|
410
|
+
params["passed_data"]["mlr_predict_friendly_name"] = mlr_predict_friendly_name
|
|
411
|
+
# Treat optimization configuration parameters passed at runtime
|
|
412
|
+
if "num_def_loads" in runtimeparams.keys():
|
|
413
|
+
optim_conf["num_def_loads"] = runtimeparams["num_def_loads"]
|
|
414
|
+
if "P_deferrable_nom" in runtimeparams.keys():
|
|
415
|
+
optim_conf["P_deferrable_nom"] = runtimeparams["P_deferrable_nom"]
|
|
416
|
+
if "def_total_hours" in runtimeparams.keys():
|
|
417
|
+
optim_conf["def_total_hours"] = runtimeparams["def_total_hours"]
|
|
418
|
+
if "def_start_timestep" in runtimeparams.keys():
|
|
419
|
+
optim_conf["def_start_timestep"] = runtimeparams["def_start_timestep"]
|
|
420
|
+
if "def_end_timestep" in runtimeparams.keys():
|
|
421
|
+
optim_conf["def_end_timestep"] = runtimeparams["def_end_timestep"]
|
|
422
|
+
if "treat_def_as_semi_cont" in runtimeparams.keys():
|
|
423
|
+
optim_conf["treat_def_as_semi_cont"] = [
|
|
424
|
+
eval(str(k).capitalize())
|
|
425
|
+
for k in runtimeparams["treat_def_as_semi_cont"]
|
|
426
|
+
]
|
|
427
|
+
if "set_def_constant" in runtimeparams.keys():
|
|
428
|
+
optim_conf["set_def_constant"] = [
|
|
429
|
+
eval(str(k).capitalize()) for k in runtimeparams["set_def_constant"]
|
|
430
|
+
]
|
|
431
|
+
if "solcast_api_key" in runtimeparams.keys():
|
|
432
|
+
retrieve_hass_conf["solcast_api_key"] = runtimeparams["solcast_api_key"]
|
|
433
|
+
optim_conf["weather_forecast_method"] = "solcast"
|
|
434
|
+
if "solcast_rooftop_id" in runtimeparams.keys():
|
|
435
|
+
retrieve_hass_conf["solcast_rooftop_id"] = runtimeparams[
|
|
436
|
+
"solcast_rooftop_id"
|
|
437
|
+
]
|
|
438
|
+
optim_conf["weather_forecast_method"] = "solcast"
|
|
439
|
+
if "solar_forecast_kwp" in runtimeparams.keys():
|
|
440
|
+
retrieve_hass_conf["solar_forecast_kwp"] = runtimeparams[
|
|
441
|
+
"solar_forecast_kwp"
|
|
442
|
+
]
|
|
443
|
+
optim_conf["weather_forecast_method"] = "solar.forecast"
|
|
444
|
+
if "weight_battery_discharge" in runtimeparams.keys():
|
|
445
|
+
optim_conf["weight_battery_discharge"] = runtimeparams[
|
|
446
|
+
"weight_battery_discharge"
|
|
447
|
+
]
|
|
448
|
+
if "weight_battery_charge" in runtimeparams.keys():
|
|
449
|
+
optim_conf["weight_battery_charge"] = runtimeparams["weight_battery_charge"]
|
|
450
|
+
if 'freq' in runtimeparams.keys():
|
|
451
|
+
retrieve_hass_conf['freq'] = pd.to_timedelta(runtimeparams['freq'], "minutes")
|
|
312
452
|
# Treat plant configuration parameters passed at runtime
|
|
313
|
-
if
|
|
314
|
-
plant_conf[
|
|
453
|
+
if "SOCtarget" in runtimeparams.keys():
|
|
454
|
+
plant_conf["SOCtarget"] = runtimeparams["SOCtarget"]
|
|
315
455
|
# Treat custom entities id's and friendly names for variables
|
|
316
|
-
if
|
|
317
|
-
params[
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
if
|
|
321
|
-
params[
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
if
|
|
325
|
-
params[
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
if
|
|
329
|
-
params[
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
if
|
|
333
|
-
params[
|
|
334
|
-
|
|
335
|
-
|
|
456
|
+
if "custom_pv_forecast_id" in runtimeparams.keys():
|
|
457
|
+
params["passed_data"]["custom_pv_forecast_id"] = runtimeparams[
|
|
458
|
+
"custom_pv_forecast_id"
|
|
459
|
+
]
|
|
460
|
+
if "custom_load_forecast_id" in runtimeparams.keys():
|
|
461
|
+
params["passed_data"]["custom_load_forecast_id"] = runtimeparams[
|
|
462
|
+
"custom_load_forecast_id"
|
|
463
|
+
]
|
|
464
|
+
if "custom_batt_forecast_id" in runtimeparams.keys():
|
|
465
|
+
params["passed_data"]["custom_batt_forecast_id"] = runtimeparams[
|
|
466
|
+
"custom_batt_forecast_id"
|
|
467
|
+
]
|
|
468
|
+
if "custom_batt_soc_forecast_id" in runtimeparams.keys():
|
|
469
|
+
params["passed_data"]["custom_batt_soc_forecast_id"] = runtimeparams[
|
|
470
|
+
"custom_batt_soc_forecast_id"
|
|
471
|
+
]
|
|
472
|
+
if "custom_grid_forecast_id" in runtimeparams.keys():
|
|
473
|
+
params["passed_data"]["custom_grid_forecast_id"] = runtimeparams[
|
|
474
|
+
"custom_grid_forecast_id"
|
|
475
|
+
]
|
|
476
|
+
if "custom_cost_fun_id" in runtimeparams.keys():
|
|
477
|
+
params["passed_data"]["custom_cost_fun_id"] = runtimeparams[
|
|
478
|
+
"custom_cost_fun_id"
|
|
479
|
+
]
|
|
480
|
+
if "custom_optim_status_id" in runtimeparams.keys():
|
|
481
|
+
params["passed_data"]["custom_optim_status_id"] = runtimeparams[
|
|
482
|
+
"custom_optim_status_id"
|
|
483
|
+
]
|
|
484
|
+
if "custom_unit_load_cost_id" in runtimeparams.keys():
|
|
485
|
+
params["passed_data"]["custom_unit_load_cost_id"] = runtimeparams[
|
|
486
|
+
"custom_unit_load_cost_id"
|
|
487
|
+
]
|
|
488
|
+
if "custom_unit_prod_price_id" in runtimeparams.keys():
|
|
489
|
+
params["passed_data"]["custom_unit_prod_price_id"] = runtimeparams[
|
|
490
|
+
"custom_unit_prod_price_id"
|
|
491
|
+
]
|
|
492
|
+
if "custom_deferrable_forecast_id" in runtimeparams.keys():
|
|
493
|
+
params["passed_data"]["custom_deferrable_forecast_id"] = runtimeparams[
|
|
494
|
+
"custom_deferrable_forecast_id"
|
|
495
|
+
]
|
|
336
496
|
# A condition to put a prefix on all published data
|
|
337
|
-
if
|
|
497
|
+
if "publish_prefix" not in runtimeparams.keys():
|
|
338
498
|
publish_prefix = ""
|
|
339
499
|
else:
|
|
340
|
-
publish_prefix = runtimeparams[
|
|
341
|
-
params[
|
|
500
|
+
publish_prefix = runtimeparams["publish_prefix"]
|
|
501
|
+
params["passed_data"]["publish_prefix"] = publish_prefix
|
|
342
502
|
# Serialize the final params
|
|
343
503
|
params = json.dumps(params)
|
|
344
504
|
return params, retrieve_hass_conf, optim_conf, plant_conf
|
|
345
505
|
|
|
346
|
-
def get_yaml_parse(
|
|
506
|
+
def get_yaml_parse(emhass_conf: dict, use_secrets: Optional[bool] = True,
|
|
347
507
|
params: Optional[str] = None) -> Tuple[dict, dict, dict]:
|
|
348
508
|
"""
|
|
349
509
|
Perform parsing of the config.yaml file.
|
|
350
510
|
|
|
351
|
-
:param
|
|
352
|
-
:type
|
|
511
|
+
:param emhass_conf: Dictionary containing the needed emhass paths
|
|
512
|
+
:type emhass_conf: dict
|
|
353
513
|
:param use_secrets: Indicate if we should use a secrets file or not.
|
|
354
514
|
Set to False for unit tests.
|
|
355
515
|
:type use_secrets: bool, optional
|
|
@@ -359,51 +519,55 @@ def get_yaml_parse(config_path: str, use_secrets: Optional[bool] = True,
|
|
|
359
519
|
:rtype: tuple(dict)
|
|
360
520
|
|
|
361
521
|
"""
|
|
362
|
-
base = config_path.parent
|
|
363
522
|
if params is None:
|
|
364
|
-
with open(config_path, 'r') as file:
|
|
523
|
+
with open(emhass_conf["config_path"], 'r') as file:
|
|
365
524
|
input_conf = yaml.load(file, Loader=yaml.FullLoader)
|
|
366
525
|
else:
|
|
367
526
|
input_conf = json.loads(params)
|
|
368
527
|
if use_secrets:
|
|
369
528
|
if params is None:
|
|
370
|
-
with open(
|
|
529
|
+
with open(emhass_conf["root_path"] / 'secrets_emhass.yaml', 'r') as file: #assume secrets file is in root path
|
|
371
530
|
input_secrets = yaml.load(file, Loader=yaml.FullLoader)
|
|
372
531
|
else:
|
|
373
|
-
input_secrets = input_conf.pop(
|
|
374
|
-
|
|
375
|
-
if
|
|
376
|
-
retrieve_hass_conf = dict(
|
|
532
|
+
input_secrets = input_conf.pop("params_secrets", None)
|
|
533
|
+
|
|
534
|
+
if type(input_conf["retrieve_hass_conf"]) == list: # if using old config version
|
|
535
|
+
retrieve_hass_conf = dict(
|
|
536
|
+
{key: d[key] for d in input_conf["retrieve_hass_conf"] for key in d}
|
|
537
|
+
)
|
|
377
538
|
else:
|
|
378
|
-
retrieve_hass_conf = input_conf.get(
|
|
379
|
-
|
|
539
|
+
retrieve_hass_conf = input_conf.get("retrieve_hass_conf", {})
|
|
540
|
+
|
|
380
541
|
if use_secrets:
|
|
381
542
|
retrieve_hass_conf.update(input_secrets)
|
|
382
543
|
else:
|
|
383
|
-
retrieve_hass_conf[
|
|
384
|
-
retrieve_hass_conf[
|
|
385
|
-
retrieve_hass_conf[
|
|
386
|
-
retrieve_hass_conf[
|
|
387
|
-
retrieve_hass_conf[
|
|
388
|
-
retrieve_hass_conf[
|
|
389
|
-
retrieve_hass_conf[
|
|
390
|
-
retrieve_hass_conf[
|
|
391
|
-
|
|
392
|
-
if
|
|
393
|
-
optim_conf = dict({key:d[key] for d in input_conf[
|
|
544
|
+
retrieve_hass_conf["hass_url"] = "http://supervisor/core/api"
|
|
545
|
+
retrieve_hass_conf["long_lived_token"] = "${SUPERVISOR_TOKEN}"
|
|
546
|
+
retrieve_hass_conf["time_zone"] = "Europe/Paris"
|
|
547
|
+
retrieve_hass_conf["lat"] = 45.83
|
|
548
|
+
retrieve_hass_conf["lon"] = 6.86
|
|
549
|
+
retrieve_hass_conf["alt"] = 4807.8
|
|
550
|
+
retrieve_hass_conf["freq"] = pd.to_timedelta(retrieve_hass_conf["freq"], "minutes")
|
|
551
|
+
retrieve_hass_conf["time_zone"] = pytz.timezone(retrieve_hass_conf["time_zone"])
|
|
552
|
+
|
|
553
|
+
if type(input_conf["optim_conf"]) == list:
|
|
554
|
+
optim_conf = dict({key: d[key] for d in input_conf["optim_conf"] for key in d})
|
|
394
555
|
else:
|
|
395
|
-
optim_conf = input_conf.get(
|
|
556
|
+
optim_conf = input_conf.get("optim_conf", {})
|
|
396
557
|
|
|
397
|
-
optim_conf[
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
558
|
+
optim_conf["list_hp_periods"] = dict(
|
|
559
|
+
(key, d[key]) for d in optim_conf["list_hp_periods"] for key in d
|
|
560
|
+
)
|
|
561
|
+
optim_conf["delta_forecast"] = pd.Timedelta(days=optim_conf["delta_forecast"])
|
|
562
|
+
|
|
563
|
+
if type(input_conf["plant_conf"]) == list:
|
|
564
|
+
plant_conf = dict({key: d[key] for d in input_conf["plant_conf"] for key in d})
|
|
402
565
|
else:
|
|
403
|
-
plant_conf = input_conf.get(
|
|
404
|
-
|
|
566
|
+
plant_conf = input_conf.get("plant_conf", {})
|
|
567
|
+
|
|
405
568
|
return retrieve_hass_conf, optim_conf, plant_conf
|
|
406
569
|
|
|
570
|
+
|
|
407
571
|
def get_injection_dict(df: pd.DataFrame, plot_size: Optional[int] = 1366) -> dict:
|
|
408
572
|
"""
|
|
409
573
|
Build a dictionary with graphs and tables for the webui.
|
|
@@ -414,61 +578,86 @@ def get_injection_dict(df: pd.DataFrame, plot_size: Optional[int] = 1366) -> dic
|
|
|
414
578
|
:type plot_size: Optional[int], optional
|
|
415
579
|
:return: A dictionary containing the graphs and tables in html format
|
|
416
580
|
:rtype: dict
|
|
417
|
-
|
|
581
|
+
|
|
418
582
|
"""
|
|
419
|
-
cols_p = [i for i in df.columns.to_list() if
|
|
583
|
+
cols_p = [i for i in df.columns.to_list() if "P_" in i]
|
|
420
584
|
# Let's round the data in the DF
|
|
421
|
-
optim_status = df[
|
|
422
|
-
df.drop(
|
|
423
|
-
cols_else = [i for i in df.columns.to_list() if
|
|
585
|
+
optim_status = df["optim_status"].unique().item()
|
|
586
|
+
df.drop("optim_status", axis=1, inplace=True)
|
|
587
|
+
cols_else = [i for i in df.columns.to_list() if "P_" not in i]
|
|
424
588
|
df = df.apply(pd.to_numeric)
|
|
425
589
|
df[cols_p] = df[cols_p].astype(int)
|
|
426
590
|
df[cols_else] = df[cols_else].round(3)
|
|
427
591
|
# Create plots
|
|
428
592
|
n_colors = len(cols_p)
|
|
429
|
-
colors = px.colors.sample_colorscale(
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
593
|
+
colors = px.colors.sample_colorscale(
|
|
594
|
+
"jet", [n / (n_colors - 1) for n in range(n_colors)]
|
|
595
|
+
)
|
|
596
|
+
fig_0 = px.line(
|
|
597
|
+
df[cols_p],
|
|
598
|
+
title="Systems powers schedule after optimization results",
|
|
599
|
+
template="presentation",
|
|
600
|
+
line_shape="hv",
|
|
601
|
+
color_discrete_sequence=colors,
|
|
602
|
+
)
|
|
603
|
+
fig_0.update_layout(xaxis_title="Timestamp", yaxis_title="System powers (W)")
|
|
604
|
+
if "SOC_opt" in df.columns.to_list():
|
|
605
|
+
fig_1 = px.line(
|
|
606
|
+
df["SOC_opt"],
|
|
607
|
+
title="Battery state of charge schedule after optimization results",
|
|
608
|
+
template="presentation",
|
|
609
|
+
line_shape="hv",
|
|
610
|
+
color_discrete_sequence=colors,
|
|
611
|
+
)
|
|
612
|
+
fig_1.update_layout(xaxis_title="Timestamp", yaxis_title="Battery SOC (%)")
|
|
613
|
+
cols_cost = [i for i in df.columns.to_list() if "cost_" in i or "unit_" in i]
|
|
440
614
|
n_colors = len(cols_cost)
|
|
441
|
-
colors = px.colors.sample_colorscale(
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
615
|
+
colors = px.colors.sample_colorscale(
|
|
616
|
+
"jet", [n / (n_colors - 1) for n in range(n_colors)]
|
|
617
|
+
)
|
|
618
|
+
fig_2 = px.line(
|
|
619
|
+
df[cols_cost],
|
|
620
|
+
title="Systems costs obtained from optimization results",
|
|
621
|
+
template="presentation",
|
|
622
|
+
line_shape="hv",
|
|
623
|
+
color_discrete_sequence=colors,
|
|
624
|
+
)
|
|
625
|
+
fig_2.update_layout(xaxis_title="Timestamp", yaxis_title="System costs (currency)")
|
|
446
626
|
# Get full path to image
|
|
447
|
-
image_path_0 = fig_0.to_html(full_html=False, default_width=
|
|
448
|
-
if
|
|
449
|
-
image_path_1 = fig_1.to_html(full_html=False, default_width=
|
|
450
|
-
image_path_2 = fig_2.to_html(full_html=False, default_width=
|
|
627
|
+
image_path_0 = fig_0.to_html(full_html=False, default_width="75%")
|
|
628
|
+
if "SOC_opt" in df.columns.to_list():
|
|
629
|
+
image_path_1 = fig_1.to_html(full_html=False, default_width="75%")
|
|
630
|
+
image_path_2 = fig_2.to_html(full_html=False, default_width="75%")
|
|
451
631
|
# The tables
|
|
452
|
-
table1 = df.reset_index().to_html(classes=
|
|
453
|
-
cost_cols = [i for i in df.columns if
|
|
632
|
+
table1 = df.reset_index().to_html(classes="mystyle", index=False)
|
|
633
|
+
cost_cols = [i for i in df.columns if "cost_" in i]
|
|
454
634
|
table2 = df[cost_cols].reset_index().sum(numeric_only=True)
|
|
455
|
-
table2[
|
|
456
|
-
table2 =
|
|
635
|
+
table2["optim_status"] = optim_status
|
|
636
|
+
table2 = (
|
|
637
|
+
table2.to_frame(name="Value")
|
|
638
|
+
.reset_index(names="Variable")
|
|
639
|
+
.to_html(classes="mystyle", index=False)
|
|
640
|
+
)
|
|
457
641
|
# The dict of plots
|
|
458
642
|
injection_dict = {}
|
|
459
|
-
injection_dict[
|
|
460
|
-
injection_dict[
|
|
461
|
-
injection_dict[
|
|
462
|
-
if
|
|
463
|
-
injection_dict[
|
|
464
|
-
injection_dict[
|
|
465
|
-
injection_dict[
|
|
466
|
-
injection_dict[
|
|
467
|
-
injection_dict[
|
|
468
|
-
|
|
643
|
+
injection_dict["title"] = "<h2>EMHASS optimization results</h2>"
|
|
644
|
+
injection_dict["subsubtitle0"] = "<h4>Plotting latest optimization results</h4>"
|
|
645
|
+
injection_dict["figure_0"] = image_path_0
|
|
646
|
+
if "SOC_opt" in df.columns.to_list():
|
|
647
|
+
injection_dict["figure_1"] = image_path_1
|
|
648
|
+
injection_dict["figure_2"] = image_path_2
|
|
649
|
+
injection_dict["subsubtitle1"] = "<h4>Last run optimization results table</h4>"
|
|
650
|
+
injection_dict["table1"] = table1
|
|
651
|
+
injection_dict["subsubtitle2"] = (
|
|
652
|
+
"<h4>Summary table for latest optimization results</h4>"
|
|
653
|
+
)
|
|
654
|
+
injection_dict["table2"] = table2
|
|
469
655
|
return injection_dict
|
|
470
656
|
|
|
471
|
-
|
|
657
|
+
|
|
658
|
+
def get_injection_dict_forecast_model_fit(
|
|
659
|
+
df_fit_pred: pd.DataFrame, mlf: MLForecaster
|
|
660
|
+
) -> dict:
|
|
472
661
|
"""
|
|
473
662
|
Build a dictionary with graphs and tables for the webui for special MLF fit case.
|
|
474
663
|
|
|
@@ -480,19 +669,26 @@ def get_injection_dict_forecast_model_fit(df_fit_pred: pd.DataFrame, mlf: MLFore
|
|
|
480
669
|
:rtype: dict
|
|
481
670
|
"""
|
|
482
671
|
fig = df_fit_pred.plot()
|
|
483
|
-
fig.layout.template =
|
|
484
|
-
fig.update_yaxes(title_text
|
|
485
|
-
fig.update_xaxes(title_text
|
|
486
|
-
image_path_0 = fig.to_html(full_html=False, default_width=
|
|
672
|
+
fig.layout.template = "presentation"
|
|
673
|
+
fig.update_yaxes(title_text=mlf.model_type)
|
|
674
|
+
fig.update_xaxes(title_text="Time")
|
|
675
|
+
image_path_0 = fig.to_html(full_html=False, default_width="75%")
|
|
487
676
|
# The dict of plots
|
|
488
677
|
injection_dict = {}
|
|
489
|
-
injection_dict[
|
|
490
|
-
injection_dict[
|
|
491
|
-
|
|
492
|
-
|
|
678
|
+
injection_dict["title"] = "<h2>Custom machine learning forecast model fit</h2>"
|
|
679
|
+
injection_dict["subsubtitle0"] = (
|
|
680
|
+
"<h4>Plotting train/test forecast model results for " + mlf.model_type + "</h4>"
|
|
681
|
+
)
|
|
682
|
+
injection_dict["subsubtitle0"] = (
|
|
683
|
+
"<h4>Forecasting variable " + mlf.var_model + "</h4>"
|
|
684
|
+
)
|
|
685
|
+
injection_dict["figure_0"] = image_path_0
|
|
493
686
|
return injection_dict
|
|
494
687
|
|
|
495
|
-
|
|
688
|
+
|
|
689
|
+
def get_injection_dict_forecast_model_tune(
|
|
690
|
+
df_pred_optim: pd.DataFrame, mlf: MLForecaster
|
|
691
|
+
) -> dict:
|
|
496
692
|
"""
|
|
497
693
|
Build a dictionary with graphs and tables for the webui for special MLF tune case.
|
|
498
694
|
|
|
@@ -504,19 +700,32 @@ def get_injection_dict_forecast_model_tune(df_pred_optim: pd.DataFrame, mlf: MLF
|
|
|
504
700
|
:rtype: dict
|
|
505
701
|
"""
|
|
506
702
|
fig = df_pred_optim.plot()
|
|
507
|
-
fig.layout.template =
|
|
508
|
-
fig.update_yaxes(title_text
|
|
509
|
-
fig.update_xaxes(title_text
|
|
510
|
-
image_path_0 = fig.to_html(full_html=False, default_width=
|
|
703
|
+
fig.layout.template = "presentation"
|
|
704
|
+
fig.update_yaxes(title_text=mlf.model_type)
|
|
705
|
+
fig.update_xaxes(title_text="Time")
|
|
706
|
+
image_path_0 = fig.to_html(full_html=False, default_width="75%")
|
|
511
707
|
# The dict of plots
|
|
512
708
|
injection_dict = {}
|
|
513
|
-
injection_dict[
|
|
514
|
-
injection_dict[
|
|
515
|
-
|
|
516
|
-
|
|
709
|
+
injection_dict["title"] = "<h2>Custom machine learning forecast model tune</h2>"
|
|
710
|
+
injection_dict["subsubtitle0"] = (
|
|
711
|
+
"<h4>Performed a tuning routine using bayesian optimization for "
|
|
712
|
+
+ mlf.model_type
|
|
713
|
+
+ "</h4>"
|
|
714
|
+
)
|
|
715
|
+
injection_dict["subsubtitle0"] = (
|
|
716
|
+
"<h4>Forecasting variable " + mlf.var_model + "</h4>"
|
|
717
|
+
)
|
|
718
|
+
injection_dict["figure_0"] = image_path_0
|
|
517
719
|
return injection_dict
|
|
518
720
|
|
|
519
|
-
|
|
721
|
+
|
|
722
|
+
def build_params(
|
|
723
|
+
params: dict,
|
|
724
|
+
params_secrets: dict,
|
|
725
|
+
options: dict,
|
|
726
|
+
addon: int,
|
|
727
|
+
logger: logging.Logger,
|
|
728
|
+
) -> dict:
|
|
520
729
|
"""
|
|
521
730
|
Build the main params dictionary from the loaded options.json when using the add-on.
|
|
522
731
|
|
|
@@ -535,45 +744,120 @@ def build_params(params: dict, params_secrets: dict, options: dict, addon: int,
|
|
|
535
744
|
"""
|
|
536
745
|
if addon == 1:
|
|
537
746
|
# Updating variables in retrieve_hass_conf
|
|
538
|
-
params[
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
params[
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
params[
|
|
546
|
-
|
|
747
|
+
params["retrieve_hass_conf"]["freq"] = options.get(
|
|
748
|
+
"optimization_time_step", params["retrieve_hass_conf"]["freq"]
|
|
749
|
+
)
|
|
750
|
+
params["retrieve_hass_conf"]["days_to_retrieve"] = options.get(
|
|
751
|
+
"historic_days_to_retrieve",
|
|
752
|
+
params["retrieve_hass_conf"]["days_to_retrieve"],
|
|
753
|
+
)
|
|
754
|
+
params["retrieve_hass_conf"]["var_PV"] = options.get(
|
|
755
|
+
"sensor_power_photovoltaics", params["retrieve_hass_conf"]["var_PV"]
|
|
756
|
+
)
|
|
757
|
+
params["retrieve_hass_conf"]["var_load"] = options.get(
|
|
758
|
+
"sensor_power_load_no_var_loads", params["retrieve_hass_conf"]["var_load"]
|
|
759
|
+
)
|
|
760
|
+
params["retrieve_hass_conf"]["load_negative"] = options.get(
|
|
761
|
+
"load_negative", params["retrieve_hass_conf"]["load_negative"]
|
|
762
|
+
)
|
|
763
|
+
params["retrieve_hass_conf"]["set_zero_min"] = options.get(
|
|
764
|
+
"set_zero_min", params["retrieve_hass_conf"]["set_zero_min"]
|
|
765
|
+
)
|
|
766
|
+
params["retrieve_hass_conf"]["var_replace_zero"] = [
|
|
767
|
+
options.get(
|
|
768
|
+
"sensor_power_photovoltaics",
|
|
769
|
+
params["retrieve_hass_conf"]["var_replace_zero"],
|
|
770
|
+
)
|
|
771
|
+
]
|
|
772
|
+
params["retrieve_hass_conf"]["var_interp"] = [
|
|
773
|
+
options.get(
|
|
774
|
+
"sensor_power_photovoltaics", params["retrieve_hass_conf"]["var_PV"]
|
|
775
|
+
),
|
|
776
|
+
options.get(
|
|
777
|
+
"sensor_power_load_no_var_loads",
|
|
778
|
+
params["retrieve_hass_conf"]["var_load"],
|
|
779
|
+
),
|
|
780
|
+
]
|
|
781
|
+
params["retrieve_hass_conf"]["method_ts_round"] = options.get(
|
|
782
|
+
"method_ts_round", params["retrieve_hass_conf"]["method_ts_round"]
|
|
783
|
+
)
|
|
547
784
|
# Update params Secrets if specified
|
|
548
|
-
params[
|
|
549
|
-
params[
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
params[
|
|
785
|
+
params["params_secrets"] = params_secrets
|
|
786
|
+
params["params_secrets"]["time_zone"] = options.get(
|
|
787
|
+
"time_zone", params_secrets["time_zone"]
|
|
788
|
+
)
|
|
789
|
+
params["params_secrets"]["lat"] = options.get("Latitude", params_secrets["lat"])
|
|
790
|
+
params["params_secrets"]["lon"] = options.get(
|
|
791
|
+
"Longitude", params_secrets["lon"]
|
|
792
|
+
)
|
|
793
|
+
params["params_secrets"]["alt"] = options.get("Altitude", params_secrets["alt"])
|
|
553
794
|
# Updating variables in optim_conf
|
|
554
|
-
params[
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
if options.get(
|
|
561
|
-
params[
|
|
562
|
-
|
|
795
|
+
params["optim_conf"]["set_use_battery"] = options.get(
|
|
796
|
+
"set_use_battery", params["optim_conf"]["set_use_battery"]
|
|
797
|
+
)
|
|
798
|
+
params["optim_conf"]["num_def_loads"] = options.get(
|
|
799
|
+
"number_of_deferrable_loads", params["optim_conf"]["num_def_loads"]
|
|
800
|
+
)
|
|
801
|
+
if options.get("list_nominal_power_of_deferrable_loads", None) != None:
|
|
802
|
+
params["optim_conf"]["P_deferrable_nom"] = [
|
|
803
|
+
i["nominal_power_of_deferrable_loads"]
|
|
804
|
+
for i in options.get("list_nominal_power_of_deferrable_loads")
|
|
805
|
+
]
|
|
806
|
+
if options.get("list_operating_hours_of_each_deferrable_load", None) != None:
|
|
807
|
+
params["optim_conf"]["def_total_hours"] = [
|
|
808
|
+
i["operating_hours_of_each_deferrable_load"]
|
|
809
|
+
for i in options.get("list_operating_hours_of_each_deferrable_load")
|
|
810
|
+
]
|
|
811
|
+
if options.get("list_treat_deferrable_load_as_semi_cont", None) != None:
|
|
812
|
+
params["optim_conf"]["treat_def_as_semi_cont"] = [
|
|
813
|
+
i["treat_deferrable_load_as_semi_cont"]
|
|
814
|
+
for i in options.get("list_treat_deferrable_load_as_semi_cont")
|
|
815
|
+
]
|
|
816
|
+
params["optim_conf"]["weather_forecast_method"] = options.get(
|
|
817
|
+
"weather_forecast_method", params["optim_conf"]["weather_forecast_method"]
|
|
818
|
+
)
|
|
563
819
|
# Update optional param secrets
|
|
564
|
-
if params[
|
|
565
|
-
params[
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
820
|
+
if params["optim_conf"]["weather_forecast_method"] == "solcast":
|
|
821
|
+
params["params_secrets"]["solcast_api_key"] = options.get(
|
|
822
|
+
"optional_solcast_api_key",
|
|
823
|
+
params_secrets.get("solcast_api_key", "123456"),
|
|
824
|
+
)
|
|
825
|
+
params["params_secrets"]["solcast_rooftop_id"] = options.get(
|
|
826
|
+
"optional_solcast_rooftop_id",
|
|
827
|
+
params_secrets.get("solcast_rooftop_id", "123456"),
|
|
828
|
+
)
|
|
829
|
+
elif params["optim_conf"]["weather_forecast_method"] == "solar.forecast":
|
|
830
|
+
params["params_secrets"]["solar_forecast_kwp"] = options.get(
|
|
831
|
+
"optional_solar_forecast_kwp",
|
|
832
|
+
params_secrets.get("solar_forecast_kwp", 5),
|
|
833
|
+
)
|
|
834
|
+
params["optim_conf"]["load_forecast_method"] = options.get(
|
|
835
|
+
"load_forecast_method", params["optim_conf"]["load_forecast_method"]
|
|
836
|
+
)
|
|
837
|
+
params["optim_conf"]["delta_forecast"] = options.get(
|
|
838
|
+
"delta_forecast_daily", params["optim_conf"]["delta_forecast"]
|
|
839
|
+
)
|
|
840
|
+
params["optim_conf"]["load_cost_forecast_method"] = options.get(
|
|
841
|
+
"load_cost_forecast_method",
|
|
842
|
+
params["optim_conf"]["load_cost_forecast_method"],
|
|
843
|
+
)
|
|
844
|
+
if options.get("list_set_deferrable_load_single_constant", None) != None:
|
|
845
|
+
params["optim_conf"]["set_def_constant"] = [
|
|
846
|
+
i["set_deferrable_load_single_constant"]
|
|
847
|
+
for i in options.get("list_set_deferrable_load_single_constant")
|
|
848
|
+
]
|
|
849
|
+
if (
|
|
850
|
+
options.get("list_peak_hours_periods_start_hours", None) != None
|
|
851
|
+
and options.get("list_peak_hours_periods_end_hours", None) != None
|
|
852
|
+
):
|
|
853
|
+
start_hours_list = [
|
|
854
|
+
i["peak_hours_periods_start_hours"]
|
|
855
|
+
for i in options["list_peak_hours_periods_start_hours"]
|
|
856
|
+
]
|
|
857
|
+
end_hours_list = [
|
|
858
|
+
i["peak_hours_periods_end_hours"]
|
|
859
|
+
for i in options["list_peak_hours_periods_end_hours"]
|
|
860
|
+
]
|
|
577
861
|
num_peak_hours = len(start_hours_list)
|
|
578
862
|
list_hp_periods_list = [{'period_hp_'+str(i+1):[{'start':start_hours_list[i]},{'end':end_hours_list[i]}]} for i in range(num_peak_hours)]
|
|
579
863
|
params['optim_conf']['list_hp_periods'] = list_hp_periods_list
|
|
@@ -645,20 +929,35 @@ def build_params(params: dict, params_secrets: dict, options: dict, addon: int,
|
|
|
645
929
|
for x in range(len(params['optim_conf']['P_deferrable_nom']), params['optim_conf']['num_def_loads']):
|
|
646
930
|
params['optim_conf']['P_deferrable_nom'].append(0)
|
|
647
931
|
# days_to_retrieve should be no less then 2
|
|
648
|
-
if params[
|
|
649
|
-
params[
|
|
650
|
-
logger.warning(
|
|
932
|
+
if params["retrieve_hass_conf"]["days_to_retrieve"] < 2:
|
|
933
|
+
params["retrieve_hass_conf"]["days_to_retrieve"] = 2
|
|
934
|
+
logger.warning(
|
|
935
|
+
"days_to_retrieve should not be lower then 2, setting days_to_retrieve to 2. Make sure your sensors also have at least 2 days of history"
|
|
936
|
+
)
|
|
651
937
|
else:
|
|
652
|
-
params[
|
|
938
|
+
params["params_secrets"] = params_secrets
|
|
653
939
|
# The params dict
|
|
654
|
-
params[
|
|
655
|
-
|
|
940
|
+
params["passed_data"] = {
|
|
941
|
+
"pv_power_forecast": None,
|
|
942
|
+
"load_power_forecast": None,
|
|
943
|
+
"load_cost_forecast": None,
|
|
944
|
+
"prod_price_forecast": None,
|
|
945
|
+
"prediction_horizon": None,
|
|
946
|
+
"soc_init": None,
|
|
947
|
+
"soc_final": None,
|
|
948
|
+
"def_total_hours": None,
|
|
949
|
+
"def_start_timestep": None,
|
|
950
|
+
"def_end_timestep": None,
|
|
951
|
+
"alpha": None,
|
|
952
|
+
"beta": None,
|
|
953
|
+
}
|
|
656
954
|
return params
|
|
657
955
|
|
|
956
|
+
|
|
658
957
|
def get_days_list(days_to_retrieve: int) -> pd.date_range:
|
|
659
958
|
"""
|
|
660
959
|
Get list of past days from today to days_to_retrieve.
|
|
661
|
-
|
|
960
|
+
|
|
662
961
|
:param days_to_retrieve: Total number of days to retrieve from the past
|
|
663
962
|
:type days_to_retrieve: int
|
|
664
963
|
:return: The list of days
|
|
@@ -667,19 +966,20 @@ def get_days_list(days_to_retrieve: int) -> pd.date_range:
|
|
|
667
966
|
"""
|
|
668
967
|
today = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0)
|
|
669
968
|
d = (today - timedelta(days=days_to_retrieve)).isoformat()
|
|
670
|
-
days_list = pd.date_range(start=d, end=today.isoformat(), freq=
|
|
671
|
-
|
|
969
|
+
days_list = pd.date_range(start=d, end=today.isoformat(), freq="D")
|
|
970
|
+
|
|
672
971
|
return days_list
|
|
673
972
|
|
|
973
|
+
|
|
674
974
|
def set_df_index_freq(df: pd.DataFrame) -> pd.DataFrame:
|
|
675
975
|
"""
|
|
676
976
|
Set the freq of a DataFrame DateTimeIndex.
|
|
677
|
-
|
|
977
|
+
|
|
678
978
|
:param df: Input DataFrame
|
|
679
979
|
:type df: pd.DataFrame
|
|
680
980
|
:return: Input DataFrame with freq defined
|
|
681
981
|
:rtype: pd.DataFrame
|
|
682
|
-
|
|
982
|
+
|
|
683
983
|
"""
|
|
684
984
|
idx_diff = np.diff(df.index)
|
|
685
985
|
sampling = pd.to_timedelta(np.median(idx_diff))
|