emhass 0.11.2__py3-none-any.whl → 0.11.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- emhass/command_line.py +702 -373
- emhass/data/associations.csv +1 -1
- emhass/forecast.py +671 -346
- emhass/machine_learning_forecaster.py +204 -105
- emhass/machine_learning_regressor.py +26 -7
- emhass/optimization.py +1017 -471
- emhass/retrieve_hass.py +226 -79
- emhass/static/data/param_definitions.json +5 -4
- emhass/utils.py +687 -443
- emhass/web_server.py +339 -232
- {emhass-0.11.2.dist-info → emhass-0.11.4.dist-info}/METADATA +17 -8
- {emhass-0.11.2.dist-info → emhass-0.11.4.dist-info}/RECORD +16 -16
- {emhass-0.11.2.dist-info → emhass-0.11.4.dist-info}/WHEEL +1 -1
- {emhass-0.11.2.dist-info → emhass-0.11.4.dist-info}/LICENSE +0 -0
- {emhass-0.11.2.dist-info → emhass-0.11.4.dist-info}/entry_points.txt +0 -0
- {emhass-0.11.2.dist-info → emhass-0.11.4.dist-info}/top_level.txt +0 -0
emhass/web_server.py
CHANGED
@@ -1,51 +1,79 @@
|
|
1
1
|
#!/usr/bin/env python3
|
2
2
|
# -*- coding: utf-8 -*-
|
3
3
|
|
4
|
-
|
4
|
+
import argparse
|
5
|
+
import json
|
6
|
+
import logging
|
7
|
+
import os
|
8
|
+
import pickle
|
9
|
+
import re
|
10
|
+
import threading
|
11
|
+
from distutils.util import strtobool
|
12
|
+
from importlib.metadata import PackageNotFoundError, version
|
13
|
+
from pathlib import Path
|
14
|
+
|
15
|
+
import yaml
|
16
|
+
from flask import Flask, make_response, request
|
17
|
+
from flask import logging as log
|
5
18
|
from jinja2 import Environment, PackageLoader
|
6
|
-
from requests import get
|
7
19
|
from waitress import serve
|
8
|
-
from importlib.metadata import version, PackageNotFoundError
|
9
|
-
from pathlib import Path
|
10
|
-
import os, json, argparse, pickle, yaml, logging, re, threading
|
11
|
-
from distutils.util import strtobool
|
12
20
|
|
13
|
-
from emhass.command_line import
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
+
from emhass.command_line import (
|
22
|
+
continual_publish,
|
23
|
+
dayahead_forecast_optim,
|
24
|
+
forecast_model_fit,
|
25
|
+
forecast_model_predict,
|
26
|
+
forecast_model_tune,
|
27
|
+
naive_mpc_optim,
|
28
|
+
perfect_forecast_optim,
|
29
|
+
publish_data,
|
30
|
+
regressor_model_fit,
|
31
|
+
regressor_model_predict,
|
32
|
+
set_input_data_dict,
|
33
|
+
weather_forecast_cache,
|
34
|
+
)
|
35
|
+
from emhass.utils import (
|
36
|
+
build_config,
|
37
|
+
build_legacy_config_params,
|
38
|
+
build_params,
|
39
|
+
build_secrets,
|
40
|
+
get_injection_dict,
|
41
|
+
get_injection_dict_forecast_model_fit,
|
42
|
+
get_injection_dict_forecast_model_tune,
|
43
|
+
param_to_config,
|
44
|
+
)
|
21
45
|
|
22
46
|
# Define the Flask instance
|
23
47
|
app = Flask(__name__)
|
24
48
|
emhass_conf = {}
|
25
49
|
|
50
|
+
|
26
51
|
def checkFileLog(refString=None) -> bool:
|
27
52
|
"""
|
28
53
|
Check logfile for error, anything after string match if provided.
|
29
54
|
|
30
|
-
:param refString: String to reduce log area to check for errors. Use to reduce log to check anything after string match (ie. an action).
|
55
|
+
:param refString: String to reduce log area to check for errors. Use to reduce log to check anything after string match (ie. an action).
|
31
56
|
:type refString: str
|
32
57
|
:return: Boolean return if error was found in logs
|
33
58
|
:rtype: bool
|
34
59
|
|
35
60
|
"""
|
36
|
-
if
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
61
|
+
if refString is not None:
|
62
|
+
logArray = grabLog(
|
63
|
+
refString
|
64
|
+
) # grab reduced log array (everything after string match)
|
65
|
+
else:
|
66
|
+
if (emhass_conf["data_path"] / "actionLogs.txt").exists():
|
67
|
+
with open(str(emhass_conf["data_path"] / "actionLogs.txt"), "r") as fp:
|
68
|
+
logArray = fp.readlines()
|
42
69
|
else:
|
43
70
|
app.logger.debug("Unable to obtain actionLogs.txt")
|
44
71
|
for logString in logArray:
|
45
|
-
|
46
|
-
|
72
|
+
if logString.split(" ", 1)[0] == "ERROR":
|
73
|
+
return True
|
47
74
|
return False
|
48
75
|
|
76
|
+
|
49
77
|
def grabLog(refString) -> list:
|
50
78
|
"""
|
51
79
|
Find string in logs, append all lines after into list to return.
|
@@ -58,31 +86,33 @@ def grabLog(refString) -> list:
|
|
58
86
|
"""
|
59
87
|
isFound = []
|
60
88
|
output = []
|
61
|
-
if (
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
89
|
+
if (emhass_conf["data_path"] / "actionLogs.txt").exists():
|
90
|
+
with open(str(emhass_conf["data_path"] / "actionLogs.txt"), "r") as fp:
|
91
|
+
logArray = fp.readlines()
|
92
|
+
# Find all string matches, log key (line Number) in isFound
|
93
|
+
for x in range(len(logArray) - 1):
|
94
|
+
if re.search(refString, logArray[x]):
|
95
|
+
isFound.append(x)
|
96
|
+
if len(isFound) != 0:
|
97
|
+
# Use last item in isFound to extract action logs
|
98
|
+
for x in range(isFound[-1], len(logArray)):
|
99
|
+
output.append(logArray[x])
|
72
100
|
return output
|
73
101
|
|
102
|
+
|
74
103
|
# Clear the log file
|
75
104
|
def clearFileLog():
|
76
105
|
"""
|
77
106
|
Clear the contents of the log file (actionLogs.txt)
|
78
107
|
|
79
108
|
"""
|
80
|
-
if (
|
81
|
-
with open(str(emhass_conf[
|
82
|
-
fp.truncate()
|
109
|
+
if (emhass_conf["data_path"] / "actionLogs.txt").exists():
|
110
|
+
with open(str(emhass_conf["data_path"] / "actionLogs.txt"), "w") as fp:
|
111
|
+
fp.truncate()
|
83
112
|
|
84
|
-
|
85
|
-
@app.route(
|
113
|
+
|
114
|
+
@app.route("/")
|
115
|
+
@app.route("/index")
|
86
116
|
def index():
|
87
117
|
"""
|
88
118
|
Render initial index page and serve to web server.
|
@@ -91,29 +121,31 @@ def index():
|
|
91
121
|
"""
|
92
122
|
app.logger.info("EMHASS server online, serving index.html...")
|
93
123
|
# Load HTML template
|
94
|
-
file_loader = PackageLoader(
|
124
|
+
file_loader = PackageLoader("emhass", "templates")
|
95
125
|
env = Environment(loader=file_loader)
|
96
|
-
#check if index.html exists
|
97
|
-
if
|
126
|
+
# check if index.html exists
|
127
|
+
if "index.html" not in env.list_templates():
|
98
128
|
app.logger.error("Unable to find index.html in emhass module")
|
99
|
-
return make_response(["ERROR: unable to find index.html in emhass module"],404)
|
100
|
-
template = env.get_template(
|
129
|
+
return make_response(["ERROR: unable to find index.html in emhass module"], 404)
|
130
|
+
template = env.get_template("index.html")
|
101
131
|
# Load cached dict (if exists), to present generated plot tables
|
102
|
-
if (emhass_conf[
|
103
|
-
with open(str(emhass_conf[
|
132
|
+
if (emhass_conf["data_path"] / "injection_dict.pkl").exists():
|
133
|
+
with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "rb") as fid:
|
104
134
|
injection_dict = pickle.load(fid)
|
105
135
|
else:
|
106
|
-
app.logger.info(
|
107
|
-
|
136
|
+
app.logger.info(
|
137
|
+
"The data container dictionary is empty... Please launch an optimization task"
|
138
|
+
)
|
139
|
+
injection_dict = {}
|
108
140
|
|
109
|
-
# replace {{basename}} in html template html with path root
|
141
|
+
# replace {{basename}} in html template html with path root
|
110
142
|
# basename = request.headers.get("X-Ingress-Path", "")
|
111
143
|
# return make_response(template.render(injection_dict=injection_dict, basename=basename))
|
112
|
-
|
144
|
+
|
113
145
|
return make_response(template.render(injection_dict=injection_dict))
|
114
146
|
|
115
147
|
|
116
|
-
@app.route(
|
148
|
+
@app.route("/configuration")
|
117
149
|
def configuration():
|
118
150
|
"""
|
119
151
|
Configuration page actions:
|
@@ -122,41 +154,46 @@ def configuration():
|
|
122
154
|
"""
|
123
155
|
app.logger.info("serving configuration.html...")
|
124
156
|
# Load HTML template
|
125
|
-
file_loader = PackageLoader(
|
157
|
+
file_loader = PackageLoader("emhass", "templates")
|
126
158
|
env = Environment(loader=file_loader)
|
127
|
-
#check if configuration.html exists
|
128
|
-
if
|
159
|
+
# check if configuration.html exists
|
160
|
+
if "configuration.html" not in env.list_templates():
|
129
161
|
app.logger.error("Unable to find configuration.html in emhass module")
|
130
|
-
return make_response(
|
131
|
-
|
162
|
+
return make_response(
|
163
|
+
["ERROR: unable to find configuration.html in emhass module"], 404
|
164
|
+
)
|
165
|
+
template = env.get_template("configuration.html")
|
132
166
|
return make_response(template.render(config=params))
|
133
167
|
|
134
168
|
|
135
|
-
@app.route(
|
169
|
+
@app.route("/template", methods=["GET"])
|
136
170
|
def template_action():
|
137
171
|
"""
|
138
|
-
template page actions:
|
172
|
+
template page actions:
|
139
173
|
Render and serve template html
|
140
174
|
|
141
175
|
"""
|
142
176
|
app.logger.info(" >> Sending rendered template table data")
|
143
|
-
file_loader = PackageLoader(
|
177
|
+
file_loader = PackageLoader("emhass", "templates")
|
144
178
|
env = Environment(loader=file_loader)
|
145
179
|
# Check if template.html exists
|
146
|
-
if
|
180
|
+
if "template.html" not in env.list_templates():
|
147
181
|
app.logger.error("Unable to find template.html in emhass module")
|
148
|
-
return make_response(
|
149
|
-
|
150
|
-
|
151
|
-
|
182
|
+
return make_response(
|
183
|
+
["WARNING: unable to find template.html in emhass module"], 404
|
184
|
+
)
|
185
|
+
template = env.get_template("template.html")
|
186
|
+
if (emhass_conf["data_path"] / "injection_dict.pkl").exists():
|
187
|
+
with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "rb") as fid:
|
152
188
|
injection_dict = pickle.load(fid)
|
153
189
|
else:
|
154
190
|
app.logger.warning("Unable to obtain plot data from injection_dict.pkl")
|
155
191
|
app.logger.warning("Try running an launch an optimization task")
|
156
|
-
injection_dict={}
|
192
|
+
injection_dict = {}
|
157
193
|
return make_response(template.render(injection_dict=injection_dict))
|
158
194
|
|
159
|
-
|
195
|
+
|
196
|
+
@app.route("/get-config", methods=["GET"])
|
160
197
|
def parameter_get():
|
161
198
|
"""
|
162
199
|
Get request action that builds, formats and sends config as json (config.json format)
|
@@ -164,21 +201,27 @@ def parameter_get():
|
|
164
201
|
"""
|
165
202
|
app.logger.debug("Obtaining current saved parameters as config")
|
166
203
|
# Build config from all possible sources (inc. legacy yaml config)
|
167
|
-
config = build_config(
|
204
|
+
config = build_config(
|
205
|
+
emhass_conf,
|
206
|
+
app.logger,
|
207
|
+
emhass_conf["defaults_path"],
|
208
|
+
emhass_conf["config_path"],
|
209
|
+
emhass_conf["legacy_config_path"],
|
210
|
+
)
|
168
211
|
if type(config) is bool and not config:
|
169
|
-
return make_response(["failed to retrieve default config file"],500)
|
212
|
+
return make_response(["failed to retrieve default config file"], 500)
|
170
213
|
# Format parameters in config with params (converting legacy json parameters from options.json if any)
|
171
|
-
params = build_params(emhass_conf,{},config,app.logger)
|
214
|
+
params = build_params(emhass_conf, {}, config, app.logger)
|
172
215
|
if type(params) is bool and not params:
|
173
|
-
return make_response(["Unable to obtain associations file"],500)
|
216
|
+
return make_response(["Unable to obtain associations file"], 500)
|
174
217
|
# Covert formatted parameters from params back into config.json format
|
175
|
-
return_config = param_to_config(params,app.logger)
|
218
|
+
return_config = param_to_config(params, app.logger)
|
176
219
|
# Send config
|
177
|
-
return make_response(return_config,201)
|
220
|
+
return make_response(return_config, 201)
|
178
221
|
|
179
222
|
|
180
223
|
# Get default Config
|
181
|
-
@app.route(
|
224
|
+
@app.route("/get-config/defaults", methods=["GET"])
|
182
225
|
def config_get():
|
183
226
|
"""
|
184
227
|
Get request action, retrieves and sends default configuration
|
@@ -186,21 +229,21 @@ def config_get():
|
|
186
229
|
"""
|
187
230
|
app.logger.debug("Obtaining default parameters")
|
188
231
|
# Build config, passing only default file
|
189
|
-
config = build_config(emhass_conf,app.logger,emhass_conf["defaults_path"])
|
232
|
+
config = build_config(emhass_conf, app.logger, emhass_conf["defaults_path"])
|
190
233
|
if type(config) is bool and not config:
|
191
|
-
return make_response(["failed to retrieve default config file"],500)
|
234
|
+
return make_response(["failed to retrieve default config file"], 500)
|
192
235
|
# Format parameters in config with params
|
193
|
-
params = build_params(emhass_conf,{},config,app.logger)
|
236
|
+
params = build_params(emhass_conf, {}, config, app.logger)
|
194
237
|
if type(params) is bool and not params:
|
195
|
-
return make_response(["Unable to obtain associations file"],500)
|
238
|
+
return make_response(["Unable to obtain associations file"], 500)
|
196
239
|
# Covert formatted parameters from params back into config.json format
|
197
|
-
return_config = param_to_config(params,app.logger)
|
240
|
+
return_config = param_to_config(params, app.logger)
|
198
241
|
# Send params
|
199
|
-
return make_response(return_config,201)
|
242
|
+
return make_response(return_config, 201)
|
200
243
|
|
201
244
|
|
202
245
|
# Get YAML-to-JSON config
|
203
|
-
@app.route(
|
246
|
+
@app.route("/get-json", methods=["POST"])
|
204
247
|
def json_convert():
|
205
248
|
"""
|
206
249
|
Post request action, receives yaml config (config_emhass.yaml or EMHASS-Add-on config page) and converts to config json format.
|
@@ -212,268 +255,317 @@ def json_convert():
|
|
212
255
|
|
213
256
|
# If filed to Parse YAML
|
214
257
|
if yaml_config is None:
|
215
|
-
return make_response(["failed to Parse YAML from data"],400)
|
258
|
+
return make_response(["failed to Parse YAML from data"], 400)
|
216
259
|
# Test YAML is legacy config format (from config_emhass.yaml)
|
217
|
-
test_legacy_config = build_legacy_config_params(
|
260
|
+
test_legacy_config = build_legacy_config_params(
|
261
|
+
emhass_conf, yaml_config, app.logger
|
262
|
+
)
|
218
263
|
if test_legacy_config:
|
219
264
|
yaml_config = test_legacy_config
|
220
265
|
# Format YAML to params (format params. check if params match legacy option.json format)
|
221
|
-
params = build_params(emhass_conf,{},yaml_config,app.logger)
|
266
|
+
params = build_params(emhass_conf, {}, yaml_config, app.logger)
|
222
267
|
if type(params) is bool and not params:
|
223
|
-
return make_response(["Unable to obtain associations file"],500)
|
268
|
+
return make_response(["Unable to obtain associations file"], 500)
|
224
269
|
# Covert formatted parameters from params back into config.json format
|
225
|
-
config = param_to_config(params,app.logger)
|
270
|
+
config = param_to_config(params, app.logger)
|
226
271
|
# convert json to str
|
227
272
|
config = json.dumps(config)
|
228
273
|
|
229
274
|
# Send params
|
230
|
-
return make_response(config,201)
|
275
|
+
return make_response(config, 201)
|
276
|
+
|
231
277
|
|
232
|
-
@app.route(
|
278
|
+
@app.route("/set-config", methods=["POST"])
|
233
279
|
def parameter_set():
|
234
280
|
"""
|
235
281
|
Receive JSON config, and save config to file (config.json and param.pkl)
|
236
282
|
|
237
283
|
"""
|
238
284
|
config = {}
|
239
|
-
if not emhass_conf[
|
240
|
-
return make_response(["Unable to Obtain defaults_path from emhass_conf"],500)
|
241
|
-
if not emhass_conf[
|
242
|
-
return make_response(["Unable to Obtain config_path from emhass_conf"],500)
|
243
|
-
|
285
|
+
if not emhass_conf["defaults_path"]:
|
286
|
+
return make_response(["Unable to Obtain defaults_path from emhass_conf"], 500)
|
287
|
+
if not emhass_conf["config_path"]:
|
288
|
+
return make_response(["Unable to Obtain config_path from emhass_conf"], 500)
|
289
|
+
|
244
290
|
# Load defaults as a reference point (for sorting) and a base to override
|
245
|
-
if
|
246
|
-
|
291
|
+
if (
|
292
|
+
os.path.exists(emhass_conf["defaults_path"])
|
293
|
+
and Path(emhass_conf["defaults_path"]).is_file()
|
294
|
+
):
|
295
|
+
with emhass_conf["defaults_path"].open("r") as data:
|
247
296
|
config = json.load(data)
|
248
297
|
else:
|
249
|
-
app.logger.warning(
|
298
|
+
app.logger.warning(
|
299
|
+
"Unable to obtain default config. only parameters passed from request will be saved to config.json"
|
300
|
+
)
|
250
301
|
|
251
302
|
# Retrieve sent config json
|
252
303
|
request_data = request.get_json(force=True)
|
253
304
|
|
254
305
|
# check if data is empty
|
255
306
|
if len(request_data) == 0:
|
256
|
-
return make_response(["failed to retrieve config json"],400)
|
257
|
-
|
307
|
+
return make_response(["failed to retrieve config json"], 400)
|
308
|
+
|
258
309
|
# Format config by converting to params (format params. check if params match legacy option.json format. If so format)
|
259
|
-
params = build_params(emhass_conf,params_secrets,request_data,app.logger)
|
310
|
+
params = build_params(emhass_conf, params_secrets, request_data, app.logger)
|
260
311
|
if type(params) is bool and not params:
|
261
|
-
return make_response(["Unable to obtain associations file"],500)
|
262
|
-
|
312
|
+
return make_response(["Unable to obtain associations file"], 500)
|
313
|
+
|
263
314
|
# Covert formatted parameters from params back into config.json format.
|
264
315
|
# Overwrite existing default parameters in config
|
265
|
-
config.update(param_to_config(params,app.logger))
|
316
|
+
config.update(param_to_config(params, app.logger))
|
266
317
|
|
267
318
|
# Save config to config.json
|
268
|
-
if os.path.exists(emhass_conf[
|
269
|
-
with emhass_conf[
|
319
|
+
if os.path.exists(emhass_conf["config_path"].parent):
|
320
|
+
with emhass_conf["config_path"].open("w") as f:
|
270
321
|
json.dump(config, f, indent=4)
|
271
|
-
else:
|
272
|
-
return make_response(["Unable to save config file"],500)
|
322
|
+
else:
|
323
|
+
return make_response(["Unable to save config file"], 500)
|
273
324
|
request_data
|
274
325
|
|
275
|
-
# Save params with updated config
|
276
|
-
if os.path.exists(emhass_conf[
|
277
|
-
with open(str(emhass_conf[
|
278
|
-
pickle.dump(
|
279
|
-
|
280
|
-
|
281
|
-
|
326
|
+
# Save params with updated config
|
327
|
+
if os.path.exists(emhass_conf["data_path"]):
|
328
|
+
with open(str(emhass_conf["data_path"] / "params.pkl"), "wb") as fid:
|
329
|
+
pickle.dump(
|
330
|
+
(
|
331
|
+
config_path,
|
332
|
+
build_params(emhass_conf, params_secrets, config, app.logger),
|
333
|
+
),
|
334
|
+
fid,
|
335
|
+
)
|
336
|
+
else:
|
337
|
+
return make_response(["Unable to save params file, missing data_path"], 500)
|
338
|
+
|
282
339
|
app.logger.info("Saved parameters from webserver")
|
283
|
-
return make_response({},201)
|
340
|
+
return make_response({}, 201)
|
284
341
|
|
285
|
-
|
342
|
+
|
343
|
+
@app.route("/action/<action_name>", methods=["POST"])
|
286
344
|
def action_call(action_name):
|
287
345
|
"""
|
288
346
|
Receive Post action, run action according to passed slug(action_name) (e.g. /action/publish-data)
|
289
347
|
|
290
348
|
:param action_name: Slug/Action string corresponding to which action to take
|
291
349
|
:type action_name: String
|
292
|
-
|
350
|
+
|
293
351
|
"""
|
294
352
|
# Setting up parameters
|
295
353
|
# Params
|
296
|
-
|
297
|
-
|
298
|
-
|
354
|
+
ActionStr = " >> Obtaining params: "
|
355
|
+
app.logger.info(ActionStr)
|
356
|
+
if (emhass_conf["data_path"] / "params.pkl").exists():
|
357
|
+
with open(str(emhass_conf["data_path"] / "params.pkl"), "rb") as fid:
|
358
|
+
emhass_conf["config_path"], params = pickle.load(fid)
|
359
|
+
# Set local costfun variable
|
360
|
+
if params.get("optim_conf", None) is not None:
|
361
|
+
costfun = params["optim_conf"].get("costfun", "profit")
|
299
362
|
params = json.dumps(params)
|
300
363
|
else:
|
301
364
|
app.logger.error("Unable to find params.pkl file")
|
302
365
|
return make_response(grabLog(ActionStr), 400)
|
303
366
|
# Runtime
|
304
|
-
runtimeparams = request.get_json(force=True,silent=True)
|
367
|
+
runtimeparams = request.get_json(force=True, silent=True)
|
305
368
|
if runtimeparams is not None:
|
306
|
-
if runtimeparams !=
|
369
|
+
if runtimeparams != "{}":
|
307
370
|
app.logger.info("Passed runtime parameters: " + str(runtimeparams))
|
308
371
|
else:
|
309
372
|
app.logger.warning("Unable to parse runtime parameters")
|
310
|
-
runtimeparams = {}
|
373
|
+
runtimeparams = {}
|
311
374
|
runtimeparams = json.dumps(runtimeparams)
|
312
375
|
|
313
376
|
# weather-forecast-cache (check before set_input_data_dict)
|
314
|
-
if action_name ==
|
377
|
+
if action_name == "weather-forecast-cache":
|
315
378
|
ActionStr = " >> Performing weather forecast, try to caching result"
|
316
379
|
app.logger.info(ActionStr)
|
317
380
|
weather_forecast_cache(emhass_conf, params, runtimeparams, app.logger)
|
318
|
-
msg =
|
381
|
+
msg = "EMHASS >> Weather Forecast has run and results possibly cached... \n"
|
319
382
|
if not checkFileLog(ActionStr):
|
320
383
|
return make_response(msg, 201)
|
321
384
|
return make_response(grabLog(ActionStr), 400)
|
322
385
|
|
323
386
|
ActionStr = " >> Setting input data dict"
|
324
387
|
app.logger.info(ActionStr)
|
325
|
-
input_data_dict = set_input_data_dict(
|
326
|
-
params, runtimeparams, action_name, app.logger
|
388
|
+
input_data_dict = set_input_data_dict(
|
389
|
+
emhass_conf, costfun, params, runtimeparams, action_name, app.logger
|
390
|
+
)
|
327
391
|
if not input_data_dict:
|
328
392
|
return make_response(grabLog(ActionStr), 400)
|
329
|
-
|
393
|
+
|
330
394
|
# If continual_publish is True, start thread with loop function
|
331
|
-
if len(continual_publish_thread) == 0 and input_data_dict[
|
395
|
+
if len(continual_publish_thread) == 0 and input_data_dict["retrieve_hass_conf"].get(
|
396
|
+
"continual_publish", False
|
397
|
+
):
|
332
398
|
# Start Thread
|
333
|
-
continualLoop = threading.Thread(
|
399
|
+
continualLoop = threading.Thread(
|
400
|
+
name="continual_publish",
|
401
|
+
target=continual_publish,
|
402
|
+
args=[input_data_dict, entity_path, app.logger],
|
403
|
+
)
|
334
404
|
continualLoop.start()
|
335
|
-
continual_publish_thread.append(continualLoop)
|
405
|
+
continual_publish_thread.append(continualLoop)
|
336
406
|
|
337
407
|
# Run action based on POST request
|
338
408
|
# If error in log when running action, return actions log (list) as response. (Using ActionStr as a reference of the action start in the log)
|
339
409
|
# publish-data
|
340
|
-
if action_name ==
|
410
|
+
if action_name == "publish-data":
|
341
411
|
ActionStr = " >> Publishing data..."
|
342
412
|
app.logger.info(ActionStr)
|
343
413
|
_ = publish_data(input_data_dict, app.logger)
|
344
|
-
msg =
|
414
|
+
msg = "EMHASS >> Action publish-data executed... \n"
|
345
415
|
if not checkFileLog(ActionStr):
|
346
416
|
return make_response(msg, 201)
|
347
417
|
return make_response(grabLog(ActionStr), 400)
|
348
418
|
# perfect-optim
|
349
|
-
elif action_name ==
|
419
|
+
elif action_name == "perfect-optim":
|
350
420
|
ActionStr = " >> Performing perfect optimization..."
|
351
421
|
app.logger.info(ActionStr)
|
352
422
|
opt_res = perfect_forecast_optim(input_data_dict, app.logger)
|
353
423
|
injection_dict = get_injection_dict(opt_res)
|
354
|
-
with open(str(emhass_conf[
|
424
|
+
with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "wb") as fid:
|
355
425
|
pickle.dump(injection_dict, fid)
|
356
|
-
msg =
|
426
|
+
msg = "EMHASS >> Action perfect-optim executed... \n"
|
357
427
|
if not checkFileLog(ActionStr):
|
358
428
|
return make_response(msg, 201)
|
359
429
|
return make_response(grabLog(ActionStr), 400)
|
360
430
|
# dayahead-optim
|
361
|
-
elif action_name ==
|
431
|
+
elif action_name == "dayahead-optim":
|
362
432
|
ActionStr = " >> Performing dayahead optimization..."
|
363
433
|
app.logger.info(ActionStr)
|
364
434
|
opt_res = dayahead_forecast_optim(input_data_dict, app.logger)
|
365
435
|
injection_dict = get_injection_dict(opt_res)
|
366
|
-
with open(str(emhass_conf[
|
436
|
+
with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "wb") as fid:
|
367
437
|
pickle.dump(injection_dict, fid)
|
368
|
-
msg =
|
438
|
+
msg = "EMHASS >> Action dayahead-optim executed... \n"
|
369
439
|
if not checkFileLog(ActionStr):
|
370
440
|
return make_response(msg, 201)
|
371
441
|
return make_response(grabLog(ActionStr), 400)
|
372
442
|
# naive-mpc-optim
|
373
|
-
elif action_name ==
|
443
|
+
elif action_name == "naive-mpc-optim":
|
374
444
|
ActionStr = " >> Performing naive MPC optimization..."
|
375
445
|
app.logger.info(ActionStr)
|
376
446
|
opt_res = naive_mpc_optim(input_data_dict, app.logger)
|
377
447
|
injection_dict = get_injection_dict(opt_res)
|
378
|
-
with open(str(emhass_conf[
|
448
|
+
with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "wb") as fid:
|
379
449
|
pickle.dump(injection_dict, fid)
|
380
|
-
msg =
|
450
|
+
msg = "EMHASS >> Action naive-mpc-optim executed... \n"
|
381
451
|
if not checkFileLog(ActionStr):
|
382
452
|
return make_response(msg, 201)
|
383
453
|
return make_response(grabLog(ActionStr), 400)
|
384
454
|
# forecast-model-fit
|
385
|
-
elif action_name ==
|
455
|
+
elif action_name == "forecast-model-fit":
|
386
456
|
ActionStr = " >> Performing a machine learning forecast model fit..."
|
387
457
|
app.logger.info(ActionStr)
|
388
458
|
df_fit_pred, _, mlf = forecast_model_fit(input_data_dict, app.logger)
|
389
|
-
injection_dict = get_injection_dict_forecast_model_fit(
|
390
|
-
|
391
|
-
with open(str(emhass_conf['data_path'] / 'injection_dict.pkl'), "wb") as fid:
|
459
|
+
injection_dict = get_injection_dict_forecast_model_fit(df_fit_pred, mlf)
|
460
|
+
with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "wb") as fid:
|
392
461
|
pickle.dump(injection_dict, fid)
|
393
|
-
msg =
|
462
|
+
msg = "EMHASS >> Action forecast-model-fit executed... \n"
|
394
463
|
if not checkFileLog(ActionStr):
|
395
464
|
return make_response(msg, 201)
|
396
465
|
return make_response(grabLog(ActionStr), 400)
|
397
466
|
# forecast-model-predict
|
398
|
-
elif action_name ==
|
467
|
+
elif action_name == "forecast-model-predict":
|
399
468
|
ActionStr = " >> Performing a machine learning forecast model predict..."
|
400
469
|
app.logger.info(ActionStr)
|
401
470
|
df_pred = forecast_model_predict(input_data_dict, app.logger)
|
402
471
|
if df_pred is None:
|
403
472
|
return make_response(grabLog(ActionStr), 400)
|
404
|
-
table1 = df_pred.reset_index().to_html(classes=
|
473
|
+
table1 = df_pred.reset_index().to_html(classes="mystyle", index=False)
|
405
474
|
injection_dict = {}
|
406
|
-
injection_dict[
|
407
|
-
|
408
|
-
|
409
|
-
|
475
|
+
injection_dict["title"] = (
|
476
|
+
"<h2>Custom machine learning forecast model predict</h2>"
|
477
|
+
)
|
478
|
+
injection_dict["subsubtitle0"] = (
|
479
|
+
"<h4>Performed a prediction using a pre-trained model</h4>"
|
480
|
+
)
|
481
|
+
injection_dict["table1"] = table1
|
482
|
+
with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "wb") as fid:
|
410
483
|
pickle.dump(injection_dict, fid)
|
411
|
-
msg =
|
484
|
+
msg = "EMHASS >> Action forecast-model-predict executed... \n"
|
412
485
|
if not checkFileLog(ActionStr):
|
413
486
|
return make_response(msg, 201)
|
414
487
|
return make_response(grabLog(ActionStr), 400)
|
415
488
|
# forecast-model-tune
|
416
|
-
elif action_name ==
|
489
|
+
elif action_name == "forecast-model-tune":
|
417
490
|
ActionStr = " >> Performing a machine learning forecast model tune..."
|
418
491
|
app.logger.info(ActionStr)
|
419
|
-
df_pred_optim, mlf = forecast_model_tune(input_data_dict, app.logger)
|
420
|
-
if df_pred_optim is None or
|
492
|
+
df_pred_optim, mlf = forecast_model_tune(input_data_dict, app.logger)
|
493
|
+
if df_pred_optim is None or mlf is None:
|
421
494
|
return make_response(grabLog(ActionStr), 400)
|
422
|
-
injection_dict = get_injection_dict_forecast_model_tune(
|
423
|
-
|
424
|
-
with open(str(emhass_conf['data_path'] / 'injection_dict.pkl'), "wb") as fid:
|
495
|
+
injection_dict = get_injection_dict_forecast_model_tune(df_pred_optim, mlf)
|
496
|
+
with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "wb") as fid:
|
425
497
|
pickle.dump(injection_dict, fid)
|
426
|
-
msg =
|
498
|
+
msg = "EMHASS >> Action forecast-model-tune executed... \n"
|
427
499
|
if not checkFileLog(ActionStr):
|
428
500
|
return make_response(msg, 201)
|
429
501
|
return make_response(grabLog(ActionStr), 400)
|
430
502
|
# regressor-model-fit
|
431
|
-
elif action_name ==
|
503
|
+
elif action_name == "regressor-model-fit":
|
432
504
|
ActionStr = " >> Performing a machine learning regressor fit..."
|
433
505
|
app.logger.info(ActionStr)
|
434
506
|
regressor_model_fit(input_data_dict, app.logger)
|
435
|
-
msg =
|
507
|
+
msg = "EMHASS >> Action regressor-model-fit executed... \n"
|
436
508
|
if not checkFileLog(ActionStr):
|
437
509
|
return make_response(msg, 201)
|
438
510
|
return make_response(grabLog(ActionStr), 400)
|
439
|
-
|
440
|
-
elif action_name ==
|
511
|
+
# regressor-model-predict
|
512
|
+
elif action_name == "regressor-model-predict":
|
441
513
|
ActionStr = " >> Performing a machine learning regressor predict..."
|
442
514
|
app.logger.info(ActionStr)
|
443
515
|
regressor_model_predict(input_data_dict, app.logger)
|
444
|
-
msg =
|
516
|
+
msg = "EMHASS >> Action regressor-model-predict executed... \n"
|
445
517
|
if not checkFileLog(ActionStr):
|
446
518
|
return make_response(msg, 201)
|
447
519
|
return make_response(grabLog(ActionStr), 400)
|
448
520
|
# Else return error
|
449
521
|
else:
|
450
522
|
app.logger.error("ERROR: passed action is not valid")
|
451
|
-
msg =
|
523
|
+
msg = "EMHASS >> ERROR: Passed action is not valid... \n"
|
452
524
|
return make_response(msg, 400)
|
453
525
|
|
526
|
+
|
454
527
|
if __name__ == "__main__":
|
455
528
|
# Parsing arguments
|
456
529
|
parser = argparse.ArgumentParser()
|
457
|
-
parser.add_argument(
|
458
|
-
|
459
|
-
|
530
|
+
parser.add_argument(
|
531
|
+
"--url",
|
532
|
+
type=str,
|
533
|
+
help="The URL to your Home Assistant instance, ex the external_url in your hass configuration",
|
534
|
+
)
|
535
|
+
parser.add_argument(
|
536
|
+
"--key",
|
537
|
+
type=str,
|
538
|
+
help="Your access key. If using EMHASS in standalone this should be a Long-Lived Access Token",
|
539
|
+
)
|
540
|
+
parser.add_argument(
|
541
|
+
"--no_response",
|
542
|
+
type=strtobool,
|
543
|
+
default="False",
|
544
|
+
help="This is set if json response errors occur",
|
545
|
+
)
|
460
546
|
args = parser.parse_args()
|
461
547
|
|
462
548
|
# Pre formatted config parameters
|
463
|
-
config = {}
|
549
|
+
config = {}
|
464
550
|
# Secrets
|
465
551
|
params_secrets = {}
|
466
552
|
# Built parameters (formatted config + secrets)
|
467
|
-
params = None
|
468
|
-
|
469
|
-
# Find env's, not not set defaults
|
553
|
+
params = None
|
554
|
+
|
555
|
+
# Find env's, not not set defaults
|
470
556
|
DATA_PATH = os.getenv("DATA_PATH", default="/app/data/")
|
471
557
|
ROOT_PATH = os.getenv("ROOT_PATH", default=str(Path(__file__).parent))
|
472
|
-
CONFIG_PATH = os.getenv(
|
473
|
-
OPTIONS_PATH = os.getenv(
|
474
|
-
DEFAULTS_PATH = os.getenv(
|
475
|
-
|
476
|
-
|
558
|
+
CONFIG_PATH = os.getenv("CONFIG_PATH", default="/share/config.json")
|
559
|
+
OPTIONS_PATH = os.getenv("OPTIONS_PATH", default="/data/options.json")
|
560
|
+
DEFAULTS_PATH = os.getenv(
|
561
|
+
"DEFAULTS_PATH", default=ROOT_PATH + "/data/config_defaults.json"
|
562
|
+
)
|
563
|
+
ASSOCIATIONS_PATH = os.getenv(
|
564
|
+
"ASSOCIATIONS_PATH", default=ROOT_PATH + "/data/associations.csv"
|
565
|
+
)
|
566
|
+
LEGACY_CONFIG_PATH = os.getenv(
|
567
|
+
"LEGACY_CONFIG_PATH", default="/app/config_emhass.yaml"
|
568
|
+
)
|
477
569
|
|
478
570
|
# Define the paths
|
479
571
|
config_path = Path(CONFIG_PATH)
|
@@ -484,50 +576,62 @@ if __name__ == "__main__":
|
|
484
576
|
data_path = Path(DATA_PATH)
|
485
577
|
root_path = Path(ROOT_PATH)
|
486
578
|
# Add paths to emhass_conf
|
487
|
-
emhass_conf[
|
488
|
-
emhass_conf[
|
489
|
-
emhass_conf[
|
490
|
-
emhass_conf[
|
491
|
-
emhass_conf[
|
492
|
-
emhass_conf[
|
493
|
-
emhass_conf[
|
579
|
+
emhass_conf["config_path"] = config_path
|
580
|
+
emhass_conf["options_path"] = options_path
|
581
|
+
emhass_conf["defaults_path"] = defaults_path
|
582
|
+
emhass_conf["associations_path"] = associations_path
|
583
|
+
emhass_conf["legacy_config_path"] = legacy_config_path
|
584
|
+
emhass_conf["data_path"] = data_path
|
585
|
+
emhass_conf["root_path"] = root_path
|
494
586
|
|
495
587
|
# Combine parameters from configuration sources (if exists)
|
496
|
-
config.update(
|
588
|
+
config.update(
|
589
|
+
build_config(
|
590
|
+
emhass_conf, app.logger, defaults_path, config_path, legacy_config_path
|
591
|
+
)
|
592
|
+
)
|
497
593
|
if type(config) is bool and not config:
|
498
594
|
raise Exception("Failed to find default config")
|
499
595
|
|
500
|
-
|
501
|
-
|
596
|
+
# Set local variables
|
597
|
+
costfun = os.getenv("LOCAL_COSTFUN", config.get("costfun", "profit"))
|
598
|
+
logging_level = os.getenv("LOGGING_LEVEL", config.get("logging_level", "INFO"))
|
502
599
|
# Temporary set logging level if debug
|
503
600
|
if logging_level == "DEBUG":
|
504
601
|
app.logger.setLevel(logging.DEBUG)
|
505
|
-
|
602
|
+
|
506
603
|
## Secrets
|
507
604
|
argument = {}
|
508
605
|
if args.url:
|
509
|
-
argument[
|
606
|
+
argument["url"] = args.url
|
510
607
|
if args.key:
|
511
|
-
argument[
|
608
|
+
argument["key"] = args.key
|
512
609
|
# Combine secrets from ENV, Arguments/ARG, Secrets file (secrets_emhass.yaml), options (options.json from addon configuration file) and/or Home Assistant Standalone API (if exist)
|
513
|
-
emhass_conf, secrets = build_secrets(
|
610
|
+
emhass_conf, secrets = build_secrets(
|
611
|
+
emhass_conf,
|
612
|
+
app.logger,
|
613
|
+
argument,
|
614
|
+
options_path,
|
615
|
+
os.getenv("SECRETS_PATH", default="/app/secrets_emhass.yaml"),
|
616
|
+
bool(args.no_response),
|
617
|
+
)
|
514
618
|
params_secrets.update(secrets)
|
515
619
|
|
516
|
-
server_ip = params_secrets.get("server_ip","0.0.0.0")
|
620
|
+
server_ip = params_secrets.get("server_ip", "0.0.0.0")
|
517
621
|
|
518
|
-
# Check if data path exists
|
519
|
-
if not os.path.isdir(emhass_conf[
|
520
|
-
app.logger.warning("Unable to find data_path: " + str(emhass_conf[
|
622
|
+
# Check if data path exists
|
623
|
+
if not os.path.isdir(emhass_conf["data_path"]):
|
624
|
+
app.logger.warning("Unable to find data_path: " + str(emhass_conf["data_path"]))
|
521
625
|
if os.path.isdir(Path("/app/data/")):
|
522
|
-
emhass_conf[
|
626
|
+
emhass_conf["data_path"] = Path("/app/data/")
|
523
627
|
else:
|
524
|
-
Path(root_path / "data/").mkdir(parents=True, exist_ok=True)
|
525
|
-
emhass_conf[
|
526
|
-
app.logger.info("data_path has been set to " + str(emhass_conf[
|
628
|
+
Path(root_path / "data/").mkdir(parents=True, exist_ok=True)
|
629
|
+
emhass_conf["data_path"] = root_path / "data/"
|
630
|
+
app.logger.info("data_path has been set to " + str(emhass_conf["data_path"]))
|
527
631
|
|
528
632
|
# Initialize this global dict
|
529
|
-
if (emhass_conf[
|
530
|
-
with open(str(emhass_conf[
|
633
|
+
if (emhass_conf["data_path"] / "injection_dict.pkl").exists():
|
634
|
+
with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "rb") as fid:
|
531
635
|
injection_dict = pickle.load(fid)
|
532
636
|
else:
|
533
637
|
injection_dict = None
|
@@ -535,68 +639,71 @@ if __name__ == "__main__":
|
|
535
639
|
# Build params from config and param_secrets (migrate params to correct config catagories), save result to params.pkl
|
536
640
|
params = build_params(emhass_conf, params_secrets, config, app.logger)
|
537
641
|
if type(params) is bool:
|
538
|
-
raise Exception("A error has occurred while building params")
|
539
|
-
|
540
|
-
|
541
|
-
|
642
|
+
raise Exception("A error has occurred while building params")
|
643
|
+
# Update params with local variables
|
644
|
+
params["optim_conf"]["costfun"] = costfun
|
645
|
+
params["optim_conf"]["logging_level"] = logging_level
|
646
|
+
|
647
|
+
# Save params to file for later reference
|
648
|
+
if os.path.exists(str(emhass_conf["data_path"])):
|
649
|
+
with open(str(emhass_conf["data_path"] / "params.pkl"), "wb") as fid:
|
542
650
|
pickle.dump((config_path, params), fid)
|
543
|
-
else:
|
544
|
-
raise Exception("missing: " + str(emhass_conf[
|
651
|
+
else:
|
652
|
+
raise Exception("missing: " + str(emhass_conf["data_path"]))
|
545
653
|
|
546
654
|
# Define loggers
|
547
|
-
|
548
|
-
|
549
|
-
|
655
|
+
formatter = logging.Formatter(
|
656
|
+
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
657
|
+
)
|
658
|
+
log.default_handler.setFormatter(formatter)
|
550
659
|
# Action file logger
|
551
|
-
fileLogger = logging.FileHandler(str(emhass_conf[
|
552
|
-
formatter = logging.Formatter(
|
553
|
-
fileLogger.setFormatter(formatter)
|
660
|
+
fileLogger = logging.FileHandler(str(emhass_conf["data_path"] / "actionLogs.txt"))
|
661
|
+
formatter = logging.Formatter("%(levelname)s - %(name)s - %(message)s")
|
662
|
+
fileLogger.setFormatter(formatter) # add format to Handler
|
554
663
|
if logging_level == "DEBUG":
|
555
664
|
app.logger.setLevel(logging.DEBUG)
|
556
|
-
ch.setLevel(logging.DEBUG)
|
557
665
|
fileLogger.setLevel(logging.DEBUG)
|
558
666
|
elif logging_level == "INFO":
|
559
667
|
app.logger.setLevel(logging.INFO)
|
560
|
-
ch.setLevel(logging.INFO)
|
561
668
|
fileLogger.setLevel(logging.INFO)
|
562
669
|
elif logging_level == "WARNING":
|
563
670
|
app.logger.setLevel(logging.WARNING)
|
564
|
-
ch.setLevel(logging.WARNING)
|
565
671
|
fileLogger.setLevel(logging.WARNING)
|
566
672
|
elif logging_level == "ERROR":
|
567
673
|
app.logger.setLevel(logging.ERROR)
|
568
|
-
ch.setLevel(logging.ERROR)
|
569
674
|
fileLogger.setLevel(logging.ERROR)
|
570
675
|
else:
|
571
676
|
app.logger.setLevel(logging.DEBUG)
|
572
|
-
ch.setLevel(logging.DEBUG)
|
573
677
|
fileLogger.setLevel(logging.DEBUG)
|
574
678
|
app.logger.propagate = False
|
575
|
-
app.logger.addHandler(
|
576
|
-
app.logger.addHandler(fileLogger)
|
679
|
+
app.logger.addHandler(fileLogger)
|
577
680
|
# Clear Action File logger file, ready for new instance
|
578
681
|
clearFileLog()
|
579
682
|
|
580
|
-
# If entity_path exists, remove any entity/metadata files
|
581
|
-
entity_path = emhass_conf[
|
582
|
-
if os.path.exists(entity_path):
|
683
|
+
# If entity_path exists, remove any entity/metadata files
|
684
|
+
entity_path = emhass_conf["data_path"] / "entities"
|
685
|
+
if os.path.exists(entity_path):
|
583
686
|
entity_pathContents = os.listdir(entity_path)
|
584
687
|
if len(entity_pathContents) > 0:
|
585
688
|
for entity in entity_pathContents:
|
586
689
|
os.remove(entity_path / entity)
|
587
|
-
|
690
|
+
|
588
691
|
# Initialise continual publish thread list
|
589
692
|
continual_publish_thread = []
|
590
|
-
|
693
|
+
|
591
694
|
# Launch server
|
592
|
-
port = int(os.environ.get(
|
593
|
-
app.logger.info(
|
594
|
-
|
595
|
-
|
596
|
-
app.logger.info(
|
695
|
+
port = int(os.environ.get("PORT", 5000))
|
696
|
+
app.logger.info(
|
697
|
+
"Launching the emhass webserver at: http://" + server_ip + ":" + str(port)
|
698
|
+
)
|
699
|
+
app.logger.info(
|
700
|
+
"Home Assistant data fetch will be performed using url: "
|
701
|
+
+ params_secrets["hass_url"]
|
702
|
+
)
|
703
|
+
app.logger.info("The data path is: " + str(emhass_conf["data_path"]))
|
704
|
+
app.logger.info("The logging is: " + str(logging_level))
|
597
705
|
try:
|
598
|
-
app.logger.info("Using core emhass version: "+version(
|
706
|
+
app.logger.info("Using core emhass version: " + version("emhass"))
|
599
707
|
except PackageNotFoundError:
|
600
708
|
app.logger.info("Using development emhass version")
|
601
709
|
serve(app, host=server_ip, port=port, threads=8)
|
602
|
-
|