emhass 0.11.2__py3-none-any.whl → 0.11.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
emhass/web_server.py CHANGED
@@ -1,51 +1,79 @@
1
1
  #!/usr/bin/env python3
2
2
  # -*- coding: utf-8 -*-
3
3
 
4
- from flask import Flask, request, make_response, render_template
4
+ import argparse
5
+ import json
6
+ import logging
7
+ import os
8
+ import pickle
9
+ import re
10
+ import threading
11
+ from distutils.util import strtobool
12
+ from importlib.metadata import PackageNotFoundError, version
13
+ from pathlib import Path
14
+
15
+ import yaml
16
+ from flask import Flask, make_response, request
17
+ from flask import logging as log
5
18
  from jinja2 import Environment, PackageLoader
6
- from requests import get
7
19
  from waitress import serve
8
- from importlib.metadata import version, PackageNotFoundError
9
- from pathlib import Path
10
- import os, json, argparse, pickle, yaml, logging, re, threading
11
- from distutils.util import strtobool
12
20
 
13
- from emhass.command_line import set_input_data_dict
14
- from emhass.command_line import perfect_forecast_optim, dayahead_forecast_optim, naive_mpc_optim
15
- from emhass.command_line import forecast_model_fit, forecast_model_predict, forecast_model_tune, weather_forecast_cache
16
- from emhass.command_line import regressor_model_fit, regressor_model_predict
17
- from emhass.command_line import publish_data, continual_publish
18
- from emhass.utils import get_injection_dict, get_injection_dict_forecast_model_fit, \
19
- get_injection_dict_forecast_model_tune, build_config, build_secrets, build_params, \
20
- param_to_config, build_legacy_config_params
21
+ from emhass.command_line import (
22
+ continual_publish,
23
+ dayahead_forecast_optim,
24
+ forecast_model_fit,
25
+ forecast_model_predict,
26
+ forecast_model_tune,
27
+ naive_mpc_optim,
28
+ perfect_forecast_optim,
29
+ publish_data,
30
+ regressor_model_fit,
31
+ regressor_model_predict,
32
+ set_input_data_dict,
33
+ weather_forecast_cache,
34
+ )
35
+ from emhass.utils import (
36
+ build_config,
37
+ build_legacy_config_params,
38
+ build_params,
39
+ build_secrets,
40
+ get_injection_dict,
41
+ get_injection_dict_forecast_model_fit,
42
+ get_injection_dict_forecast_model_tune,
43
+ param_to_config,
44
+ )
21
45
 
22
46
  # Define the Flask instance
23
47
  app = Flask(__name__)
24
48
  emhass_conf = {}
25
49
 
50
+
26
51
  def checkFileLog(refString=None) -> bool:
27
52
  """
28
53
  Check logfile for error, anything after string match if provided.
29
54
 
30
- :param refString: String to reduce log area to check for errors. Use to reduce log to check anything after string match (ie. an action).
55
+ :param refString: String to reduce log area to check for errors. Use to reduce log to check anything after string match (ie. an action).
31
56
  :type refString: str
32
57
  :return: Boolean return if error was found in logs
33
58
  :rtype: bool
34
59
 
35
60
  """
36
- if (refString is not None):
37
- logArray = grabLog(refString) #grab reduced log array (everything after string match)
38
- else:
39
- if ((emhass_conf['data_path'] / 'actionLogs.txt')).exists():
40
- with open(str(emhass_conf['data_path'] / 'actionLogs.txt'), "r") as fp:
41
- logArray = fp.readlines()
61
+ if refString is not None:
62
+ logArray = grabLog(
63
+ refString
64
+ ) # grab reduced log array (everything after string match)
65
+ else:
66
+ if (emhass_conf["data_path"] / "actionLogs.txt").exists():
67
+ with open(str(emhass_conf["data_path"] / "actionLogs.txt"), "r") as fp:
68
+ logArray = fp.readlines()
42
69
  else:
43
70
  app.logger.debug("Unable to obtain actionLogs.txt")
44
71
  for logString in logArray:
45
- if (logString.split(' ', 1)[0] == "ERROR"):
46
- return True
72
+ if logString.split(" ", 1)[0] == "ERROR":
73
+ return True
47
74
  return False
48
75
 
76
+
49
77
  def grabLog(refString) -> list:
50
78
  """
51
79
  Find string in logs, append all lines after into list to return.
@@ -58,31 +86,33 @@ def grabLog(refString) -> list:
58
86
  """
59
87
  isFound = []
60
88
  output = []
61
- if ((emhass_conf['data_path'] / 'actionLogs.txt')).exists():
62
- with open(str(emhass_conf['data_path'] / 'actionLogs.txt'), "r") as fp:
63
- logArray = fp.readlines()
64
- # Find all string matches, log key (line Number) in isFound
65
- for x in range(len(logArray)-1):
66
- if (re.search(refString,logArray[x])):
67
- isFound.append(x)
68
- if len(isFound) != 0:
69
- # Use last item in isFound to extract action logs
70
- for x in range(isFound[-1],len(logArray)):
71
- output.append(logArray[x])
89
+ if (emhass_conf["data_path"] / "actionLogs.txt").exists():
90
+ with open(str(emhass_conf["data_path"] / "actionLogs.txt"), "r") as fp:
91
+ logArray = fp.readlines()
92
+ # Find all string matches, log key (line Number) in isFound
93
+ for x in range(len(logArray) - 1):
94
+ if re.search(refString, logArray[x]):
95
+ isFound.append(x)
96
+ if len(isFound) != 0:
97
+ # Use last item in isFound to extract action logs
98
+ for x in range(isFound[-1], len(logArray)):
99
+ output.append(logArray[x])
72
100
  return output
73
101
 
102
+
74
103
  # Clear the log file
75
104
  def clearFileLog():
76
105
  """
77
106
  Clear the contents of the log file (actionLogs.txt)
78
107
 
79
108
  """
80
- if ((emhass_conf['data_path'] / 'actionLogs.txt')).exists():
81
- with open(str(emhass_conf['data_path'] / 'actionLogs.txt'), "w") as fp:
82
- fp.truncate()
109
+ if (emhass_conf["data_path"] / "actionLogs.txt").exists():
110
+ with open(str(emhass_conf["data_path"] / "actionLogs.txt"), "w") as fp:
111
+ fp.truncate()
83
112
 
84
- @app.route('/')
85
- @app.route('/index')
113
+
114
+ @app.route("/")
115
+ @app.route("/index")
86
116
  def index():
87
117
  """
88
118
  Render initial index page and serve to web server.
@@ -91,29 +121,31 @@ def index():
91
121
  """
92
122
  app.logger.info("EMHASS server online, serving index.html...")
93
123
  # Load HTML template
94
- file_loader = PackageLoader('emhass', 'templates')
124
+ file_loader = PackageLoader("emhass", "templates")
95
125
  env = Environment(loader=file_loader)
96
- #check if index.html exists
97
- if 'index.html' not in env.list_templates():
126
+ # check if index.html exists
127
+ if "index.html" not in env.list_templates():
98
128
  app.logger.error("Unable to find index.html in emhass module")
99
- return make_response(["ERROR: unable to find index.html in emhass module"],404)
100
- template = env.get_template('index.html')
129
+ return make_response(["ERROR: unable to find index.html in emhass module"], 404)
130
+ template = env.get_template("index.html")
101
131
  # Load cached dict (if exists), to present generated plot tables
102
- if (emhass_conf['data_path'] / 'injection_dict.pkl').exists():
103
- with open(str(emhass_conf['data_path'] / 'injection_dict.pkl'), "rb") as fid:
132
+ if (emhass_conf["data_path"] / "injection_dict.pkl").exists():
133
+ with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "rb") as fid:
104
134
  injection_dict = pickle.load(fid)
105
135
  else:
106
- app.logger.info("The data container dictionary is empty... Please launch an optimization task")
107
- injection_dict={}
136
+ app.logger.info(
137
+ "The data container dictionary is empty... Please launch an optimization task"
138
+ )
139
+ injection_dict = {}
108
140
 
109
- # replace {{basename}} in html template html with path root
141
+ # replace {{basename}} in html template html with path root
110
142
  # basename = request.headers.get("X-Ingress-Path", "")
111
143
  # return make_response(template.render(injection_dict=injection_dict, basename=basename))
112
-
144
+
113
145
  return make_response(template.render(injection_dict=injection_dict))
114
146
 
115
147
 
116
- @app.route('/configuration')
148
+ @app.route("/configuration")
117
149
  def configuration():
118
150
  """
119
151
  Configuration page actions:
@@ -122,41 +154,46 @@ def configuration():
122
154
  """
123
155
  app.logger.info("serving configuration.html...")
124
156
  # Load HTML template
125
- file_loader = PackageLoader('emhass', 'templates')
157
+ file_loader = PackageLoader("emhass", "templates")
126
158
  env = Environment(loader=file_loader)
127
- #check if configuration.html exists
128
- if 'configuration.html' not in env.list_templates():
159
+ # check if configuration.html exists
160
+ if "configuration.html" not in env.list_templates():
129
161
  app.logger.error("Unable to find configuration.html in emhass module")
130
- return make_response(["ERROR: unable to find configuration.html in emhass module"],404)
131
- template = env.get_template('configuration.html')
162
+ return make_response(
163
+ ["ERROR: unable to find configuration.html in emhass module"], 404
164
+ )
165
+ template = env.get_template("configuration.html")
132
166
  return make_response(template.render(config=params))
133
167
 
134
168
 
135
- @app.route('/template', methods=['GET'])
169
+ @app.route("/template", methods=["GET"])
136
170
  def template_action():
137
171
  """
138
- template page actions:
172
+ template page actions:
139
173
  Render and serve template html
140
174
 
141
175
  """
142
176
  app.logger.info(" >> Sending rendered template table data")
143
- file_loader = PackageLoader('emhass', 'templates')
177
+ file_loader = PackageLoader("emhass", "templates")
144
178
  env = Environment(loader=file_loader)
145
179
  # Check if template.html exists
146
- if 'template.html' not in env.list_templates():
180
+ if "template.html" not in env.list_templates():
147
181
  app.logger.error("Unable to find template.html in emhass module")
148
- return make_response(["WARNING: unable to find template.html in emhass module"],404)
149
- template = env.get_template('template.html')
150
- if (emhass_conf['data_path'] / 'injection_dict.pkl').exists():
151
- with open(str(emhass_conf['data_path'] / 'injection_dict.pkl'), "rb") as fid:
182
+ return make_response(
183
+ ["WARNING: unable to find template.html in emhass module"], 404
184
+ )
185
+ template = env.get_template("template.html")
186
+ if (emhass_conf["data_path"] / "injection_dict.pkl").exists():
187
+ with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "rb") as fid:
152
188
  injection_dict = pickle.load(fid)
153
189
  else:
154
190
  app.logger.warning("Unable to obtain plot data from injection_dict.pkl")
155
191
  app.logger.warning("Try running an launch an optimization task")
156
- injection_dict={}
192
+ injection_dict = {}
157
193
  return make_response(template.render(injection_dict=injection_dict))
158
194
 
159
- @app.route('/get-config', methods=['GET'])
195
+
196
+ @app.route("/get-config", methods=["GET"])
160
197
  def parameter_get():
161
198
  """
162
199
  Get request action that builds, formats and sends config as json (config.json format)
@@ -164,21 +201,27 @@ def parameter_get():
164
201
  """
165
202
  app.logger.debug("Obtaining current saved parameters as config")
166
203
  # Build config from all possible sources (inc. legacy yaml config)
167
- config = build_config(emhass_conf,app.logger,emhass_conf["defaults_path"],emhass_conf["config_path"],emhass_conf["legacy_config_path"])
204
+ config = build_config(
205
+ emhass_conf,
206
+ app.logger,
207
+ emhass_conf["defaults_path"],
208
+ emhass_conf["config_path"],
209
+ emhass_conf["legacy_config_path"],
210
+ )
168
211
  if type(config) is bool and not config:
169
- return make_response(["failed to retrieve default config file"],500)
212
+ return make_response(["failed to retrieve default config file"], 500)
170
213
  # Format parameters in config with params (converting legacy json parameters from options.json if any)
171
- params = build_params(emhass_conf,{},config,app.logger)
214
+ params = build_params(emhass_conf, {}, config, app.logger)
172
215
  if type(params) is bool and not params:
173
- return make_response(["Unable to obtain associations file"],500)
216
+ return make_response(["Unable to obtain associations file"], 500)
174
217
  # Covert formatted parameters from params back into config.json format
175
- return_config = param_to_config(params,app.logger)
218
+ return_config = param_to_config(params, app.logger)
176
219
  # Send config
177
- return make_response(return_config,201)
220
+ return make_response(return_config, 201)
178
221
 
179
222
 
180
223
  # Get default Config
181
- @app.route('/get-config/defaults', methods=['GET'])
224
+ @app.route("/get-config/defaults", methods=["GET"])
182
225
  def config_get():
183
226
  """
184
227
  Get request action, retrieves and sends default configuration
@@ -186,21 +229,21 @@ def config_get():
186
229
  """
187
230
  app.logger.debug("Obtaining default parameters")
188
231
  # Build config, passing only default file
189
- config = build_config(emhass_conf,app.logger,emhass_conf["defaults_path"])
232
+ config = build_config(emhass_conf, app.logger, emhass_conf["defaults_path"])
190
233
  if type(config) is bool and not config:
191
- return make_response(["failed to retrieve default config file"],500)
234
+ return make_response(["failed to retrieve default config file"], 500)
192
235
  # Format parameters in config with params
193
- params = build_params(emhass_conf,{},config,app.logger)
236
+ params = build_params(emhass_conf, {}, config, app.logger)
194
237
  if type(params) is bool and not params:
195
- return make_response(["Unable to obtain associations file"],500)
238
+ return make_response(["Unable to obtain associations file"], 500)
196
239
  # Covert formatted parameters from params back into config.json format
197
- return_config = param_to_config(params,app.logger)
240
+ return_config = param_to_config(params, app.logger)
198
241
  # Send params
199
- return make_response(return_config,201)
242
+ return make_response(return_config, 201)
200
243
 
201
244
 
202
245
  # Get YAML-to-JSON config
203
- @app.route('/get-json', methods=['POST'])
246
+ @app.route("/get-json", methods=["POST"])
204
247
  def json_convert():
205
248
  """
206
249
  Post request action, receives yaml config (config_emhass.yaml or EMHASS-Add-on config page) and converts to config json format.
@@ -212,268 +255,317 @@ def json_convert():
212
255
 
213
256
  # If filed to Parse YAML
214
257
  if yaml_config is None:
215
- return make_response(["failed to Parse YAML from data"],400)
258
+ return make_response(["failed to Parse YAML from data"], 400)
216
259
  # Test YAML is legacy config format (from config_emhass.yaml)
217
- test_legacy_config = build_legacy_config_params(emhass_conf,yaml_config, app.logger)
260
+ test_legacy_config = build_legacy_config_params(
261
+ emhass_conf, yaml_config, app.logger
262
+ )
218
263
  if test_legacy_config:
219
264
  yaml_config = test_legacy_config
220
265
  # Format YAML to params (format params. check if params match legacy option.json format)
221
- params = build_params(emhass_conf,{},yaml_config,app.logger)
266
+ params = build_params(emhass_conf, {}, yaml_config, app.logger)
222
267
  if type(params) is bool and not params:
223
- return make_response(["Unable to obtain associations file"],500)
268
+ return make_response(["Unable to obtain associations file"], 500)
224
269
  # Covert formatted parameters from params back into config.json format
225
- config = param_to_config(params,app.logger)
270
+ config = param_to_config(params, app.logger)
226
271
  # convert json to str
227
272
  config = json.dumps(config)
228
273
 
229
274
  # Send params
230
- return make_response(config,201)
275
+ return make_response(config, 201)
276
+
231
277
 
232
- @app.route('/set-config', methods=['POST'])
278
+ @app.route("/set-config", methods=["POST"])
233
279
  def parameter_set():
234
280
  """
235
281
  Receive JSON config, and save config to file (config.json and param.pkl)
236
282
 
237
283
  """
238
284
  config = {}
239
- if not emhass_conf['defaults_path']:
240
- return make_response(["Unable to Obtain defaults_path from emhass_conf"],500)
241
- if not emhass_conf['config_path']:
242
- return make_response(["Unable to Obtain config_path from emhass_conf"],500)
243
-
285
+ if not emhass_conf["defaults_path"]:
286
+ return make_response(["Unable to Obtain defaults_path from emhass_conf"], 500)
287
+ if not emhass_conf["config_path"]:
288
+ return make_response(["Unable to Obtain config_path from emhass_conf"], 500)
289
+
244
290
  # Load defaults as a reference point (for sorting) and a base to override
245
- if os.path.exists(emhass_conf['defaults_path']) and Path(emhass_conf['defaults_path']).is_file():
246
- with emhass_conf['defaults_path'].open('r') as data:
291
+ if (
292
+ os.path.exists(emhass_conf["defaults_path"])
293
+ and Path(emhass_conf["defaults_path"]).is_file()
294
+ ):
295
+ with emhass_conf["defaults_path"].open("r") as data:
247
296
  config = json.load(data)
248
297
  else:
249
- app.logger.warning("Unable to obtain default config. only parameters passed from request will be saved to config.json")
298
+ app.logger.warning(
299
+ "Unable to obtain default config. only parameters passed from request will be saved to config.json"
300
+ )
250
301
 
251
302
  # Retrieve sent config json
252
303
  request_data = request.get_json(force=True)
253
304
 
254
305
  # check if data is empty
255
306
  if len(request_data) == 0:
256
- return make_response(["failed to retrieve config json"],400)
257
-
307
+ return make_response(["failed to retrieve config json"], 400)
308
+
258
309
  # Format config by converting to params (format params. check if params match legacy option.json format. If so format)
259
- params = build_params(emhass_conf,params_secrets,request_data,app.logger)
310
+ params = build_params(emhass_conf, params_secrets, request_data, app.logger)
260
311
  if type(params) is bool and not params:
261
- return make_response(["Unable to obtain associations file"],500)
262
-
312
+ return make_response(["Unable to obtain associations file"], 500)
313
+
263
314
  # Covert formatted parameters from params back into config.json format.
264
315
  # Overwrite existing default parameters in config
265
- config.update(param_to_config(params,app.logger))
316
+ config.update(param_to_config(params, app.logger))
266
317
 
267
318
  # Save config to config.json
268
- if os.path.exists(emhass_conf['config_path'].parent):
269
- with emhass_conf['config_path'].open('w') as f:
319
+ if os.path.exists(emhass_conf["config_path"].parent):
320
+ with emhass_conf["config_path"].open("w") as f:
270
321
  json.dump(config, f, indent=4)
271
- else:
272
- return make_response(["Unable to save config file"],500)
322
+ else:
323
+ return make_response(["Unable to save config file"], 500)
273
324
  request_data
274
325
 
275
- # Save params with updated config
276
- if os.path.exists(emhass_conf['data_path']):
277
- with open(str(emhass_conf['data_path'] / 'params.pkl'), "wb") as fid:
278
- pickle.dump((config_path, build_params(emhass_conf,params_secrets,config,app.logger)), fid)
279
- else:
280
- return make_response(["Unable to save params file, missing data_path"],500)
281
-
326
+ # Save params with updated config
327
+ if os.path.exists(emhass_conf["data_path"]):
328
+ with open(str(emhass_conf["data_path"] / "params.pkl"), "wb") as fid:
329
+ pickle.dump(
330
+ (
331
+ config_path,
332
+ build_params(emhass_conf, params_secrets, config, app.logger),
333
+ ),
334
+ fid,
335
+ )
336
+ else:
337
+ return make_response(["Unable to save params file, missing data_path"], 500)
338
+
282
339
  app.logger.info("Saved parameters from webserver")
283
- return make_response({},201)
340
+ return make_response({}, 201)
284
341
 
285
- @app.route('/action/<action_name>', methods=['POST'])
342
+
343
+ @app.route("/action/<action_name>", methods=["POST"])
286
344
  def action_call(action_name):
287
345
  """
288
346
  Receive Post action, run action according to passed slug(action_name) (e.g. /action/publish-data)
289
347
 
290
348
  :param action_name: Slug/Action string corresponding to which action to take
291
349
  :type action_name: String
292
-
350
+
293
351
  """
294
352
  # Setting up parameters
295
353
  # Params
296
- if (emhass_conf['data_path'] / 'params.pkl').exists():
297
- with open(str(emhass_conf['data_path'] / 'params.pkl'), "rb") as fid:
298
- emhass_conf['config_path'], params = pickle.load(fid)
354
+ ActionStr = " >> Obtaining params: "
355
+ app.logger.info(ActionStr)
356
+ if (emhass_conf["data_path"] / "params.pkl").exists():
357
+ with open(str(emhass_conf["data_path"] / "params.pkl"), "rb") as fid:
358
+ emhass_conf["config_path"], params = pickle.load(fid)
359
+ # Set local costfun variable
360
+ if params.get("optim_conf", None) is not None:
361
+ costfun = params["optim_conf"].get("costfun", "profit")
299
362
  params = json.dumps(params)
300
363
  else:
301
364
  app.logger.error("Unable to find params.pkl file")
302
365
  return make_response(grabLog(ActionStr), 400)
303
366
  # Runtime
304
- runtimeparams = request.get_json(force=True,silent=True)
367
+ runtimeparams = request.get_json(force=True, silent=True)
305
368
  if runtimeparams is not None:
306
- if runtimeparams != '{}':
369
+ if runtimeparams != "{}":
307
370
  app.logger.info("Passed runtime parameters: " + str(runtimeparams))
308
371
  else:
309
372
  app.logger.warning("Unable to parse runtime parameters")
310
- runtimeparams = {}
373
+ runtimeparams = {}
311
374
  runtimeparams = json.dumps(runtimeparams)
312
375
 
313
376
  # weather-forecast-cache (check before set_input_data_dict)
314
- if action_name == 'weather-forecast-cache':
377
+ if action_name == "weather-forecast-cache":
315
378
  ActionStr = " >> Performing weather forecast, try to caching result"
316
379
  app.logger.info(ActionStr)
317
380
  weather_forecast_cache(emhass_conf, params, runtimeparams, app.logger)
318
- msg = f'EMHASS >> Weather Forecast has run and results possibly cached... \n'
381
+ msg = "EMHASS >> Weather Forecast has run and results possibly cached... \n"
319
382
  if not checkFileLog(ActionStr):
320
383
  return make_response(msg, 201)
321
384
  return make_response(grabLog(ActionStr), 400)
322
385
 
323
386
  ActionStr = " >> Setting input data dict"
324
387
  app.logger.info(ActionStr)
325
- input_data_dict = set_input_data_dict(emhass_conf, costfun,
326
- params, runtimeparams, action_name, app.logger)
388
+ input_data_dict = set_input_data_dict(
389
+ emhass_conf, costfun, params, runtimeparams, action_name, app.logger
390
+ )
327
391
  if not input_data_dict:
328
392
  return make_response(grabLog(ActionStr), 400)
329
-
393
+
330
394
  # If continual_publish is True, start thread with loop function
331
- if len(continual_publish_thread) == 0 and input_data_dict['retrieve_hass_conf'].get('continual_publish',False):
395
+ if len(continual_publish_thread) == 0 and input_data_dict["retrieve_hass_conf"].get(
396
+ "continual_publish", False
397
+ ):
332
398
  # Start Thread
333
- continualLoop = threading.Thread(name='continual_publish',target=continual_publish,args=[input_data_dict,entity_path,app.logger])
399
+ continualLoop = threading.Thread(
400
+ name="continual_publish",
401
+ target=continual_publish,
402
+ args=[input_data_dict, entity_path, app.logger],
403
+ )
334
404
  continualLoop.start()
335
- continual_publish_thread.append(continualLoop)
405
+ continual_publish_thread.append(continualLoop)
336
406
 
337
407
  # Run action based on POST request
338
408
  # If error in log when running action, return actions log (list) as response. (Using ActionStr as a reference of the action start in the log)
339
409
  # publish-data
340
- if action_name == 'publish-data':
410
+ if action_name == "publish-data":
341
411
  ActionStr = " >> Publishing data..."
342
412
  app.logger.info(ActionStr)
343
413
  _ = publish_data(input_data_dict, app.logger)
344
- msg = f'EMHASS >> Action publish-data executed... \n'
414
+ msg = "EMHASS >> Action publish-data executed... \n"
345
415
  if not checkFileLog(ActionStr):
346
416
  return make_response(msg, 201)
347
417
  return make_response(grabLog(ActionStr), 400)
348
418
  # perfect-optim
349
- elif action_name == 'perfect-optim':
419
+ elif action_name == "perfect-optim":
350
420
  ActionStr = " >> Performing perfect optimization..."
351
421
  app.logger.info(ActionStr)
352
422
  opt_res = perfect_forecast_optim(input_data_dict, app.logger)
353
423
  injection_dict = get_injection_dict(opt_res)
354
- with open(str(emhass_conf['data_path'] / 'injection_dict.pkl'), "wb") as fid:
424
+ with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "wb") as fid:
355
425
  pickle.dump(injection_dict, fid)
356
- msg = f'EMHASS >> Action perfect-optim executed... \n'
426
+ msg = "EMHASS >> Action perfect-optim executed... \n"
357
427
  if not checkFileLog(ActionStr):
358
428
  return make_response(msg, 201)
359
429
  return make_response(grabLog(ActionStr), 400)
360
430
  # dayahead-optim
361
- elif action_name == 'dayahead-optim':
431
+ elif action_name == "dayahead-optim":
362
432
  ActionStr = " >> Performing dayahead optimization..."
363
433
  app.logger.info(ActionStr)
364
434
  opt_res = dayahead_forecast_optim(input_data_dict, app.logger)
365
435
  injection_dict = get_injection_dict(opt_res)
366
- with open(str(emhass_conf['data_path'] / 'injection_dict.pkl'), "wb") as fid:
436
+ with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "wb") as fid:
367
437
  pickle.dump(injection_dict, fid)
368
- msg = f'EMHASS >> Action dayahead-optim executed... \n'
438
+ msg = "EMHASS >> Action dayahead-optim executed... \n"
369
439
  if not checkFileLog(ActionStr):
370
440
  return make_response(msg, 201)
371
441
  return make_response(grabLog(ActionStr), 400)
372
442
  # naive-mpc-optim
373
- elif action_name == 'naive-mpc-optim':
443
+ elif action_name == "naive-mpc-optim":
374
444
  ActionStr = " >> Performing naive MPC optimization..."
375
445
  app.logger.info(ActionStr)
376
446
  opt_res = naive_mpc_optim(input_data_dict, app.logger)
377
447
  injection_dict = get_injection_dict(opt_res)
378
- with open(str(emhass_conf['data_path'] / 'injection_dict.pkl'), "wb") as fid:
448
+ with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "wb") as fid:
379
449
  pickle.dump(injection_dict, fid)
380
- msg = f'EMHASS >> Action naive-mpc-optim executed... \n'
450
+ msg = "EMHASS >> Action naive-mpc-optim executed... \n"
381
451
  if not checkFileLog(ActionStr):
382
452
  return make_response(msg, 201)
383
453
  return make_response(grabLog(ActionStr), 400)
384
454
  # forecast-model-fit
385
- elif action_name == 'forecast-model-fit':
455
+ elif action_name == "forecast-model-fit":
386
456
  ActionStr = " >> Performing a machine learning forecast model fit..."
387
457
  app.logger.info(ActionStr)
388
458
  df_fit_pred, _, mlf = forecast_model_fit(input_data_dict, app.logger)
389
- injection_dict = get_injection_dict_forecast_model_fit(
390
- df_fit_pred, mlf)
391
- with open(str(emhass_conf['data_path'] / 'injection_dict.pkl'), "wb") as fid:
459
+ injection_dict = get_injection_dict_forecast_model_fit(df_fit_pred, mlf)
460
+ with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "wb") as fid:
392
461
  pickle.dump(injection_dict, fid)
393
- msg = f'EMHASS >> Action forecast-model-fit executed... \n'
462
+ msg = "EMHASS >> Action forecast-model-fit executed... \n"
394
463
  if not checkFileLog(ActionStr):
395
464
  return make_response(msg, 201)
396
465
  return make_response(grabLog(ActionStr), 400)
397
466
  # forecast-model-predict
398
- elif action_name == 'forecast-model-predict':
467
+ elif action_name == "forecast-model-predict":
399
468
  ActionStr = " >> Performing a machine learning forecast model predict..."
400
469
  app.logger.info(ActionStr)
401
470
  df_pred = forecast_model_predict(input_data_dict, app.logger)
402
471
  if df_pred is None:
403
472
  return make_response(grabLog(ActionStr), 400)
404
- table1 = df_pred.reset_index().to_html(classes='mystyle', index=False)
473
+ table1 = df_pred.reset_index().to_html(classes="mystyle", index=False)
405
474
  injection_dict = {}
406
- injection_dict['title'] = '<h2>Custom machine learning forecast model predict</h2>'
407
- injection_dict['subsubtitle0'] = '<h4>Performed a prediction using a pre-trained model</h4>'
408
- injection_dict['table1'] = table1
409
- with open(str(emhass_conf['data_path'] / 'injection_dict.pkl'), "wb") as fid:
475
+ injection_dict["title"] = (
476
+ "<h2>Custom machine learning forecast model predict</h2>"
477
+ )
478
+ injection_dict["subsubtitle0"] = (
479
+ "<h4>Performed a prediction using a pre-trained model</h4>"
480
+ )
481
+ injection_dict["table1"] = table1
482
+ with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "wb") as fid:
410
483
  pickle.dump(injection_dict, fid)
411
- msg = f'EMHASS >> Action forecast-model-predict executed... \n'
484
+ msg = "EMHASS >> Action forecast-model-predict executed... \n"
412
485
  if not checkFileLog(ActionStr):
413
486
  return make_response(msg, 201)
414
487
  return make_response(grabLog(ActionStr), 400)
415
488
  # forecast-model-tune
416
- elif action_name == 'forecast-model-tune':
489
+ elif action_name == "forecast-model-tune":
417
490
  ActionStr = " >> Performing a machine learning forecast model tune..."
418
491
  app.logger.info(ActionStr)
419
- df_pred_optim, mlf = forecast_model_tune(input_data_dict, app.logger)
420
- if df_pred_optim is None or mlf is None:
492
+ df_pred_optim, mlf = forecast_model_tune(input_data_dict, app.logger)
493
+ if df_pred_optim is None or mlf is None:
421
494
  return make_response(grabLog(ActionStr), 400)
422
- injection_dict = get_injection_dict_forecast_model_tune(
423
- df_pred_optim, mlf)
424
- with open(str(emhass_conf['data_path'] / 'injection_dict.pkl'), "wb") as fid:
495
+ injection_dict = get_injection_dict_forecast_model_tune(df_pred_optim, mlf)
496
+ with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "wb") as fid:
425
497
  pickle.dump(injection_dict, fid)
426
- msg = f'EMHASS >> Action forecast-model-tune executed... \n'
498
+ msg = "EMHASS >> Action forecast-model-tune executed... \n"
427
499
  if not checkFileLog(ActionStr):
428
500
  return make_response(msg, 201)
429
501
  return make_response(grabLog(ActionStr), 400)
430
502
  # regressor-model-fit
431
- elif action_name == 'regressor-model-fit':
503
+ elif action_name == "regressor-model-fit":
432
504
  ActionStr = " >> Performing a machine learning regressor fit..."
433
505
  app.logger.info(ActionStr)
434
506
  regressor_model_fit(input_data_dict, app.logger)
435
- msg = f'EMHASS >> Action regressor-model-fit executed... \n'
507
+ msg = "EMHASS >> Action regressor-model-fit executed... \n"
436
508
  if not checkFileLog(ActionStr):
437
509
  return make_response(msg, 201)
438
510
  return make_response(grabLog(ActionStr), 400)
439
- # regressor-model-predict
440
- elif action_name == 'regressor-model-predict':
511
+ # regressor-model-predict
512
+ elif action_name == "regressor-model-predict":
441
513
  ActionStr = " >> Performing a machine learning regressor predict..."
442
514
  app.logger.info(ActionStr)
443
515
  regressor_model_predict(input_data_dict, app.logger)
444
- msg = f'EMHASS >> Action regressor-model-predict executed... \n'
516
+ msg = "EMHASS >> Action regressor-model-predict executed... \n"
445
517
  if not checkFileLog(ActionStr):
446
518
  return make_response(msg, 201)
447
519
  return make_response(grabLog(ActionStr), 400)
448
520
  # Else return error
449
521
  else:
450
522
  app.logger.error("ERROR: passed action is not valid")
451
- msg = f'EMHASS >> ERROR: Passed action is not valid... \n'
523
+ msg = "EMHASS >> ERROR: Passed action is not valid... \n"
452
524
  return make_response(msg, 400)
453
525
 
526
+
454
527
  if __name__ == "__main__":
455
528
  # Parsing arguments
456
529
  parser = argparse.ArgumentParser()
457
- parser.add_argument('--url', type=str, help='The URL to your Home Assistant instance, ex the external_url in your hass configuration')
458
- parser.add_argument('--key', type=str, help='Your access key. If using EMHASS in standalone this should be a Long-Lived Access Token')
459
- parser.add_argument('--no_response', type=strtobool, default='False', help='This is set if json response errors occur')
530
+ parser.add_argument(
531
+ "--url",
532
+ type=str,
533
+ help="The URL to your Home Assistant instance, ex the external_url in your hass configuration",
534
+ )
535
+ parser.add_argument(
536
+ "--key",
537
+ type=str,
538
+ help="Your access key. If using EMHASS in standalone this should be a Long-Lived Access Token",
539
+ )
540
+ parser.add_argument(
541
+ "--no_response",
542
+ type=strtobool,
543
+ default="False",
544
+ help="This is set if json response errors occur",
545
+ )
460
546
  args = parser.parse_args()
461
547
 
462
548
  # Pre formatted config parameters
463
- config = {}
549
+ config = {}
464
550
  # Secrets
465
551
  params_secrets = {}
466
552
  # Built parameters (formatted config + secrets)
467
- params = None
468
-
469
- # Find env's, not not set defaults
553
+ params = None
554
+
555
+ # Find env's, not not set defaults
470
556
  DATA_PATH = os.getenv("DATA_PATH", default="/app/data/")
471
557
  ROOT_PATH = os.getenv("ROOT_PATH", default=str(Path(__file__).parent))
472
- CONFIG_PATH = os.getenv('CONFIG_PATH', default="/share/config.json")
473
- OPTIONS_PATH = os.getenv('OPTIONS_PATH', default="/data/options.json")
474
- DEFAULTS_PATH = os.getenv('DEFAULTS_PATH', default=ROOT_PATH +"/data/config_defaults.json")
475
- ASSOCIATIONS_PATH = os.getenv('ASSOCIATIONS_PATH', default=ROOT_PATH + "/data/associations.csv")
476
- LEGACY_CONFIG_PATH = os.getenv("LEGACY_CONFIG_PATH", default="/app/config_emhass.yaml")
558
+ CONFIG_PATH = os.getenv("CONFIG_PATH", default="/share/config.json")
559
+ OPTIONS_PATH = os.getenv("OPTIONS_PATH", default="/data/options.json")
560
+ DEFAULTS_PATH = os.getenv(
561
+ "DEFAULTS_PATH", default=ROOT_PATH + "/data/config_defaults.json"
562
+ )
563
+ ASSOCIATIONS_PATH = os.getenv(
564
+ "ASSOCIATIONS_PATH", default=ROOT_PATH + "/data/associations.csv"
565
+ )
566
+ LEGACY_CONFIG_PATH = os.getenv(
567
+ "LEGACY_CONFIG_PATH", default="/app/config_emhass.yaml"
568
+ )
477
569
 
478
570
  # Define the paths
479
571
  config_path = Path(CONFIG_PATH)
@@ -484,50 +576,62 @@ if __name__ == "__main__":
484
576
  data_path = Path(DATA_PATH)
485
577
  root_path = Path(ROOT_PATH)
486
578
  # Add paths to emhass_conf
487
- emhass_conf['config_path'] = config_path
488
- emhass_conf['options_path'] = options_path
489
- emhass_conf['defaults_path'] = defaults_path
490
- emhass_conf['associations_path'] = associations_path
491
- emhass_conf['legacy_config_path'] = legacy_config_path
492
- emhass_conf['data_path'] = data_path
493
- emhass_conf['root_path'] = root_path
579
+ emhass_conf["config_path"] = config_path
580
+ emhass_conf["options_path"] = options_path
581
+ emhass_conf["defaults_path"] = defaults_path
582
+ emhass_conf["associations_path"] = associations_path
583
+ emhass_conf["legacy_config_path"] = legacy_config_path
584
+ emhass_conf["data_path"] = data_path
585
+ emhass_conf["root_path"] = root_path
494
586
 
495
587
  # Combine parameters from configuration sources (if exists)
496
- config.update(build_config(emhass_conf,app.logger,defaults_path,config_path,legacy_config_path))
588
+ config.update(
589
+ build_config(
590
+ emhass_conf, app.logger, defaults_path, config_path, legacy_config_path
591
+ )
592
+ )
497
593
  if type(config) is bool and not config:
498
594
  raise Exception("Failed to find default config")
499
595
 
500
- costfun = os.getenv('LOCAL_COSTFUN', config.get('costfun', 'profit'))
501
- logging_level = os.getenv('LOGGING_LEVEL', config.get('logging_level','INFO'))
596
+ # Set local variables
597
+ costfun = os.getenv("LOCAL_COSTFUN", config.get("costfun", "profit"))
598
+ logging_level = os.getenv("LOGGING_LEVEL", config.get("logging_level", "INFO"))
502
599
  # Temporary set logging level if debug
503
600
  if logging_level == "DEBUG":
504
601
  app.logger.setLevel(logging.DEBUG)
505
-
602
+
506
603
  ## Secrets
507
604
  argument = {}
508
605
  if args.url:
509
- argument['url'] = args.url
606
+ argument["url"] = args.url
510
607
  if args.key:
511
- argument['key'] = args.key
608
+ argument["key"] = args.key
512
609
  # Combine secrets from ENV, Arguments/ARG, Secrets file (secrets_emhass.yaml), options (options.json from addon configuration file) and/or Home Assistant Standalone API (if exist)
513
- emhass_conf, secrets = build_secrets(emhass_conf,app.logger,argument,options_path,os.getenv('SECRETS_PATH', default='/app/secrets_emhass.yaml'), bool(args.no_response))
610
+ emhass_conf, secrets = build_secrets(
611
+ emhass_conf,
612
+ app.logger,
613
+ argument,
614
+ options_path,
615
+ os.getenv("SECRETS_PATH", default="/app/secrets_emhass.yaml"),
616
+ bool(args.no_response),
617
+ )
514
618
  params_secrets.update(secrets)
515
619
 
516
- server_ip = params_secrets.get("server_ip","0.0.0.0")
620
+ server_ip = params_secrets.get("server_ip", "0.0.0.0")
517
621
 
518
- # Check if data path exists
519
- if not os.path.isdir(emhass_conf['data_path']):
520
- app.logger.warning("Unable to find data_path: " + str(emhass_conf['data_path']))
622
+ # Check if data path exists
623
+ if not os.path.isdir(emhass_conf["data_path"]):
624
+ app.logger.warning("Unable to find data_path: " + str(emhass_conf["data_path"]))
521
625
  if os.path.isdir(Path("/app/data/")):
522
- emhass_conf['data_path'] = Path("/app/data/")
626
+ emhass_conf["data_path"] = Path("/app/data/")
523
627
  else:
524
- Path(root_path / "data/").mkdir(parents=True, exist_ok=True)
525
- emhass_conf['data_path'] = root_path / "data/"
526
- app.logger.info("data_path has been set to " + str(emhass_conf['data_path']))
628
+ Path(root_path / "data/").mkdir(parents=True, exist_ok=True)
629
+ emhass_conf["data_path"] = root_path / "data/"
630
+ app.logger.info("data_path has been set to " + str(emhass_conf["data_path"]))
527
631
 
528
632
  # Initialize this global dict
529
- if (emhass_conf['data_path'] / 'injection_dict.pkl').exists():
530
- with open(str(emhass_conf['data_path'] / 'injection_dict.pkl'), "rb") as fid:
633
+ if (emhass_conf["data_path"] / "injection_dict.pkl").exists():
634
+ with open(str(emhass_conf["data_path"] / "injection_dict.pkl"), "rb") as fid:
531
635
  injection_dict = pickle.load(fid)
532
636
  else:
533
637
  injection_dict = None
@@ -535,68 +639,71 @@ if __name__ == "__main__":
535
639
  # Build params from config and param_secrets (migrate params to correct config catagories), save result to params.pkl
536
640
  params = build_params(emhass_conf, params_secrets, config, app.logger)
537
641
  if type(params) is bool:
538
- raise Exception("A error has occurred while building params")
539
-
540
- if os.path.exists(str(emhass_conf['data_path'])):
541
- with open(str(emhass_conf['data_path'] / 'params.pkl'), "wb") as fid:
642
+ raise Exception("A error has occurred while building params")
643
+ # Update params with local variables
644
+ params["optim_conf"]["costfun"] = costfun
645
+ params["optim_conf"]["logging_level"] = logging_level
646
+
647
+ # Save params to file for later reference
648
+ if os.path.exists(str(emhass_conf["data_path"])):
649
+ with open(str(emhass_conf["data_path"] / "params.pkl"), "wb") as fid:
542
650
  pickle.dump((config_path, params), fid)
543
- else:
544
- raise Exception("missing: " + str(emhass_conf['data_path']))
651
+ else:
652
+ raise Exception("missing: " + str(emhass_conf["data_path"]))
545
653
 
546
654
  # Define loggers
547
- ch = logging.StreamHandler()
548
- formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
549
- ch.setFormatter(formatter)
655
+ formatter = logging.Formatter(
656
+ "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
657
+ )
658
+ log.default_handler.setFormatter(formatter)
550
659
  # Action file logger
551
- fileLogger = logging.FileHandler(str(emhass_conf['data_path'] / 'actionLogs.txt'))
552
- formatter = logging.Formatter('%(levelname)s - %(name)s - %(message)s')
553
- fileLogger.setFormatter(formatter) # add format to Handler
660
+ fileLogger = logging.FileHandler(str(emhass_conf["data_path"] / "actionLogs.txt"))
661
+ formatter = logging.Formatter("%(levelname)s - %(name)s - %(message)s")
662
+ fileLogger.setFormatter(formatter) # add format to Handler
554
663
  if logging_level == "DEBUG":
555
664
  app.logger.setLevel(logging.DEBUG)
556
- ch.setLevel(logging.DEBUG)
557
665
  fileLogger.setLevel(logging.DEBUG)
558
666
  elif logging_level == "INFO":
559
667
  app.logger.setLevel(logging.INFO)
560
- ch.setLevel(logging.INFO)
561
668
  fileLogger.setLevel(logging.INFO)
562
669
  elif logging_level == "WARNING":
563
670
  app.logger.setLevel(logging.WARNING)
564
- ch.setLevel(logging.WARNING)
565
671
  fileLogger.setLevel(logging.WARNING)
566
672
  elif logging_level == "ERROR":
567
673
  app.logger.setLevel(logging.ERROR)
568
- ch.setLevel(logging.ERROR)
569
674
  fileLogger.setLevel(logging.ERROR)
570
675
  else:
571
676
  app.logger.setLevel(logging.DEBUG)
572
- ch.setLevel(logging.DEBUG)
573
677
  fileLogger.setLevel(logging.DEBUG)
574
678
  app.logger.propagate = False
575
- app.logger.addHandler(ch)
576
- app.logger.addHandler(fileLogger)
679
+ app.logger.addHandler(fileLogger)
577
680
  # Clear Action File logger file, ready for new instance
578
681
  clearFileLog()
579
682
 
580
- # If entity_path exists, remove any entity/metadata files
581
- entity_path = emhass_conf['data_path'] / "entities"
582
- if os.path.exists(entity_path):
683
+ # If entity_path exists, remove any entity/metadata files
684
+ entity_path = emhass_conf["data_path"] / "entities"
685
+ if os.path.exists(entity_path):
583
686
  entity_pathContents = os.listdir(entity_path)
584
687
  if len(entity_pathContents) > 0:
585
688
  for entity in entity_pathContents:
586
689
  os.remove(entity_path / entity)
587
-
690
+
588
691
  # Initialise continual publish thread list
589
692
  continual_publish_thread = []
590
-
693
+
591
694
  # Launch server
592
- port = int(os.environ.get('PORT', 5000))
593
- app.logger.info("Launching the emhass webserver at: http://"+server_ip+":"+str(port))
594
- app.logger.info("Home Assistant data fetch will be performed using url: "+params_secrets['hass_url'])
595
- app.logger.info("The data path is: "+str(emhass_conf['data_path']))
596
- app.logger.info("The logging is: "+str(logging_level))
695
+ port = int(os.environ.get("PORT", 5000))
696
+ app.logger.info(
697
+ "Launching the emhass webserver at: http://" + server_ip + ":" + str(port)
698
+ )
699
+ app.logger.info(
700
+ "Home Assistant data fetch will be performed using url: "
701
+ + params_secrets["hass_url"]
702
+ )
703
+ app.logger.info("The data path is: " + str(emhass_conf["data_path"]))
704
+ app.logger.info("The logging is: " + str(logging_level))
597
705
  try:
598
- app.logger.info("Using core emhass version: "+version('emhass'))
706
+ app.logger.info("Using core emhass version: " + version("emhass"))
599
707
  except PackageNotFoundError:
600
708
  app.logger.info("Using development emhass version")
601
709
  serve(app, host=server_ip, port=port, threads=8)
602
-