emhass 0.7.7__tar.gz → 0.8.0__tar.gz

{emhass-0.7.7 → emhass-0.8.0}/CHANGELOG.md

@@ -1,8 +1,22 @@
 # Changelog
 
+## [0.8.0] - 2024-02-25
+### Improvement
+- Thanks to the great work from @GeoDerp we now have a unified/centralized Dockerfile that allows for testing different installation configuration methods in one place. This greatly helps testing, notably emulating the add-on environment. This will improve overall testing for both teh core code and the add-on. Again many thanks!
+- There were also a lot of nice improveements from @GeoDerp to the webui, namely: styling, dynamic table, optimization feedback after button press, logging, a new clear button, etc.
+- From now on we will unify the semantic versioning for both the main core code and the add-on.
+
+## [0.7.8] - 2024-02-18
+### Improvement
+Added some nice logging functionalities and responsiveness on the webui.
+Thanks to @GeoDerp for this great work!
+- new actionLogs.txt is generated in datapath storing sessions app.logger info
+- on successful html button press, fetch is called to get html containing latest table data
+- on html button press, If app.logger ERROR is present, send action log back and present on page.
+
 ## [0.7.7] - 2024-02-10
 ### Improvement
-- Bumped the webui. Index/style added runtime parameter dictionary inputs. Thanks to @GeoDerp
+- Bumped the webui. Some great new features and styling. Now it is possible to pass data directly as lsit of values when using the buttons in the webui. Thanks to @GeoDerp
 - Added two additional testing environment options. Thanks to @GeoDerp
 ### Fix
 - Bump markupsafe from 2.1.4 to 2.1.5
@@ -565,6 +579,7 @@
 [0.7.5]: https://github.com/davidusb-geek/emhass/releases/tag/v0.7.5
 [0.7.6]: https://github.com/davidusb-geek/emhass/releases/tag/v0.7.6
 [0.7.7]: https://github.com/davidusb-geek/emhass/releases/tag/v0.7.7
+[0.7.8]: https://github.com/davidusb-geek/emhass/releases/tag/v0.7.8
 
 # Notes
 All notable changes to this project will be documented in this file.

{emhass-0.7.7 → emhass-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: emhass
-Version: 0.7.7
+Version: 0.8.0
 Summary: An Energy Management System for Home Assistant
 Home-page: https://github.com/davidusb-geek/emhass
 Author: David HERNANDEZ
@@ -460,7 +460,7 @@ Check the dedicated section in the documentation here: [https://emhass.readthedo
 
 ## Development
 
-Pull request are very much accepted on this project. For development you can find some instructions here [Development](./docs/develop.md)
+Pull request are very much accepted on this project. For development you can find some instructions here [Development](https://emhass.readthedocs.io/en/latest/develop.html)
 
 ## Troubleshooting
 

{emhass-0.7.7 → emhass-0.8.0}/README.md

@@ -425,7 +425,7 @@ Check the dedicated section in the documentation here: [https://emhass.readthedo
 
 ## Development
 
-Pull request are very much accepted on this project. For development you can find some instructions here [Development](./docs/develop.md)
+Pull request are very much accepted on this project. For development you can find some instructions here [Development](https://emhass.readthedocs.io/en/latest/develop.html)
 
 ## Troubleshooting
 

{emhass-0.7.7 → emhass-0.8.0}/setup.py

@@ -19,7 +19,7 @@ long_description = (here / 'README.md').read_text(encoding='utf-8')
 
 setup(
     name='emhass',  # Required
-    version='0.7.7',  # Required
+    version='0.8.0',  # Required
     description='An Energy Management System for Home Assistant',  # Optional
     long_description=long_description,  # Optional
     long_description_content_type='text/markdown',  # Optional (see note above)
@@ -62,5 +62,6 @@ setup(
             'emhass=emhass.command_line:main',
         ],
     },
-    package_data={'emhass': ['templates/index.html','static/style.css','static/img/emhass_icon.png','static/img/emhass_logo_short.svg']},
+    package_data={'emhass': ['templates/index.html','templates/template.html',
+    'static/style.css','static/img/emhass_icon.png','static/img/emhass_logo_short.svg']},
 )

{emhass-0.7.7 → emhass-0.8.0}/src/emhass/command_line.py

@@ -75,12 +75,14 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str,
         else:
             days_list = utils.get_days_list(retrieve_hass_conf['days_to_retrieve'])
             var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']]
-            rh.get_data(days_list, var_list,
-                        minimal_response=False, significant_changes_only=False)
-        rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'],
+            if not rh.get_data(days_list, var_list,
+                        minimal_response=False, significant_changes_only=False):
+                return False 
+        if not rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'],
                         set_zero_min = retrieve_hass_conf['set_zero_min'], 
                         var_replace_zero = retrieve_hass_conf['var_replace_zero'], 
-                        var_interp = retrieve_hass_conf['var_interp'])
+                        var_interp = retrieve_hass_conf['var_interp']):
+            return False
         df_input_data = rh.df_final.copy()
         # What we don't need for this type of action
         P_PV_forecast, P_load_forecast, df_input_data_dayahead = None, None, None
@@ -89,6 +91,9 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str,
         df_weather = fcst.get_weather_forecast(method=optim_conf['weather_forecast_method'])
         P_PV_forecast = fcst.get_power_from_weather(df_weather)
         P_load_forecast = fcst.get_load_forecast(method=optim_conf['load_forecast_method'])
+        if isinstance(P_load_forecast,bool) and not P_load_forecast:
+            logger.error("Unable to get sensor_power_photovoltaics or sensor_power_load_no_var_loads")
+            return False
         df_input_data_dayahead = pd.DataFrame(np.transpose(np.vstack([P_PV_forecast.values,P_load_forecast.values])),
                                               index=P_PV_forecast.index,
                                               columns=['P_PV_forecast', 'P_load_forecast'])
@@ -107,12 +112,14 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str,
         else:
             days_list = utils.get_days_list(1)
             var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']]
-            rh.get_data(days_list, var_list,
-                        minimal_response=False, significant_changes_only=False)
-        rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'],
+            if not rh.get_data(days_list, var_list,
+                        minimal_response=False, significant_changes_only=False):
+                return False
+        if not rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'],
                         set_zero_min = retrieve_hass_conf['set_zero_min'], 
                         var_replace_zero = retrieve_hass_conf['var_replace_zero'], 
-                        var_interp = retrieve_hass_conf['var_interp'])
+                        var_interp = retrieve_hass_conf['var_interp']):
+            return False
         df_input_data = rh.df_final.copy()
         # Get PV and load forecasts
         df_weather = fcst.get_weather_forecast(method=optim_conf['weather_forecast_method'])
@@ -143,7 +150,8 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str,
         else:
             days_list = utils.get_days_list(days_to_retrieve)
             var_list = [var_model]
-            rh.get_data(days_list, var_list)
+            if not rh.get_data(days_list, var_list):
+                return False
             df_input_data = rh.df_final.copy()
     elif set_type == "publish-data":
         df_input_data, df_input_data_dayahead = None, None
@@ -415,7 +423,7 @@ def forecast_model_tune(input_data_dict: dict, logger: logging.Logger,
                 mlf = pickle.load(inp)
         else:
             logger.error("The ML forecaster file was not found, please run a model fit method before this tune method")
-            return
+            return None, None
     # Tune the model
     df_pred_optim = mlf.tune(debug=debug)
     # Save model
@@ -540,6 +548,9 @@ def publish_data(input_data_dict: dict, logger: logging.Logger,
                                     publish_prefix = publish_prefix)
     # Publish the optimization status
     custom_cost_fun_id = params['passed_data']['custom_optim_status_id']
+    if "optim_status" not in opt_res_latest:
+        opt_res_latest["optim_status"] = 'Optimal'
+        logger.warning("no optim_status in opt_res_latest, run an optimization task first")
     input_data_dict['rh'].post_data(opt_res_latest['optim_status'], idx_closest, 
                                     custom_cost_fun_id["entity_id"], 
                                     custom_cost_fun_id["unit_of_measurement"],

{emhass-0.7.7 → emhass-0.8.0}/src/emhass/forecast.py

@@ -585,12 +585,14 @@ class Forecast(object):
                 with open(pathlib.Path(self.root) / 'data' / 'test_df_final.pkl', 'rb') as inp:
                     rh.df_final, days_list, _ = pickle.load(inp)
             else:
-                days_list = get_days_list(days_min_load_forecast)
-                rh.get_data(days_list, var_list)
-            rh.prepare_data(self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'],
+                days_list = get_days_list(days_min_load_forecast) 
+                if not rh.get_data(days_list, var_list):
+                    return False
+            if  not rh.prepare_data(self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'],
                             set_zero_min = self.retrieve_hass_conf['set_zero_min'], 
                             var_replace_zero = var_replace_zero, 
-                            var_interp = var_interp)
+                            var_interp = var_interp):
+                return False
             df = rh.df_final.copy()[[self.var_load_new]]
         if method == 'naive': # using a naive approach
             mask_forecast_out = (df.index > days_list[-1] - self.optim_conf['delta_forecast'])

{emhass-0.7.7 → emhass-0.8.0}/src/emhass/retrieve_hass.py

@@ -92,6 +92,7 @@ class RetrieveHass:
         """
         self.logger.info("Retrieve hass get data method initiated...")
         self.df_final = pd.DataFrame()
+        x = 0 #iterate based on days
         # Looping on each day from days list
         for day in days_list:
         
@@ -115,8 +116,14 @@ class RetrieveHass:
                 try:
                     response = get(url, headers=headers)
                 except Exception:
-                    return "Request Get Error"
+                    self.logger.error("Unable to access Home Assistance instance, check URL")
+                    self.logger.error("If using addon, try setting url and token to 'empty'")
+                    return False
                 else:
+                    if response.status_code == 401:
+                        self.logger.error("Unable to access Home Assistance instance, TOKEN/KEY")
+                        self.logger.error("If using addon, try setting url and token to 'empty'")
+                        return False
                     if response.status_code > 299:
                         return f"Request Get Error: {response.status_code}"
                 '''import bz2 # Uncomment to save a serialized data for tests
@@ -126,13 +133,18 @@ class RetrieveHass:
                 try: # Sometimes when there are connection problems we need to catch empty retrieved json
                     data = response.json()[0]
                 except IndexError:
-                    self.logger.error("The retrieved JSON is empty, check that correct day or variable names are passed")
-                    self.logger.error("Either the names of the passed variables are not correct or days_to_retrieve is larger than the recorded history of your sensor (check your recorder settings)")
-                    break
+                    if x is 0:
+                        self.logger.error("The retrieved JSON is empty, A sensor:" + var + " may have 0 days of history or passed sensor may not be correct")
+                    else:
+                        self.logger.error("The retrieved JSON is empty, days_to_retrieve may be larger than the recorded history of sensor:" + var + "  (check your recorder settings)")
+                    return False
                 df_raw = pd.DataFrame.from_dict(data)
                 if len(df_raw) == 0:
-                    self.logger.error("Retrieved empty Dataframe, check that correct day or variable names are passed")
-                    self.logger.error("Either the names of the passed variables are not correct or days_to_retrieve is larger than the recorded history of your sensor (check your recorder settings)")
+                    if x is 0:
+                        self.logger.error("The retrieved Dataframe is empty, A sensor:" + var + " may have 0 days of history or passed sensor may not be correct")
+                    else:
+                        self.logger.error("Retrieved empty Dataframe, days_to_retrieve may be larger than the recorded history of sensor:" + var + "  (check your recorder settings)")
+                    return False
                 if i == 0: # Defining the DataFrame container
                     from_date = pd.to_datetime(df_raw['last_changed'], format="ISO8601").min()
                     to_date = pd.to_datetime(df_raw['last_changed'], format="ISO8601").max()
@@ -147,11 +159,13 @@ class RetrieveHass:
                 df_tp = df_tp.resample(self.freq).mean()
                 df_day = pd.concat([df_day, df_tp], axis=1)
             
+            x += 1
             self.df_final = pd.concat([self.df_final, df_day], axis=0)
         self.df_final = set_df_index_freq(self.df_final)
         if self.df_final.index.freq != self.freq:
             self.logger.error("The inferred freq from data is not equal to the defined freq in passed parameters")
-
+            return False
+        return True
     
     def prepare_data(self, var_load: str, load_negative: Optional[bool] = False, set_zero_min: Optional[bool] = True,
                      var_replace_zero: Optional[list] = None, var_interp: Optional[list] = None) -> None:
@@ -185,6 +199,10 @@ class RetrieveHass:
             self.df_final.drop([var_load], inplace=True, axis=1)
         except KeyError:
             self.logger.error("Variable "+var_load+" was not found. This is typically because no data could be retrieved from Home Assistant")
+            return False
+        except ValueError:
+            self.logger.error("sensor.power_photovoltaics and sensor.power_load_no_var_loads should not be the same")
+            return False   
         if set_zero_min: # Apply minimum values
             self.df_final.clip(lower=0.0, inplace=True, axis=1)
             self.df_final.replace(to_replace=0.0, value=np.nan, inplace=True)
@@ -215,6 +233,7 @@ class RetrieveHass:
             self.df_final.index = self.df_final.index.tz_convert(self.time_zone)
         # Drop datetimeindex duplicates on final DF
         self.df_final = self.df_final[~self.df_final.index.duplicated(keep='first')]
+        return True
     
     @staticmethod
     def get_attr_data_dict(data_df: pd.DataFrame, idx: int, entity_id: str, 

{emhass-0.7.7 → emhass-0.8.0}/src/emhass/static/style.css

@@ -568,6 +568,10 @@ button {
   overflow-x: auto !important;
   margin: 0 auto;
   margin-bottom: 10px;
+  animation-name: fadeInOpacity;
+  animation-iteration-count: 1;
+  animation-timing-function: ease-in-out;
+  animation-duration: .8s;
 }
 
 /* Set min size for diagrams  */
@@ -688,7 +692,7 @@ th {
   font-size: 4.0em;
   animation-name: fadeInOpacity;
   animation-iteration-count: 1;
-  animation-timing-function: ease-in;
+  animation-timing-function: ease-in-out;
   animation-duration: .5s;
 }
 
@@ -699,7 +703,7 @@ th {
   font-size: 4.0em;
   animation-name: fadeInOpacity;
   animation-iteration-count: 1;
-  animation-timing-function: ease-in;
+  animation-timing-function: ease-in-out;
   animation-duration: .5s;
 }
 
@@ -741,7 +745,10 @@ th {
   margin: 0 auto;
   width: fit-content;
   height: fit-content;
-  transition: 0.3s;
+  animation-name: fadeInOpacity;
+  animation-iteration-count: 1;
+  animation-timing-function: ease-in-out;
+  animation-duration: .8s;
 }
 
 #alert-text {
@@ -807,7 +814,8 @@ th {
   text-align: center;
 }
 
-#input-select {
+#input-select,
+#input-clear {
   width: 77px;
 }
 

{emhass-0.7.7 → emhass-0.8.0}/src/emhass/templates/index.html

@@ -18,7 +18,7 @@
             <!-- action button elements section -->
             <div class="loading-div">
                 <h4>Use the buttons below to manually launch different optimization tasks</h4>
-                <div id=loader></div> <!-- post status dynamic element -->
+                <div id=loader></div> <!-- status dynamic element -->
             </div>
             <button type="button" id="perfect-optim" class="button button1">Perfect Optimization</button>
             <button type="button" id="dayahead-optim" class="button button2">Day-ahead Optimization</button>
@@ -44,6 +44,7 @@
                         <option value="List" selected>List</option>
                         <option value="Box">Box</option>
                     </select>
+                    <button type="button" id="input-clear">Clear</button>
                 </div>
             </div>
             <div id="input-container"> <!-- (Box/List) dynamic input elements will be added here -->
@@ -57,18 +58,22 @@
                 <p id="alert-text"></p>
             </div>
         </div>
+        <!-- -->
         <br><br><br>
         <!-- dynamic table/diagram elements section -->
     </div>
-    {% for plot in injection_dict %} <!-- diagrams/tables elements will be added here -->
-    <div class="table_div">
-        {{injection_dict[plot]}}
-    </div>
-    {% endfor %}
-    <!-- -->
-    <footer class="footer">
-        <p style="margin-top:10px; text-align:center;">&copy; MIT License | Copyright (c) 2021-2023 David HERNANDEZ</p>
-    </footer>
+    <div id="template"> <!-- table/diagram container element  -->
+        {% for plot in injection_dict %} <!-- diagrams/tables elements will be added here -->
+        <div class="table_div">
+            {{injection_dict[plot]}}
+        </div>
+        {% endfor %}
+        <div>
+            <!-- -->
+            <footer class="footer">
+                <p style="margin-top:10px; text-align:center;">&copy; MIT License | Copyright (c) 2021-2023 David
+                    HERNANDEZ</p>
+            </footer>
 </body>
 
 
@@ -83,16 +88,20 @@
     //function pushing data via post, triggered by button action
     async function formAction(action) {
         var data = inputToJson()
-        if (data !== 0) { //don't run if there is an error in the data Json
-            showChangeStatus("loading") // show loading div for status
-            response = await fetch(`{{ basename }}/action/${action}`, {
+        if (data !== 0) { //don't run if there is an error in the input (box/list) Json data 
+            showChangeStatus("loading", {}) // show loading div for status
+            const response = await fetch(`{{ basename }}/action/${action}`, { //fetch data from webserver.py
                 method: "POST",
                 headers: {
                     "Content-Type": "application/json",
                 },
-                body: JSON.stringify(data), //post can only send data via strings
-            }).then(saveStorage()) //save to storage if successful 
-            showChangeStatus(response.status) //replace loading, show tick or cross
+                body: JSON.stringify(data), //note that post can only send data via strings
+            })
+            if (response.status == 201) {
+                showChangeStatus(response.status, {})
+                saveStorage() //save to storage if successful
+            } //if successful 
+            else { showChangeStatus(response.status, await response.json()) } // else get Log data from response
         }
         else {
             showChangeStatus("remove") //replace loading, show tick or cross with none 
@@ -100,27 +109,52 @@
     }
 
     //function in control of status icons of post above
-    function showChangeStatus(status) {
-        var loading = document.getElementById("loader") //element showing status 
-        if (status === "remove") { //show loading logo
+    async function showChangeStatus(status, logJson) {
+        var loading = document.getElementById("loader") //element showing statuses
+        if (status === "remove") { //remove all
             loading.innerHTML = "";
-            loading.classList.remove("loading"); //append class with loading animation styling
+            loading.classList.remove("loading");
         }
         else if (status === "loading") { //show loading logo
             loading.innerHTML = "";
             loading.classList.add("loading"); //append class with loading animation styling
         }
-        else if (status === 201) { //then show a tick 
+        else if (status === 201) { //if status is 201, then show a tick 
             loading.classList.remove("loading")
             loading.innerHTML = `<p class=tick>&#x2713;</p>`
+            getTemplate() //get updated templates
 
         }
         else { //then show a cross 
             loading.classList.remove("loading")
-            loading.innerHTML = `<p class=cross>&#x292C;</p>`
+            loading.innerHTML = `<p class=cross>&#x292C;</p>` //show cross icon to indicate an error
+            if (logJson.length != 0) {
+                document.getElementById("alert-text").textContent = "\r\n\u2022 " + logJson.join("\r\n\u2022 ") //show received log data in alert box
+                document.getElementById("alert").style.display = "block";
+                document.getElementById("alert").style.textAlign = "left";
+            }
         }
     }
 
+    //get rendered html template with containing new table data  
+    async function getTemplate() { //fetch data from webserver.py
+        let htmlTemplateData = ""
+        response = await fetch(`{{ basename }}/template/table-template`, {
+            method: "GET"
+        })
+        blob = await response.blob() //get data blob
+        htmlTemplateData = await new Response(blob).text() //obtain html from blob
+        templateDiv = document.getElementById("template") //get template container element to override 
+        templateDiv.innerHTML = htmlTemplateData //override container inner html with new data
+        var scripts = Array.from(templateDiv.getElementsByTagName('script')); //replace script tags manually
+        for (const script of scripts) {
+            var TempScript = document.createElement("script");
+            TempScript.innerHTML = script.innerHTML
+            script.parentElement.appendChild(TempScript)
+        }
+    }
+
+
     //test localStorage support
     function testStorage() {
         try {
@@ -142,7 +176,7 @@
         if (testStorage()) { //if local storage exists and works
             let selectElement = document.getElementById('input-select') // select button element
             var input_container = document.getElementById('input-container'); // container div containing all dynamic input elements (Box/List)
-            if (localStorage.getItem("input_container_content")) { //If items already stored in local storage, then override default
+            if (localStorage.getItem("input_container_content") && localStorage.getItem("input_container_content") !== "{}" ) { //If items already stored in local storage, then override default
                 if (selectElement.value == "Box") { //if Box is selected, show saved json data into box 
                     document.getElementById("text-area").value = localStorage.getItem("input_container_content");
                 }
@@ -177,7 +211,7 @@
         input_container_child = input_container.firstElementChild; //work out which element is first inside container div
         var jsonReturnData = {};
 
-        if (input_container_child == null){ //if no elements in container then return empty
+        if (input_container_child == null) { //if no elements in container then return empty
             return (jsonReturnData)
         };
         //if List return box json
@@ -200,6 +234,7 @@
             catch (error) { //if json error, show in alert box
                 document.getElementById("alert-text").textContent = "\r\n" + error + "\r\n" + "JSON Error: String values may not be wrapped in quotes"
                 document.getElementById("alert").style.display = "block";
+                document.getElementById("alert").style.textAlign = "center";
                 return (0)
             }
         }
@@ -265,10 +300,7 @@
         //if box is selected, remove input-list elements and replace (with text-area)
         if (selectElement.value == "Box") {
             if (input_container_child_name == "input-list" || input_container_child === null) { // if input list exists or no Box element
-                inputListArr = input_container.getElementsByClassName('input-list');
-                while (inputListArr[0]) { //while there is still input-list elements remove 
-                    inputListArr[0].parentNode.removeChild(inputListArr[0]);
-                };
+                input_container.innerHTML = ""; //remove input-list list elements via erasing container innerHTML
                 let div = document.createElement('div'); //add input-box element
                 div.className = "input-box";
                 div.innerHTML = `
@@ -291,6 +323,30 @@
         }
     }
 
+    //clear stored input data from localStorage (if any), clear input elements
+    async function ClearInputData(id) {
+        if (testStorage() && localStorage.getItem("input_container_content") !== null) {
+            localStorage.setItem("input_container_content", "{}")
+        }
+        ClearInputElements();
+
+    }
+
+    //clear input elements
+    async function ClearInputElements() {
+        let selectElement = document.getElementById('input-select')
+        var input_container = document.getElementById('input-container'); 
+        if (selectElement.value == "Box") { 
+            document.getElementById("text-area").value = "{}";
+        }
+        if (selectElement.value == "List") { 
+            input_container.innerHTML = "";
+        }
+
+    }
+
+
+
     //add listeners to buttons
     [
         "dayahead-optim",
@@ -310,6 +366,9 @@
     [
         "input-select"
     ].forEach((id) => document.getElementById(id).addEventListener('change', () => getSavedData(id)));
+    [
+        "input-clear"
+    ].forEach((id) => document.getElementById(id).addEventListener('click', () => ClearInputData(id)));
 </script>
 
 </html>

emhass-0.8.0/src/emhass/templates/template.html

@@ -0,0 +1,5 @@
+{% for plot in injection_dict %} <!-- diagrams/tables elements will be added here -->
+<div class="table_div">
+    {{injection_dict[plot]}}
+</div>
+{% endfor %}

{emhass-0.7.7 → emhass-0.8.0}/src/emhass/utils.py

@@ -610,7 +610,7 @@ def build_params(params: dict, params_secrets: dict, options: dict, addon: int,
         params['optim_conf']['load_cost_forecast_method'] = options.get('load_cost_forecast_method',params['optim_conf']['load_cost_forecast_method'])
         if options.get('list_set_deferrable_load_single_constant',None) != None: 
             params['optim_conf']['set_def_constant'] = [i['set_deferrable_load_single_constant'] for i in options.get('list_set_deferrable_load_single_constant')]
-        if options.get('list_peak_hours_periods_start_hours',None) != None:
+        if options.get('list_peak_hours_periods_start_hours',None) != None and options.get('list_peak_hours_periods_end_hours',None) != None:
             start_hours_list = [i['peak_hours_periods_start_hours'] for i in options['list_peak_hours_periods_start_hours']]
             end_hours_list = [i['peak_hours_periods_end_hours'] for i in options['list_peak_hours_periods_end_hours']]
             num_peak_hours = len(start_hours_list)
@@ -656,6 +656,41 @@ def build_params(params: dict, params_secrets: dict, options: dict, addon: int,
         params['plant_conf']['SOCmin'] = options.get('battery_minimum_state_of_charge',params['plant_conf']['SOCmin']) 
         params['plant_conf']['SOCmax'] = options.get('battery_maximum_state_of_charge',params['plant_conf']['SOCmax']) 
         params['plant_conf']['SOCtarget'] = options.get('battery_target_state_of_charge',params['plant_conf']['SOCtarget'])
+
+                # Check parameter lists have the same amounts as deferrable loads
+        # If not, set defaults it fill in gaps
+        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['def_start_timestep']):
+            logger.warning("def_start_timestep / list_start_timesteps_of_each_deferrable_load does not match number in num_def_loads, adding default values to parameter")
+            for x in range(len(params['optim_conf']['def_start_timestep']), params['optim_conf']['num_def_loads']):
+                params['optim_conf']['def_start_timestep'].append(0)
+        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['def_end_timestep']):
+            logger.warning("def_end_timestep / list_end_timesteps_of_each_deferrable_load does not match number in num_def_loads, adding default values to parameter")
+            for x in range(len(params['optim_conf']['def_end_timestep']), params['optim_conf']['num_def_loads']):
+                params['optim_conf']['def_end_timestep'].append(0)
+        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['set_def_constant']):
+            logger.warning("set_def_constant / list_set_deferrable_load_single_constant does not match number in num_def_loads, adding default values to parameter")
+            for x in range(len(params['optim_conf']['set_def_constant']), params['optim_conf']['num_def_loads']):
+                params['optim_conf']['set_def_constant'].append(False)
+        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['treat_def_as_semi_cont']):
+            logger.warning("treat_def_as_semi_cont / list_treat_deferrable_load_as_semi_cont does not match number in num_def_loads, adding default values to parameter")
+            for x in range(len(params['optim_conf']['treat_def_as_semi_cont']), params['optim_conf']['num_def_loads']):
+                params['optim_conf']['treat_def_as_semi_cont'].append(True)        
+        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['def_total_hours']):
+            logger.warning("def_total_hours / list_operating_hours_of_each_deferrable_load does not match number in num_def_loads, adding default values to parameter")
+            for x in range(len(params['optim_conf']['def_total_hours']), params['optim_conf']['num_def_loads']):
+                params['optim_conf']['def_total_hours'].append(0)                   
+        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['P_deferrable_nom']):
+            logger.warning("P_deferrable_nom / list_nominal_power_of_deferrable_loads does not match number in num_def_loads, adding default values to parameter")
+            for x in range(len(params['optim_conf']['P_deferrable_nom']), params['optim_conf']['num_def_loads']):
+                params['optim_conf']['P_deferrable_nom'].append(0)   
+        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['list_hp_periods']):
+            logger.warning("list_hp_periods / list_peak_hours_periods_(start&end)_hours does not match number in num_def_loads, adding default values to parameter")
+            for x in range(len(params['optim_conf']['list_hp_periods']), params['optim_conf']['num_def_loads']):
+                params['optim_conf']['list_hp_periods'].append({'period_hp_'+str(x+1):[{'start':'02:54'},{'end':'20:24'}]})
+        # days_to_retrieve should be no less then 2
+        if params['retrieve_hass_conf']['days_to_retrieve'] < 2:
+            params['retrieve_hass_conf']['days_to_retrieve'] = 2
+            logger.warning("days_to_retrieve should not be lower then 2, setting days_to_retrieve to 2. Make sure your sensors also have at least 2 days of history")
     else:
         params['params_secrets'] = params_secrets
     # The params dict

{emhass-0.7.7 → emhass-0.8.0}/src/emhass/web_server.py

@@ -7,7 +7,7 @@ from requests import get
 from waitress import serve
 from importlib.metadata import version, PackageNotFoundError
 from pathlib import Path
-import os, json, argparse, pickle, yaml, logging
+import os, json, argparse, pickle, yaml, logging,  re
 from distutils.util import strtobool
 import pandas as pd
 
@@ -21,7 +21,41 @@ from emhass.utils import get_injection_dict, get_injection_dict_forecast_model_f
 # Define the Flask instance
 app = Flask(__name__)
 
+#check logfile for error, anything after string match if provided 
+def checkFileLog(refString=None):
+    if (refString is not None): 
+       logArray = grabLog(refString) #grab reduced log array
+    else: 
+        if ((data_path / 'actionLogs.txt')).exists():
+            with open(str(data_path / 'actionLogs.txt'), "r") as fp:
+                    logArray = fp.readlines()
+    for logString in logArray:
+            if (logString.split(' ', 1)[0] == "ERROR"):
+                return True     
+    return False
 
+#find string in logs, append all lines after to return
+def grabLog(refString): 
+    isFound = []
+    output = []
+    if ((data_path / 'actionLogs.txt')).exists():
+            with open(str(data_path / 'actionLogs.txt'), "r") as fp:
+                    logArray = fp.readlines()
+            for x in range(len(logArray)-1): #find all matches and log key in isFound
+                if (re.search(refString,logArray[x])):
+                   isFound.append(x)
+            if len(isFound) != 0:
+                for x in range(isFound[-1],len(logArray)): #use isFound to extract last related action logs  
+                    output.append(logArray[x])
+    return output
+
+#clear the log file
+def clearFileLog(): 
+    if ((data_path / 'actionLogs.txt')).exists():
+        with open(str(data_path / 'actionLogs.txt'), "w") as fp:
+            fp.truncate()    
+
+#initial index page render
 @app.route('/')
 def index():
     app.logger.info("EMHASS server online, serving index.html...")
@@ -39,6 +73,24 @@ def index():
     basename = request.headers.get("X-Ingress-Path", "")
     return make_response(template.render(injection_dict=injection_dict, basename=basename))
 
+#get actions 
+@app.route('/template/<action_name>', methods=['GET'])
+def template_action(action_name):
+    app.logger.info(" >> Sending rendered template table data")
+    if action_name == 'table-template':
+        file_loader = PackageLoader('emhass', 'templates')
+        env = Environment(loader=file_loader)
+        template = env.get_template('template.html')
+        if (data_path / 'injection_dict.pkl').exists():
+            with open(str(data_path / 'injection_dict.pkl'), "rb") as fid:
+                injection_dict = pickle.load(fid)
+        else:
+            app.logger.warning("The data container dictionary is empty... Please launch an optimization task")
+            injection_dict={}        
+        basename = request.headers.get("X-Ingress-Path", "")    
+        return make_response(template.render(injection_dict=injection_dict, basename=basename))
+
+#post actions 
 @app.route('/action/<action_name>', methods=['POST'])
 def action_call(action_name):
     with open(str(data_path / 'params.pkl'), "rb") as fid:
@@ -46,49 +98,71 @@ def action_call(action_name):
     runtimeparams = request.get_json(force=True)
     params = json.dumps(params)
     runtimeparams = json.dumps(runtimeparams)
+    ActionStr = " >> Setting input data dict"
+    app.logger.info(ActionStr)
     input_data_dict = set_input_data_dict(config_path, str(data_path), costfun, 
         params, runtimeparams, action_name, app.logger)
+    if not input_data_dict:
+        return make_response(grabLog(ActionStr), 400)
     if action_name == 'publish-data':
-        app.logger.info(" >> Publishing data...")
+        ActionStr = " >> Publishing data..."
+        app.logger.info(ActionStr)
         _ = publish_data(input_data_dict, app.logger)
         msg = f'EMHASS >> Action publish-data executed... \n'
-        return make_response(msg, 201)
+        if not checkFileLog(ActionStr):
+            return make_response(msg, 201)
+        return make_response(grabLog(ActionStr), 400)
     elif action_name == 'perfect-optim':
-        app.logger.info(" >> Performing perfect optimization...")
+        ActionStr = " >> Performing perfect optimization..."
+        app.logger.info(ActionStr)
         opt_res = perfect_forecast_optim(input_data_dict, app.logger)
         injection_dict = get_injection_dict(opt_res)
         with open(str(data_path / 'injection_dict.pkl'), "wb") as fid:
             pickle.dump(injection_dict, fid)
         msg = f'EMHASS >> Action perfect-optim executed... \n'
-        return make_response(msg, 201)
+        if not checkFileLog(ActionStr):
+            return make_response(msg, 201)
+        return make_response(grabLog(ActionStr), 400)
     elif action_name == 'dayahead-optim':
-        app.logger.info(" >> Performing dayahead optimization...")
+        ActionStr = " >> Performing dayahead optimization..."
+        app.logger.info(ActionStr)
         opt_res = dayahead_forecast_optim(input_data_dict, app.logger)
         injection_dict = get_injection_dict(opt_res)
         with open(str(data_path / 'injection_dict.pkl'), "wb") as fid:
             pickle.dump(injection_dict, fid)
         msg = f'EMHASS >> Action dayahead-optim executed... \n'
-        return make_response(msg, 201)
+        if not checkFileLog(ActionStr):
+            return make_response(msg, 201)
+        return make_response(grabLog(ActionStr), 400)
     elif action_name == 'naive-mpc-optim':
-        app.logger.info(" >> Performing naive MPC optimization...")
+        ActionStr = " >> Performing naive MPC optimization..."
+        app.logger.info(ActionStr)
         opt_res = naive_mpc_optim(input_data_dict, app.logger)
         injection_dict = get_injection_dict(opt_res)
         with open(str(data_path / 'injection_dict.pkl'), "wb") as fid:
             pickle.dump(injection_dict, fid)
         msg = f'EMHASS >> Action naive-mpc-optim executed... \n'
-        return make_response(msg, 201)
+        if not checkFileLog(ActionStr):
+            return make_response(msg, 201)
+        return make_response(grabLog(ActionStr), 400)
     elif action_name == 'forecast-model-fit':
-        app.logger.info(" >> Performing a machine learning forecast model fit...")
+        ActionStr = " >> Performing a machine learning forecast model fit..."
+        app.logger.info(ActionStr)
         df_fit_pred, _, mlf = forecast_model_fit(input_data_dict, app.logger)
         injection_dict = get_injection_dict_forecast_model_fit(
             df_fit_pred, mlf)
         with open(str(data_path / 'injection_dict.pkl'), "wb") as fid:
             pickle.dump(injection_dict, fid)
         msg = f'EMHASS >> Action forecast-model-fit executed... \n'
-        return make_response(msg, 201)
+        if not checkFileLog(ActionStr):
+            return make_response(msg, 201)
+        return make_response(grabLog(ActionStr), 400)
     elif action_name == 'forecast-model-predict':
-        app.logger.info(" >> Performing a machine learning forecast model predict...")
+        ActionStr = " >> Performing a machine learning forecast model predict..."
+        app.logger.info(ActionStr)
         df_pred = forecast_model_predict(input_data_dict, app.logger)
+        if df_pred is None:
+            return make_response(grabLog(ActionStr), 400)
         table1 = df_pred.reset_index().to_html(classes='mystyle', index=False)
         injection_dict = {}
         injection_dict['title'] = '<h2>Custom machine learning forecast model predict</h2>'
@@ -97,16 +171,23 @@ def action_call(action_name):
         with open(str(data_path / 'injection_dict.pkl'), "wb") as fid:
             pickle.dump(injection_dict, fid)
         msg = f'EMHASS >> Action forecast-model-predict executed... \n'
-        return make_response(msg, 201)
+        if not checkFileLog(ActionStr):
+            return make_response(msg, 201)
+        return make_response(grabLog(ActionStr), 400)
     elif action_name == 'forecast-model-tune':
-        app.logger.info(" >> Performing a machine learning forecast model tune...")
-        df_pred_optim, mlf = forecast_model_tune(input_data_dict, app.logger)
+        ActionStr = " >> Performing a machine learning forecast model tune..."
+        app.logger.info(ActionStr)
+        df_pred_optim, mlf = forecast_model_tune(input_data_dict, app.logger)    
+        if df_pred_optim is None or  mlf is None:
+            return make_response(grabLog(ActionStr), 400)
         injection_dict = get_injection_dict_forecast_model_tune(
             df_pred_optim, mlf)
         with open(str(data_path / 'injection_dict.pkl'), "wb") as fid:
             pickle.dump(injection_dict, fid)
         msg = f'EMHASS >> Action forecast-model-tune executed... \n'
-        return make_response(msg, 201)
+        if not checkFileLog(ActionStr):
+            return make_response(msg, 201)
+        return make_response(grabLog(ActionStr), 400)
     else:
         app.logger.error("ERROR: passed action is not valid")
         msg = f'EMHASS >> ERROR: Passed action is not valid... \n'
@@ -124,12 +205,14 @@ if __name__ == "__main__":
     use_options = os.getenv('USE_OPTIONS', default=False)
     # Define the paths
     if args.addon==1:
-        OPTIONS_PATH = os.getenv('OPTIONS_PATH', default="/data/options.json")
+        OPTIONS_PATH = os.getenv('OPTIONS_PATH', default="/app/options.json")
         options_json = Path(OPTIONS_PATH)
-        CONFIG_PATH = os.getenv("CONFIG_PATH", default="/usr/src/config_emhass.yaml")
+        CONFIG_PATH = os.getenv("CONFIG_PATH", default="/app/config_emhass.yaml")
         #Obtain url and key from ENV or ARG
         hass_url = os.getenv("EMHASS_URL", default=args.url)
-        key =  os.getenv("EMHASS_KEY", default=args.key) 
+        key =  os.getenv("SUPERVISOR_TOKEN", default=args.key) 
+        if hass_url != "http://supervisor/core/api":
+            key =  os.getenv("EMHASS_KEY", key)  
         #If url or key is None, Set as empty string to reduce NoneType errors bellow
         if key is None: key = ""
         if hass_url is None: hass_url = ""
@@ -139,7 +222,7 @@ if __name__ == "__main__":
                 options = json.load(data)
         else:
             app.logger.error("options.json does not exists")
-        DATA_PATH = os.getenv("DATA_PATH", default="/share/")
+        DATA_PATH = os.getenv("DATA_PATH", default="/app/data/")
     else:
         if use_options:
             OPTIONS_PATH = os.getenv('OPTIONS_PATH', default="/app/options.json")
@@ -256,27 +339,39 @@ if __name__ == "__main__":
         pickle.dump((config_path, params), fid)
 
     # Define logger
-    ch = logging.StreamHandler()
+    #stream logger
+    ch = logging.StreamHandler() 
     formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
     ch.setFormatter(formatter)
+    #Action File logger
+    fileLogger = logging.FileHandler(str(data_path / 'actionLogs.txt')) 
+    formatter = logging.Formatter('%(levelname)s - %(name)s - %(message)s')
+    fileLogger.setFormatter(formatter) # add format to Handler
     if logging_level == "DEBUG":
         app.logger.setLevel(logging.DEBUG)
         ch.setLevel(logging.DEBUG)
+        fileLogger.setLevel(logging.DEBUG)
     elif logging_level == "INFO":
         app.logger.setLevel(logging.INFO)
         ch.setLevel(logging.INFO)
+        fileLogger.setLevel(logging.INFO)
     elif logging_level == "WARNING":
         app.logger.setLevel(logging.WARNING)
         ch.setLevel(logging.WARNING)
+        fileLogger.setLevel(logging.WARNING)
     elif logging_level == "ERROR":
         app.logger.setLevel(logging.ERROR)
         ch.setLevel(logging.ERROR)
+        fileLogger.setLevel(logging.ERROR)
     else:
         app.logger.setLevel(logging.DEBUG)
         ch.setLevel(logging.DEBUG)
+        fileLogger.setLevel(logging.DEBUG)
     app.logger.propagate = False
     app.logger.addHandler(ch)
-    
+    app.logger.addHandler(fileLogger)   
+    clearFileLog() #Clear Action File logger file, ready for new instance
+
     # Launch server
     port = int(os.environ.get('PORT', 5000))
     app.logger.info("Launching the emhass webserver at: http://"+web_ui_url+":"+str(port))

{emhass-0.7.7 → emhass-0.8.0}/src/emhass.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: emhass
-Version: 0.7.7
+Version: 0.8.0
 Summary: An Energy Management System for Home Assistant
 Home-page: https://github.com/davidusb-geek/emhass
 Author: David HERNANDEZ
@@ -460,7 +460,7 @@ Check the dedicated section in the documentation here: [https://emhass.readthedo
 
 ## Development
 
-Pull request are very much accepted on this project. For development you can find some instructions here [Development](./docs/develop.md)
+Pull request are very much accepted on this project. For development you can find some instructions here [Development](https://emhass.readthedocs.io/en/latest/develop.html)
 
 ## Troubleshooting
 

{emhass-0.7.7 → emhass-0.8.0}/src/emhass.egg-info/SOURCES.txt

@@ -40,6 +40,7 @@ src/emhass/static/style.css
 src/emhass/static/img/emhass_icon.png
 src/emhass/static/img/emhass_logo_short.svg
 src/emhass/templates/index.html
+src/emhass/templates/template.html
 tests/test_command_line_utils.py
 tests/test_forecast.py
 tests/test_machine_learning_forecaster.py

{emhass-0.7.7 → emhass-0.8.0}/tests/test_retrieve_hass.py

@@ -84,7 +84,7 @@ class TestRetrieveHass(unittest.TestCase):
         days_list = get_days_list(1)
         var_list = [self.retrieve_hass_conf['var_load']]
         response = self.rh.get_data(days_list, var_list)
-        self.assertEqual(response, "Request Get Error")
+        self.assertFalse(response)
 
     def test_get_data_mock(self):
         with requests_mock.mock() as m: