emhass 0.10.5__py3-none-any.whl → 0.11.0__py3-none-any.whl

emhass/utils.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import csv
+import os
 from typing import Tuple, Optional
 from datetime import datetime, timedelta, timezone
 import logging
@@ -9,6 +11,7 @@ import json
 import copy
 import numpy as np
 import pandas as pd
+from requests import get
 import yaml
 import pytz
 import ast
@@ -19,7 +22,6 @@ pd.options.plotting.backend = "plotly"
 
 from emhass.machine_learning_forecaster import MLForecaster
 
-
 def get_root(file: str, num_parent: Optional[int] = 3) -> str:
     """
     Get the root absolute path of the working directory.
@@ -62,7 +64,10 @@ def get_logger(fun_name: str, emhass_conf: dict, save_to_file: Optional[bool] =
     logger.propagate = True
     logger.fileSetting = save_to_file
     if save_to_file:
-        ch = logging.FileHandler(emhass_conf['data_path'] / 'logger_emhass.log')
+        if  os.path.isdir(emhass_conf['data_path']):
+            ch = logging.FileHandler(emhass_conf['data_path'] / 'logger_emhass.log')
+        else:
+            raise Exception("Unable to access data_path: "+emhass_conf['data_path'])  
     else:
         ch = logging.StreamHandler()
     if logging_level == "DEBUG":
@@ -89,7 +94,7 @@ def get_logger(fun_name: str, emhass_conf: dict, save_to_file: Optional[bool] =
     return logger, ch
 
 
-def get_forecast_dates(freq: int, delta_forecast: int, timedelta_days: Optional[int] = 0
+def get_forecast_dates(freq: int, delta_forecast: int, time_zone: datetime.tzinfo, timedelta_days: Optional[int] = 0
                        ) -> pd.core.indexes.datetimes.DatetimeIndex:
     """
     Get the date_range list of the needed future dates using the delta_forecast parameter.
@@ -109,7 +114,7 @@ def get_forecast_dates(freq: int, delta_forecast: int, timedelta_days: Optional[
     end_forecast = (start_forecast + pd.Timedelta(days=delta_forecast)).replace(microsecond=0)
     forecast_dates = pd.date_range(start=start_forecast, 
         end=end_forecast+timedelta(days=timedelta_days)-freq, 
-        freq=freq).round(freq, ambiguous='infer', nonexistent='shift_forward')
+        freq=freq, tz=time_zone).tz_convert('utc').round(freq, ambiguous='infer', nonexistent='shift_forward').tz_convert(time_zone)
     return forecast_dates
 
 
@@ -121,7 +126,7 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
 
     :param runtimeparams: Json string containing the runtime parameters dict.
     :type runtimeparams: str
-    :param params: Configuration parameters passed from data/options.json
+    :param params: Built configuration parameters
     :type params: str
     :param retrieve_hass_conf: Container for data retrieving parameters.
     :type retrieve_hass_conf: dict
@@ -137,14 +142,17 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
     :rtype: Tuple[str, dict]
 
     """
+    # check if passed params is a dict
     if (params != None) and (params != "null"):
-        params = json.loads(params)
+        if type(params) is str:
+            params = json.loads(params)
     else:
         params = {}
+
     # Some default data needed
     custom_deferrable_forecast_id = []
     custom_predicted_temperature_id = []
-    for k in range(optim_conf["num_def_loads"]):
+    for k in range(optim_conf['number_of_deferrable_loads']):
         custom_deferrable_forecast_id.append(
             {
                 "entity_id": "sensor.p_deferrable{}".format(k),
@@ -224,12 +232,20 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
             params["passed_data"][key] = value
     else:
         params["passed_data"] = default_passed_dict
+
     if runtimeparams is not None:
-        runtimeparams = json.loads(runtimeparams)
-        freq = int(retrieve_hass_conf["freq"].seconds / 60.0)
-        delta_forecast = int(optim_conf["delta_forecast"].days)
-        forecast_dates = get_forecast_dates(freq, delta_forecast)
-        if set_type == "regressor-model-fit":            
+        if type(runtimeparams) is str:
+            runtimeparams = json.loads(runtimeparams)
+       # Format required date/time parameters 
+        optimization_time_step = int(
+            retrieve_hass_conf['optimization_time_step'].seconds / 60.0)
+        delta_forecast = int(optim_conf['delta_forecast_daily'].days)
+        time_zone = retrieve_hass_conf["time_zone"]
+        forecast_dates = get_forecast_dates(
+            optimization_time_step, delta_forecast, time_zone)
+       
+       # regressor-model-fit
+        if set_type == "regressor-model-fit":
             if "csv_file" in runtimeparams:
                 csv_file = runtimeparams["csv_file"]
                 params["passed_data"]["csv_file"] = csv_file
@@ -249,6 +265,8 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
             else:
                 date_features = runtimeparams["date_features"]
                 params["passed_data"]["date_features"] = date_features
+        
+        # regressor-model-predict
         if set_type == "regressor-model-predict":
             if "new_values" in runtimeparams:
                 new_values = runtimeparams["new_values"]
@@ -262,6 +280,7 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
             if "target" in runtimeparams:
                 target = runtimeparams["target"]
                 params["passed_data"]["target"] = target
+        
         # Treating special data passed for MPC control case
         if set_type == "naive-mpc-optim":
             if "prediction_horizon" not in runtimeparams.keys():
@@ -270,74 +289,79 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
                 prediction_horizon = runtimeparams["prediction_horizon"]
             params["passed_data"]["prediction_horizon"] = prediction_horizon
             if "soc_init" not in runtimeparams.keys():
-                soc_init = plant_conf["SOCtarget"]
+                soc_init = plant_conf['battery_target_state_of_charge']
             else:
                 soc_init = runtimeparams["soc_init"]
             params["passed_data"]["soc_init"] = soc_init
             if "soc_final" not in runtimeparams.keys():
-                soc_final = plant_conf["SOCtarget"]
+                soc_final = plant_conf['battery_target_state_of_charge']
             else:
                 soc_final = runtimeparams["soc_final"]
             params["passed_data"]["soc_final"] = soc_final
-            if "def_total_hours" not in runtimeparams.keys():
-                def_total_hours = optim_conf["def_total_hours"]
+            if 'operating_hours_of_each_deferrable_load' not in runtimeparams.keys():
+                def_total_hours = optim_conf['operating_hours_of_each_deferrable_load']
             else:
-                def_total_hours = runtimeparams["def_total_hours"]
-            params["passed_data"]["def_total_hours"] = def_total_hours
-            if "def_start_timestep" not in runtimeparams.keys():
-                def_start_timestep = optim_conf["def_start_timestep"]
+                def_total_hours = runtimeparams['operating_hours_of_each_deferrable_load']
+            params["passed_data"]['operating_hours_of_each_deferrable_load'] = def_total_hours
+            if 'start_timesteps_of_each_deferrable_load' in runtimeparams.keys():
+                def_start_timestep = runtimeparams['start_timesteps_of_each_deferrable_load']
             else:
-                def_start_timestep = runtimeparams["def_start_timestep"]
-            params["passed_data"]["def_start_timestep"] = def_start_timestep
-            if "def_end_timestep" not in runtimeparams.keys():
-                def_end_timestep = optim_conf["def_end_timestep"]
+                def_start_timestep = runtimeparams.get(
+                    'def_start_timestep', optim_conf['start_timesteps_of_each_deferrable_load'])
+            params["passed_data"]['start_timesteps_of_each_deferrable_load'] = def_start_timestep
+            if 'end_timesteps_of_each_deferrable_load' in runtimeparams.keys():
+                def_end_timestep = runtimeparams['end_timesteps_of_each_deferrable_load']
             else:
-                def_end_timestep = runtimeparams["def_end_timestep"]
-            params["passed_data"]["def_end_timestep"] = def_end_timestep
-            forecast_dates = copy.deepcopy(forecast_dates)[0:prediction_horizon]
+                def_end_timestep = runtimeparams.get(
+                    'def_end_timestep', optim_conf['end_timesteps_of_each_deferrable_load'])
+            params["passed_data"]['end_timesteps_of_each_deferrable_load'] = def_end_timestep
+            forecast_dates = copy.deepcopy(forecast_dates)[
+                0:prediction_horizon]
         else:
             params["passed_data"]["prediction_horizon"] = None
             params["passed_data"]["soc_init"] = None
             params["passed_data"]["soc_final"] = None
-            params["passed_data"]["def_total_hours"] = None
-            params["passed_data"]["def_start_timestep"] = None
-            params["passed_data"]["def_end_timestep"] = None
+            params["passed_data"]['operating_hours_of_each_deferrable_load'] = None
+            params["passed_data"]['start_timesteps_of_each_deferrable_load'] = None
+            params["passed_data"]['end_timesteps_of_each_deferrable_load'] = None
         # Treat passed forecast data lists
-        list_forecast_key = ['pv_power_forecast', 'load_power_forecast', 'load_cost_forecast', 'prod_price_forecast', 'outdoor_temperature_forecast']
-        forecast_methods = ['weather_forecast_method', 'load_forecast_method', 'load_cost_forecast_method', 'prod_price_forecast_method', 'outdoor_temperature_forecast_method']
-        # Param to save forecast cache (i.e. Solcast)
-        if "weather_forecast_cache" not in runtimeparams.keys():
-            weather_forecast_cache = False
-        else:
-            weather_forecast_cache = runtimeparams["weather_forecast_cache"]
-        params["passed_data"]["weather_forecast_cache"] = weather_forecast_cache  
-        # Param to make sure optimization only uses cached data. (else produce error)
-        if "weather_forecast_cache_only" not in runtimeparams.keys():
-            weather_forecast_cache_only = False
-        else:
-            weather_forecast_cache_only = runtimeparams["weather_forecast_cache_only"]
-        params["passed_data"]["weather_forecast_cache_only"] = weather_forecast_cache_only  
+        list_forecast_key = ['pv_power_forecast', 'load_power_forecast',
+                             'load_cost_forecast', 'prod_price_forecast', 'outdoor_temperature_forecast']
+        forecast_methods = ['weather_forecast_method', 'load_forecast_method', 'load_cost_forecast_method',
+                            'production_price_forecast_method', 'outdoor_temperature_forecast_method']
+        
+        # Loop forecasts, check if value is a list and greater than or equal to forecast_dates
         for method, forecast_key in enumerate(list_forecast_key):
             if forecast_key in runtimeparams.keys():
                 if type(runtimeparams[forecast_key]) == list and len(runtimeparams[forecast_key]) >= len(forecast_dates):
                     params['passed_data'][forecast_key] = runtimeparams[forecast_key]
                     optim_conf[forecast_methods[method]] = 'list'
                 else:
-                    logger.error(f"ERROR: The passed data is either not a list or the length is not correct, length should be {str(len(forecast_dates))}")
-                    logger.error(f"Passed type is {str(type(runtimeparams[forecast_key]))} and length is {str(len(runtimeparams[forecast_key]))}")
-                list_non_digits = [x for x in runtimeparams[forecast_key] if not (isinstance(x, int) or isinstance(x, float))]
+                    logger.error(
+                        f"ERROR: The passed data is either not a list or the length is not correct, length should be {str(len(forecast_dates))}")
+                    logger.error(
+                        f"Passed type is {str(type(runtimeparams[forecast_key]))} and length is {str(len(runtimeparams[forecast_key]))}")
+                # Check if string contains list, if so extract
+                if type(runtimeparams[forecast_key]) == str:
+                    if type(ast.literal_eval(runtimeparams[forecast_key])) == list:
+                        runtimeparams[forecast_key] = ast.literal_eval(runtimeparams[forecast_key])
+                list_non_digits = [x for x in runtimeparams[forecast_key] if not (
+                    isinstance(x, int) or isinstance(x, float))]
                 if len(list_non_digits) > 0:
-                    logger.warning(f"There are non numeric values on the passed data for {forecast_key}, check for missing values (nans, null, etc)")
+                    logger.warning(
+                        f"There are non numeric values on the passed data for {forecast_key}, check for missing values (nans, null, etc)")
                     for x in list_non_digits:
-                        logger.warning(f"This value in {forecast_key} was detected as non digits: {str(x)}")
+                        logger.warning(
+                            f"This value in {forecast_key} was detected as non digits: {str(x)}")
             else:
                 params['passed_data'][forecast_key] = None
+       
         # Treat passed data for forecast model fit/predict/tune at runtime
-        if "days_to_retrieve" not in runtimeparams.keys():
-            days_to_retrieve = 9
+        if 'historic_days_to_retrieve' in runtimeparams.keys():
+            days_to_retrieve = runtimeparams['historic_days_to_retrieve']
         else:
-            days_to_retrieve = runtimeparams["days_to_retrieve"]
-        params["passed_data"]["days_to_retrieve"] = days_to_retrieve
+            days_to_retrieve = runtimeparams.get('days_to_retrieve', 9)
+        params["passed_data"]['historic_days_to_retrieve'] = days_to_retrieve
         if "model_type" not in runtimeparams.keys():
             model_type = "load_forecast"
         else:
@@ -371,12 +395,14 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
         if "perform_backtest" not in runtimeparams.keys():
             perform_backtest = False
         else:
-            perform_backtest = ast.literal_eval(str(runtimeparams["perform_backtest"]).capitalize())
+            perform_backtest = ast.literal_eval(
+                str(runtimeparams["perform_backtest"]).capitalize())
         params["passed_data"]["perform_backtest"] = perform_backtest
         if "model_predict_publish" not in runtimeparams.keys():
             model_predict_publish = False
         else:
-            model_predict_publish = ast.literal_eval(str(runtimeparams["model_predict_publish"]).capitalize())
+            model_predict_publish = ast.literal_eval(
+                str(runtimeparams["model_predict_publish"]).capitalize())
         params["passed_data"]["model_predict_publish"] = model_predict_publish
         if "model_predict_entity_id" not in runtimeparams.keys():
             model_predict_entity_id = "sensor.p_load_forecast_custom_model"
@@ -408,6 +434,7 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
         else:
             mlr_predict_friendly_name = runtimeparams["mlr_predict_friendly_name"]
         params["passed_data"]["mlr_predict_friendly_name"] = mlr_predict_friendly_name
+        
         # Treat passed data for other parameters
         if "alpha" not in runtimeparams.keys():
             alpha = 0.5
@@ -419,68 +446,124 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
         else:
             beta = runtimeparams["beta"]
         params["passed_data"]["beta"] = beta
-        # Treat optimization configuration parameters passed at runtime
-        if "num_def_loads" in runtimeparams.keys():
-            optim_conf["num_def_loads"] = runtimeparams["num_def_loads"]
-        if "P_deferrable_nom" in runtimeparams.keys():
-            optim_conf["P_deferrable_nom"] = runtimeparams["P_deferrable_nom"]
-        if "def_total_hours" in runtimeparams.keys():
-            optim_conf["def_total_hours"] = runtimeparams["def_total_hours"]
-        if "def_start_timestep" in runtimeparams.keys():
-            optim_conf["def_start_timestep"] = runtimeparams["def_start_timestep"]
-        if "def_end_timestep" in runtimeparams.keys():
-            optim_conf["def_end_timestep"] = runtimeparams["def_end_timestep"]
+        # Param to save forecast cache (i.e. Solcast)
+        if "weather_forecast_cache" not in runtimeparams.keys():
+            weather_forecast_cache = False
+        else:
+            weather_forecast_cache = runtimeparams["weather_forecast_cache"]
+        params["passed_data"]["weather_forecast_cache"] = weather_forecast_cache
+        # Param to make sure optimization only uses cached data. (else produce error)
+        if "weather_forecast_cache_only" not in runtimeparams.keys():
+            weather_forecast_cache_only = False
+        else:
+            weather_forecast_cache_only = runtimeparams["weather_forecast_cache_only"]
+        params["passed_data"]["weather_forecast_cache_only"] = weather_forecast_cache_only
+        # A condition to manually save entity data under data_path/entities after optimization
+        if "entity_save" not in runtimeparams.keys():
+            entity_save = ""
+        else:
+            entity_save = runtimeparams["entity_save"]
+        params["passed_data"]["entity_save"] = entity_save
+        # A condition to put a prefix on all published data, or check for saved data under prefix name
+        if "publish_prefix" not in runtimeparams.keys():
+            publish_prefix = ""
+        else:
+            publish_prefix = runtimeparams["publish_prefix"]
+        params["passed_data"]["publish_prefix"] = publish_prefix
+
+        # Treat optimization (optim_conf) configuration parameters passed at runtime
+        if 'number_of_deferrable_loads' in runtimeparams.keys():
+            optim_conf['number_of_deferrable_loads'] = runtimeparams['number_of_deferrable_loads']
+        if 'num_def_loads' in runtimeparams.keys():
+            optim_conf['number_of_deferrable_loads'] = runtimeparams['num_def_loads']
+        if 'nominal_power_of_deferrable_loads' in runtimeparams.keys():
+            optim_conf['nominal_power_of_deferrable_loads'] = runtimeparams['nominal_power_of_deferrable_loads']
+        if 'P_deferrable_nom' in runtimeparams.keys():
+            optim_conf['nominal_power_of_deferrable_loads'] = runtimeparams['P_deferrable_nom']
+        if 'operating_hours_of_each_deferrable_load' in runtimeparams.keys():
+            optim_conf['operating_hours_of_each_deferrable_load'] = runtimeparams['operating_hours_of_each_deferrable_load']
+        if 'def_total_hours' in runtimeparams.keys():
+            optim_conf['operating_hours_of_each_deferrable_load'] = runtimeparams['def_total_hours']
+        if 'start_timesteps_of_each_deferrable_load' in runtimeparams.keys():
+            optim_conf['start_timesteps_of_each_deferrable_load'] = runtimeparams['start_timesteps_of_each_deferrable_load']
+        if 'end_timesteps_of_each_deferrable_load' in runtimeparams.keys():
+            optim_conf['end_timesteps_of_each_deferrable_load'] = runtimeparams['end_timesteps_of_each_deferrable_load']
         if "def_current_state" in runtimeparams.keys():
-            optim_conf["def_current_state"] = [bool(s) for s in runtimeparams["def_current_state"]]
-        if "treat_def_as_semi_cont" in runtimeparams.keys():
-            optim_conf["treat_def_as_semi_cont"] = [
+            optim_conf["def_current_state"] = [
+                bool(s) for s in runtimeparams["def_current_state"]]
+        if 'treat_deferrable_load_as_semi_cont' in runtimeparams.keys():
+            optim_conf['treat_deferrable_load_as_semi_cont'] = [
+                ast.literal_eval(str(k).capitalize())
+                for k in runtimeparams['treat_deferrable_load_as_semi_cont']
+            ]
+        if 'treat_def_as_semi_cont' in runtimeparams.keys():
+            optim_conf['treat_deferrable_load_as_semi_cont'] = [
                 ast.literal_eval(str(k).capitalize())
-                for k in runtimeparams["treat_def_as_semi_cont"]
+                for k in runtimeparams['treat_def_as_semi_cont']
             ]
-        if "set_def_constant" in runtimeparams.keys():
-            optim_conf["set_def_constant"] = [
-                ast.literal_eval(str(k).capitalize()) for k in runtimeparams["set_def_constant"]
+        if 'set_deferrable_load_single_constant' in runtimeparams.keys():
+            optim_conf['set_deferrable_load_single_constant'] = [
+                ast.literal_eval(str(k).capitalize()) for k in runtimeparams['set_deferrable_load_single_constant']
             ]
-        if "def_start_penalty" in runtimeparams.keys():
-            optim_conf["def_start_penalty"] = [
-                ast.literal_eval(str(k).capitalize()) for k in runtimeparams["def_start_penalty"]
+        if 'set_def_constant' in runtimeparams.keys():
+            optim_conf['set_deferrable_load_single_constant'] = [
+                ast.literal_eval(str(k).capitalize()) for k in runtimeparams['set_def_constant']
+            ]
+        if 'set_deferrable_startup_penalty' in runtimeparams.keys():
+            optim_conf['set_deferrable_startup_penalty'] = [
+                ast.literal_eval(str(k).capitalize()) for k in runtimeparams['set_deferrable_startup_penalty']
+            ]
+        if 'def_start_penalty' in runtimeparams.keys():
+            optim_conf['set_deferrable_startup_penalty'] = [
+                ast.literal_eval(str(k).capitalize()) for k in runtimeparams['def_start_penalty']
             ]
         if 'def_load_config' in runtimeparams:
             optim_conf["def_load_config"] = runtimeparams['def_load_config']
+        if 'weight_battery_discharge' in runtimeparams.keys():
+            optim_conf['weight_battery_discharge'] = runtimeparams[
+                'weight_battery_discharge'
+            ]
+        if 'weight_battery_charge' in runtimeparams.keys():
+            optim_conf['weight_battery_charge'] = runtimeparams['weight_battery_charge']
+
+        # Treat retrieve data from Home Assistant (retrieve_hass_conf) configuration parameters passed at runtime
+        if 'optimization_time_step' in runtimeparams.keys():
+            retrieve_hass_conf['optimization_time_step'] = pd.to_timedelta(
+                runtimeparams['optimization_time_step'], "minutes")
+        if 'continual_publish' in runtimeparams.keys():
+            retrieve_hass_conf['continual_publish'] = bool(
+                runtimeparams['continual_publish'])
         if "solcast_api_key" in runtimeparams.keys():
             retrieve_hass_conf["solcast_api_key"] = runtimeparams["solcast_api_key"]
-            optim_conf["weather_forecast_method"] = "solcast"
+            optim_conf['weather_forecast_method'] = "solcast"
         if "solcast_rooftop_id" in runtimeparams.keys():
             retrieve_hass_conf["solcast_rooftop_id"] = runtimeparams[
                 "solcast_rooftop_id"
             ]
-            optim_conf["weather_forecast_method"] = "solcast"
+            optim_conf['weather_forecast_method'] = "solcast"
         if "solar_forecast_kwp" in runtimeparams.keys():
             retrieve_hass_conf["solar_forecast_kwp"] = runtimeparams[
                 "solar_forecast_kwp"
             ]
-            optim_conf["weather_forecast_method"] = "solar.forecast"
-        if "weight_battery_discharge" in runtimeparams.keys():
-            optim_conf["weight_battery_discharge"] = runtimeparams[
-                "weight_battery_discharge"
-            ]
-        if "weight_battery_charge" in runtimeparams.keys():
-            optim_conf["weight_battery_charge"] = runtimeparams["weight_battery_charge"]
-        if 'freq' in runtimeparams.keys():
-            retrieve_hass_conf['freq'] = pd.to_timedelta(runtimeparams['freq'], "minutes")
-        if 'continual_publish' in runtimeparams.keys():
-            retrieve_hass_conf['continual_publish'] = bool(runtimeparams['continual_publish'])
-        # Treat plant configuration parameters passed at runtime
-        if "SOCmin" in runtimeparams.keys():
-            plant_conf["SOCmin"] = runtimeparams["SOCmin"]
-        if "SOCmax" in runtimeparams.keys():
-            plant_conf["SOCmax"] = runtimeparams["SOCmax"]
-        if "SOCtarget" in runtimeparams.keys():
-            plant_conf["SOCtarget"] = runtimeparams["SOCtarget"]
-        if "Pd_max" in runtimeparams.keys():
-            plant_conf["Pd_max"] = runtimeparams["Pd_max"]
-        if "Pc_max" in runtimeparams.keys():
-            plant_conf["Pc_max"] = runtimeparams["Pc_max"]
+            optim_conf['weather_forecast_method'] = "solar.forecast"
+
+        # Treat system model parameters (plant) configuration parameters passed at runtime
+        if 'battery_minimum_state_of_charge' in runtimeparams.keys() or 'SOCmin' in runtimeparams.keys():
+            plant_conf['battery_minimum_state_of_charge'] = runtimeparams.get(
+                'battery_minimum_state_of_charge', runtimeparams.get('SOCmin'))
+        if 'battery_maximum_state_of_charge' in runtimeparams.keys() or 'SOCmax' in runtimeparams.keys():
+            plant_conf['battery_maximum_state_of_charge'] = runtimeparams.get(
+                'battery_maximum_state_of_charge', runtimeparams.get('SOCmax'))
+        if 'battery_target_state_of_charge' in runtimeparams.keys() or 'SOCtarget' in runtimeparams.keys():
+            plant_conf['battery_target_state_of_charge'] = runtimeparams.get(
+                'battery_target_state_of_charge', runtimeparams.get('SOCtarget'))
+        if 'battery_discharge_power_max' in runtimeparams.keys() or 'Pd_max' in runtimeparams.keys():
+            plant_conf['battery_discharge_power_max'] = runtimeparams.get(
+                'battery_discharge_power_max', runtimeparams.get('Pd_max'))
+        if 'battery_charge_power_max' in runtimeparams.keys() or 'Pc_max' in runtimeparams.keys():
+            plant_conf['battery_charge_power_max'] = runtimeparams.get(
+                'battery_charge_power_max', runtimeparams.get('Pc_max'))
+        
         # Treat custom entities id's and friendly names for variables
         if "custom_pv_forecast_id" in runtimeparams.keys():
             params["passed_data"]["custom_pv_forecast_id"] = runtimeparams[
@@ -534,84 +617,47 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
             params["passed_data"]["custom_predicted_temperature_id"] = runtimeparams[
                 "custom_predicted_temperature_id"
             ]
-        # A condition to put a prefix on all published data, or check for saved data under prefix name
-        if "publish_prefix" not in runtimeparams.keys():
-            publish_prefix = ""
-        else:
-            publish_prefix = runtimeparams["publish_prefix"]
-        params["passed_data"]["publish_prefix"] = publish_prefix
-        # A condition to manually save entity data under data_path/entities after optimization
-        if "entity_save" not in runtimeparams.keys():
-            entity_save = ""
-        else:
-            entity_save = runtimeparams["entity_save"]
-        params["passed_data"]["entity_save"] = entity_save
+       
     # Serialize the final params
-    params = json.dumps(params)
+    params = json.dumps(params, default=str)
     return params, retrieve_hass_conf, optim_conf, plant_conf
 
 
-def get_yaml_parse(emhass_conf: dict, use_secrets: Optional[bool] = True,
-                   params: Optional[str] = None) -> Tuple[dict, dict, dict]:
+def get_yaml_parse(params: str, logger: logging.Logger) -> Tuple[dict, dict, dict]:
     """
-    Perform parsing of the config.yaml file.
+    Perform parsing of the params into the configuration catagories 
     
-    :param emhass_conf: Dictionary containing the needed emhass paths
-    :type emhass_conf: dict
-    :param use_secrets: Indicate if we should use a secrets file or not.
-        Set to False for unit tests.
-    :type use_secrets: bool, optional
-    :param params: Configuration parameters passed from data/options.json
+    :param params: Built configuration parameters
     :type params: str
+    :param logger: The logger object
+    :type logger: logging.Logger
     :return: A tuple with the dictionaries containing the parsed data
     :rtype: tuple(dict)
 
     """
-    if params is None:
-        with open(emhass_conf["config_path"], 'r') as file:
-            input_conf = yaml.load(file, Loader=yaml.FullLoader)
-    else:
-        input_conf = json.loads(params)
-    if use_secrets:
-        if params is None:
-            with open(emhass_conf["config_path"].parent / 'secrets_emhass.yaml', 'r') as file: # Assume secrets and config file paths are the same 
-                input_secrets = yaml.load(file, Loader=yaml.FullLoader)
+    if params:
+        if type(params) is str:
+            input_conf = json.loads(params)
         else:
-            input_secrets = input_conf.pop("params_secrets", None)
-
-    if type(input_conf["retrieve_hass_conf"]) == list:  # if using old config version
-        retrieve_hass_conf = dict(
-            {key: d[key] for d in input_conf["retrieve_hass_conf"] for key in d}
-        )
+            input_conf = params
     else:
-        retrieve_hass_conf = input_conf.get("retrieve_hass_conf", {})
+        input_conf = {}
+        logger.error("No params have been detected for get_yaml_parse")
+        return False, False, False
 
-    if use_secrets:
-        retrieve_hass_conf.update(input_secrets)
-    else:
-        retrieve_hass_conf["hass_url"] = "http://supervisor/core/api"
-        retrieve_hass_conf["long_lived_token"] = "${SUPERVISOR_TOKEN}"
-        retrieve_hass_conf["time_zone"] = "Europe/Paris"
-        retrieve_hass_conf["lat"] = 45.83
-        retrieve_hass_conf["lon"] = 6.86
-        retrieve_hass_conf["alt"] = 4807.8
-    retrieve_hass_conf["freq"] = pd.to_timedelta(retrieve_hass_conf["freq"], "minutes")
-    retrieve_hass_conf["time_zone"] = pytz.timezone(retrieve_hass_conf["time_zone"])
-
-    if type(input_conf["optim_conf"]) == list:
-        optim_conf = dict({key: d[key] for d in input_conf["optim_conf"] for key in d})
-    else:
-        optim_conf = input_conf.get("optim_conf", {})
+    optim_conf = input_conf.get("optim_conf", {})
 
-    optim_conf["list_hp_periods"] = dict(
-        (key, d[key]) for d in optim_conf["list_hp_periods"] for key in d
-    )
-    optim_conf["delta_forecast"] = pd.Timedelta(days=optim_conf["delta_forecast"])
+    retrieve_hass_conf = input_conf.get("retrieve_hass_conf", {})
 
-    if type(input_conf["plant_conf"]) == list:
-        plant_conf = dict({key: d[key] for d in input_conf["plant_conf"] for key in d})
-    else:
-        plant_conf = input_conf.get("plant_conf", {})
+    plant_conf = input_conf.get("plant_conf", {})
+
+    # Format time parameters
+    if optim_conf.get('delta_forecast_daily',None) is not None:
+        optim_conf['delta_forecast_daily'] = pd.Timedelta(days=optim_conf['delta_forecast_daily'])
+    if retrieve_hass_conf.get('optimization_time_step',None) is not None:
+        retrieve_hass_conf['optimization_time_step'] = pd.to_timedelta(retrieve_hass_conf['optimization_time_step'], "minutes")
+    if retrieve_hass_conf.get('time_zone',None) is not None:
+        retrieve_hass_conf["time_zone"] = pytz.timezone(retrieve_hass_conf["time_zone"])    
 
     return retrieve_hass_conf, optim_conf, plant_conf
 
@@ -762,156 +808,402 @@ def get_injection_dict_forecast_model_tune(df_pred_optim: pd.DataFrame, mlf: MLF
     injection_dict["figure_0"] = image_path_0
     return injection_dict
 
+def build_config(emhass_conf: dict, logger: logging.Logger, defaults_path: str, config_path: Optional[str] = None, 
+                 legacy_config_path: Optional[str] = None) -> dict:
+    """
+    Retrieve parameters from configuration files. 
+    priority order (low - high) = defaults_path, config_path legacy_config_path
 
-def build_params(params: dict, params_secrets: dict, options: dict, addon: int,
+    :param emhass_conf: Dictionary containing the needed emhass paths
+    :type emhass_conf: dict
+    :param logger: The logger object
+    :type logger: logging.Logger
+    :param defaults_path: path to config file for parameter defaults (config_defaults.json)
+    :type defaults_path: str
+    :param config_path: path to the main configuration file (config.json)
+    :type config_path: str
+    :param legacy_config_path: path to legacy config file (config_emhass.yaml)
+    :type legacy_config_path: str
+    :return: The built config dictionary
+    :rtype: dict
+    """
+
+    # Read default parameters (default root_path/data/config_defaults.json)
+    if defaults_path and pathlib.Path(defaults_path).is_file():
+        with defaults_path.open('r') as data:
+            config = json.load(data)
+    else:
+        logger.error("config_defaults.json. does not exist ")
+        return False
+    
+    # Read user config parameters if provided (default /share/config.json)
+    if config_path and pathlib.Path(config_path).is_file():
+        with config_path.open('r') as data:
+            # Set override default parameters (config_defaults) with user given parameters (config.json)
+            logger.info("Obtaining parameters from config.json:")
+            config.update(json.load(data))
+    else:
+        logger.info("config.json does not exist, or has not been passed")
+        logger.info("you may like to generate the config.json file on the configuration page")
+
+    # Check to see if legacy config_emhass.yaml was provided (default /app/config_emhass.yaml)
+    # Convert legacy parameter definitions/format to match config.json
+    if legacy_config_path and pathlib.Path(legacy_config_path).is_file():
+        with open(legacy_config_path, 'r') as data:
+            legacy_config = yaml.load(data, Loader=yaml.FullLoader)
+            legacy_config_parameters = build_legacy_config_params(emhass_conf,legacy_config,logger)
+            if type(legacy_config_parameters) is not bool:
+                logger.info("Obtaining parameters from config_emhass.yaml:")
+                config.update(legacy_config_parameters)     
+
+    return config
+
+
+def build_legacy_config_params(emhass_conf: dict, legacy_config: dict,
+                 logger: logging.Logger) -> dict:
+    """
+    Build a config dictionary with legacy config_emhass.yaml file. 
+    Uses the associations file to convert parameter naming conventions (to config.json/config_defaults.json).
+    Extracts the parameter values and formats to match config.json.
+
+    :param emhass_conf: Dictionary containing the needed emhass paths
+    :type emhass_conf: dict
+    :param legacy_config: The legacy config dictionary
+    :type legacy_config: dict
+    :param logger: The logger object
+    :type logger: logging.Logger
+    :return: The built config dictionary
+    :rtype: dict
+    """
+
+
+    # Association file key reference
+    # association[0] = config catagories
+    # association[1] = legacy parameter name
+    # association[2] = parameter (config.json/config_defaults.json)
+    # association[3] = parameter list name if exists (not used, from legacy options.json) 
+
+    # Check each config catagories exists, else create blank dict for categories (avoid errors)
+    legacy_config['retrieve_hass_conf'] = legacy_config.get('retrieve_hass_conf',{})
+    legacy_config['optim_conf'] = legacy_config.get('optim_conf',{})
+    legacy_config['plant_conf'] = legacy_config.get('plant_conf',{})
+    config = {}
+
+    # Use associations list to map legacy parameter name with config.json parameter name
+    if emhass_conf['associations_path'].exists():
+            with emhass_conf['associations_path'].open('r') as data:
+                associations = list(csv.reader(data, delimiter=","))
+    else:
+        logger.error("Cant find associations file (associations.csv) in: "  + str(emhass_conf['associations_path']))
+        return False
+    
+    # Loop through all parameters in association file
+    # Append config with existing legacy config parameters (converting alternative parameter naming conventions with associations list)
+    for association in associations:
+        # if legacy config catagories exists and if legacy parameter exists in config catagories
+        if legacy_config.get(association[0],None) is not None and legacy_config[association[0]].get(association[1],None) is not None:
+            config[association[2]] = legacy_config[association[0]][association[1]]
+            
+            # If config now has load_peak_hour_periods, extract from list of dict
+            if association[2] == "load_peak_hour_periods" and type(config[association[2]]) is list:
+                config[association[2]] = dict((key, d[key]) for d in config[association[2]] for key in d)
+                
+    return config
+    # params['associations_dict'] = associations_dict
+
+def param_to_config(param: dict,
                  logger: logging.Logger) -> dict:
     """
-    Build the main params dictionary from the loaded options.json when using the add-on.
+    A function that extracts the parameters from param back to the config.json format.
+    Extracts parameters from config catagories.
+    Attempts to exclude secrets hosed in retrieve_hass_conf.
+    
+    :param params: Built configuration parameters
+    :type param: dict
+    :param logger: The logger object
+    :type logger: logging.Logger
+    :return: The built config dictionary
+    :rtype: dict
+    """ 
+    logger.debug("Converting param to config")
+
+    return_config = {}
+
+    config_catagories = ["retrieve_hass_conf","optim_conf","plant_conf"]
+    secret_params = ["hass_url", "time_zone", "Latitude", "Longitude", "Altitude", "long_lived_token", "solcast_api_key", "solcast_rooftop_id", "solar_forecast_kwp"]
+    
+    # Loop through config catagories that contain config params, and extract
+    for config in config_catagories:
+        for parameter in param[config]:
+                # If parameter is not a secret, append to return_config
+                if parameter not in secret_params:
+                    return_config[str(parameter)] = param[config][parameter]
+    
+    return return_config
+
+def build_secrets(emhass_conf: dict, logger: logging.Logger, argument: Optional[dict] = {}, options_path: Optional[str] = None, 
+                 secrets_path: Optional[str] = None, no_response: Optional[bool] = False) -> Tuple[dict, dict]:    
+    """
+    Retrieve and build parameters from secrets locations (ENV, ARG, Secrets file (secrets_emhass.yaml/options.json) and/or Home Assistant (via API))
+    priority order (lwo to high) = Defaults (written in function), ENV, Options json file, Home Assistant API,  Secrets yaml file, Arguments
+    
+    :param emhass_conf: Dictionary containing the needed emhass paths
+    :type emhass_conf: dict
+    :param logger: The logger object
+    :type logger: logging.Logger
+    :param argument: dictionary of secrets arguments passed (url,key)
+    :type argument: dict
+    :param options_path: path to the options file (options.json) (usually provided bt EMHASS-Add-on)
+    :type options_path: str
+    :param secrets_path: path to secrets file (secrets_emhass.yaml)
+    :type secrets_path: str
+    :param no_response: bypass get request to Home Assistant (json response errors)
+    :type no_response: bool
+    :return: Updated emhass_conf, the built secrets dictionary
+    :rtype: Tuple[dict, dict]:
+    """
+
+    #Set defaults to be overwritten
+    params_secrets = {
+        "hass_url": "https://myhass.duckdns.org/",
+        "long_lived_token": "thatverylongtokenhere",
+        "time_zone": "Europe/Paris",
+        "Latitude": 45.83,
+        "Longitude": 6.86,
+        "Altitude": 4807.8,
+        "solcast_api_key": "yoursecretsolcastapikey",
+        "solcast_rooftop_id": "yourrooftopid",
+        "solar_forecast_kwp": 5
+    }
+
+    # Obtain Secrets from ENV?
+    params_secrets['hass_url'] = os.getenv("EMHASS_URL",params_secrets['hass_url'])
+    params_secrets['long_lived_token'] = os.getenv("SUPERVISOR_TOKEN", params_secrets['long_lived_token'])   
+    params_secrets['time_zone'] = os.getenv("TIME_ZONE", params_secrets['time_zone'])
+    params_secrets['Latitude'] = float(os.getenv("LAT", params_secrets['Latitude']))
+    params_secrets['Longitude'] = float(os.getenv("LON",  params_secrets['Longitude']))
+    params_secrets['Altitude'] = float(os.getenv("ALT", params_secrets['Altitude']))      
+
+    # Obtain secrets from options.json (Generated from EMHASS-Add-on, Home Assistant addon Configuration page) or Home Assistant API (from local Supervisor API)?
+    # Use local supervisor API to obtain secrets from Home Assistant if hass_url in options.json is empty and SUPERVISOR_TOKEN ENV exists (provided by Home Assistant when running the container as addon)
+    options = {}
+    if options_path and pathlib.Path(options_path).is_file():
+        with options_path.open('r') as data:
+            options = json.load(data)
+            
+            # Obtain secrets from Home Assistant?
+            url_from_options = options.get('hass_url', 'empty')
+            key_from_options = options.get('long_lived_token', 'empty') 
+
+            # If data path specified by options.json, overwrite emhass_conf['data_path']
+            if options.get('data_path', None) != None and pathlib.Path(options['data_path']).exists():
+                emhass_conf['data_path'] = pathlib.Path(options['data_path']);  
+            
+            # Check to use Home Assistant local API
+            if not no_response and \
+                (url_from_options == 'empty' or url_from_options == '' or url_from_options == "http://supervisor/core/api") and \
+                os.getenv("SUPERVISOR_TOKEN", None) is not None:
+                
+                params_secrets['long_lived_token'] = os.getenv("SUPERVISOR_TOKEN",None)
+                params_secrets['hass_url'] = "http://supervisor/core/api"
+                headers = {
+                "Authorization": "Bearer " + params_secrets['long_lived_token'],
+                "content-type": "application/json"
+                }
+                # Obtain secrets from Home Assistant via API
+                logger.debug("Obtaining secrets from Home Assistant Supervisor API")
+                response = get((params_secrets['hass_url'] + "/config"), headers=headers)
+                if response.status_code < 400:
+                    config_hass = response.json()
+                    params_secrets = {
+                    'hass_url': params_secrets['hass_url'],
+                    'long_lived_token': params_secrets['long_lived_token'],
+                    'time_zone': config_hass['time_zone'],
+                    'Latitude': config_hass['latitude'],
+                    'Longitude': config_hass['longitude'],
+                    'Altitude': config_hass['elevation']
+                    }
+                else: 
+                    # Obtain the url and key secrets if any from options.json (default /app/options.json)
+                    logger.warning("Error obtaining secrets from Home Assistant Supervisor API")
+                    logger.debug("Obtaining url and key secrets from options.json")
+                    if url_from_options != 'empty' and url_from_options != '':
+                        params_secrets['hass_url'] = url_from_options
+                    if key_from_options != 'empty' and key_from_options != '':
+                        params_secrets['long_lived_token'] = key_from_options
+                    if options.get('time_zone',"empty") != "empty" and options['time_zone'] != '':
+                        params_secrets['time_zone'] = options['time_zone']
+                    if options.get('Latitude',None) is not None and bool(options['Latitude']):
+                        params_secrets['Latitude'] = options['Latitude']
+                    if options.get('Longitude',None) is not None and bool(options['Longitude']):
+                        params_secrets['Longitude'] = options['Longitude']         
+                    if options.get('Altitude',None) is not None and bool(options['Altitude']):
+                        params_secrets['Altitude'] = options['Altitude']
+            else:
+                # Obtain the url and key secrets if any from options.json (default /app/options.json)
+                logger.debug("Obtaining url and key secrets from options.json")
+                if url_from_options != 'empty' and url_from_options != '':
+                    params_secrets['hass_url'] = url_from_options
+                if key_from_options != 'empty' and key_from_options != '':
+                    params_secrets['long_lived_token'] = key_from_options
+                if options.get('time_zone',"empty") != "empty" and options['time_zone'] != '':
+                    params_secrets['time_zone'] = options['time_zone']
+                if options.get('Latitude',None) is not None and bool(options['Latitude']):
+                    params_secrets['Latitude'] = options['Latitude']
+                if options.get('Longitude',None) is not None and bool(options['Longitude']):
+                    params_secrets['Longitude'] = options['Longitude']
+                if options.get('Altitude',None) is not None and bool(options['Altitude']):
+                    params_secrets['Altitude'] = options['Altitude']
+                
+            # Obtain the forecast secrets (if any) from options.json (default /app/options.json)
+            forecast_secrets = ["solcast_api_key","solcast_rooftop_id","solar_forecast_kwp"]
+            if any(x in forecast_secrets for x in list(options.keys())):
+                logger.debug("Obtaining forecast secrets from options.json")
+                if options.get('solcast_api_key',"empty") != "empty" and options['solcast_api_key'] != '':
+                    params_secrets['solcast_api_key'] = options['solcast_api_key']
+                if options.get('solcast_rooftop_id',"empty") != "empty" and options['solcast_rooftop_id'] != '':
+                    params_secrets['solcast_rooftop_id'] = options['solcast_rooftop_id']
+                if options.get('solar_forecast_kwp',None) and bool(options['solar_forecast_kwp']):
+                    params_secrets['solar_forecast_kwp'] = options['solar_forecast_kwp']
+     
+    # Obtain secrets from secrets_emhass.yaml? (default /app/secrets_emhass.yaml)
+    if secrets_path and pathlib.Path(secrets_path).is_file():
+        logger.debug("Obtaining secrets from secrets file")
+        with open(pathlib.Path(secrets_path), 'r') as file:
+            params_secrets.update(yaml.load(file, Loader=yaml.FullLoader))
+
+    # Receive key and url from ARG/arguments? 
+    if argument.get('url',None) is not None:
+       params_secrets['hass_url'] = argument['url']
+       logger.debug("Obtaining url from passed argument")  
+    if argument.get('key',None) is not None:
+        params_secrets['long_lived_token'] = argument['key']  
+        logger.debug("Obtaining long_lived_token from passed argument") 
+ 
+    return emhass_conf, params_secrets 
+ 
+   
 
-    :param params: The main params dictionary
-    :type params: dict
-    :param params_secrets: The dictionary containing the secret protected variables
+def build_params(emhass_conf: dict, params_secrets: dict, config: dict,
+                 logger: logging.Logger) -> dict:
+    """
+    Build the main params dictionary from the config and secrets
+    Appends configuration catagories used by emhass to the parameters. (with use of the associations file as a reference)
+    
+    :param emhass_conf: Dictionary containing the needed emhass paths
+    :type emhass_conf: dict
+    :param params_secrets: The dictionary containing the built secret variables
     :type params_secrets: dict
-    :param options: The load dictionary from options.json
-    :type options: dict
-    :param addon: A "bool" to select if we are using the add-on
-    :type addon: int
+    :param config: The dictionary of built config parameters 
+    :type config: dict
     :param logger: The logger object
     :type logger: logging.Logger
-    :return: The builded dictionary
+    :return: The built param dictionary
     :rtype: dict
     """
-    if addon == 1:
-        # Updating variables in retrieve_hass_conf
-        params["retrieve_hass_conf"]["freq"] = options.get("optimization_time_step", params["retrieve_hass_conf"]["freq"])
-        params["retrieve_hass_conf"]["days_to_retrieve"] = options.get("historic_days_to_retrieve", params["retrieve_hass_conf"]["days_to_retrieve"])
-        params["retrieve_hass_conf"]["var_PV"] = options.get("sensor_power_photovoltaics", params["retrieve_hass_conf"]["var_PV"])
-        params["retrieve_hass_conf"]["var_load"] = options.get("sensor_power_load_no_var_loads", params["retrieve_hass_conf"]["var_load"])
-        params["retrieve_hass_conf"]["load_negative"] = options.get("load_negative", params["retrieve_hass_conf"]["load_negative"])
-        params["retrieve_hass_conf"]["set_zero_min"] = options.get("set_zero_min", params["retrieve_hass_conf"]["set_zero_min"])
-        params["retrieve_hass_conf"]["var_replace_zero"] = [options.get("sensor_power_photovoltaics", params["retrieve_hass_conf"]["var_replace_zero"])]
-        params["retrieve_hass_conf"]["var_interp"] = [
-            options.get("sensor_power_photovoltaics", params["retrieve_hass_conf"]["var_PV"]),
-            options.get("sensor_power_load_no_var_loads", params["retrieve_hass_conf"]["var_load"])
-        ]
-        params["retrieve_hass_conf"]["method_ts_round"] = options.get("method_ts_round", params["retrieve_hass_conf"]["method_ts_round"])
-        params["retrieve_hass_conf"]["continual_publish"] = options.get("continual_publish", params["retrieve_hass_conf"]["continual_publish"])
-        # Update params Secrets if specified
-        params["params_secrets"] = params_secrets
-        params["params_secrets"]["time_zone"] = options.get("time_zone", params_secrets["time_zone"])
-        params["params_secrets"]["lat"] = options.get("Latitude", params_secrets["lat"])
-        params["params_secrets"]["lon"] = options.get("Longitude", params_secrets["lon"])
-        params["params_secrets"]["alt"] = options.get("Altitude", params_secrets["alt"])
-        # Updating variables in optim_conf
-        params["optim_conf"]["set_use_battery"] = options.get("set_use_battery", params["optim_conf"]["set_use_battery"])
-        params["optim_conf"]["num_def_loads"] = options.get("number_of_deferrable_loads", params["optim_conf"]["num_def_loads"])
-        if options.get("list_nominal_power_of_deferrable_loads", None) != None:
-            params["optim_conf"]["P_deferrable_nom"] = [i["nominal_power_of_deferrable_loads"] for i in options.get("list_nominal_power_of_deferrable_loads")]
-        if options.get("list_operating_hours_of_each_deferrable_load", None) != None:
-            params["optim_conf"]["def_total_hours"] = [i["operating_hours_of_each_deferrable_load"] for i in options.get("list_operating_hours_of_each_deferrable_load")]
-        if options.get("list_treat_deferrable_load_as_semi_cont", None) != None:
-            params["optim_conf"]["treat_def_as_semi_cont"] = [i["treat_deferrable_load_as_semi_cont"] for i in options.get("list_treat_deferrable_load_as_semi_cont")]
-        if options.get("list_set_deferrable_load_single_constant", None) != None:
-            params["optim_conf"]["set_def_constant"] = [i["set_deferrable_load_single_constant"] for i in options.get("list_set_deferrable_load_single_constant")]
-        if options.get("list_set_deferrable_startup_penalty", None) != None:
-            params["optim_conf"]["def_start_penalty"] = [i["set_deferrable_startup_penalty"] for i in options.get("list_set_deferrable_startup_penalty")]
-        params["optim_conf"]["weather_forecast_method"] = options.get("weather_forecast_method", params["optim_conf"]["weather_forecast_method"])
-        # Update optional param secrets
-        if params["optim_conf"]["weather_forecast_method"] == "solcast":
-            params["params_secrets"]["solcast_api_key"] = options.get("optional_solcast_api_key", params_secrets.get("solcast_api_key", "123456"))
-            params["params_secrets"]["solcast_rooftop_id"] = options.get("optional_solcast_rooftop_id", params_secrets.get("solcast_rooftop_id", "123456"))
-        elif params["optim_conf"]["weather_forecast_method"] == "solar.forecast":
-            params["params_secrets"]["solar_forecast_kwp"] = options.get("optional_solar_forecast_kwp", params_secrets.get("solar_forecast_kwp", 5))
-        params["optim_conf"]["load_forecast_method"] = options.get("load_forecast_method", params["optim_conf"]["load_forecast_method"])
-        params["optim_conf"]["delta_forecast"] = options.get("delta_forecast_daily", params["optim_conf"]["delta_forecast"])
-        params["optim_conf"]["load_cost_forecast_method"] = options.get("load_cost_forecast_method", params["optim_conf"]["load_cost_forecast_method"])
-        if (options.get("list_peak_hours_periods_start_hours", None) != None and options.get("list_peak_hours_periods_end_hours", None) != None):
-            start_hours_list = [i["peak_hours_periods_start_hours"] for i in options["list_peak_hours_periods_start_hours"]]
-            end_hours_list = [i["peak_hours_periods_end_hours"] for i in options["list_peak_hours_periods_end_hours"]]
+    if type(params_secrets) is not dict:
+        params_secrets = {}
+    
+    params = {}
+    #Start with blank config catagories
+    params['retrieve_hass_conf'] = {}
+    params['params_secrets'] = {}
+    params['optim_conf'] = {}
+    params['plant_conf'] = {}
+
+    # Obtain associations to categorize parameters to their corresponding config catagories 
+    if emhass_conf.get('associations_path', get_root(__file__, num_parent=2) / 'data/associations.csv').exists():
+        with emhass_conf['associations_path'].open('r') as data:
+            associations = list(csv.reader(data, delimiter=","))
+    else:
+        logger.error("Unable to obtain the associations file (associations.csv) in: "  + str(emhass_conf['associations_path']))
+        return False
+
+    # Association file key reference
+    # association[0] = config catagories
+    # association[1] = legacy parameter name
+    # association[2] = parameter (config.json/config_defaults.json)
+    # association[3] = parameter list name if exists (not used, from legacy options.json) 
+
+    # Use association list to append parameters from config into params (with corresponding config catagories)
+    for association in associations:
+        # If parameter has list_ name and parameter in config is presented with its list name 
+        # (ie, config parameter is in legacy options.json format)
+        if len(association) == 4 and config.get(association[3],None) is not None:
+            # Extract lists of dictionaries
+            if config[association[3]] and type(config[association[3]][0]) is dict:
+                params[association[0]][association[2]] = [i[association[2]] for i in config[association[3]]]
+            else:
+                params[association[0]][association[2]] = config[association[3]]
+        # Else, directly set value of config parameter to param  
+        elif config.get(association[2],None) is not None:
+            params[association[0]][association[2]] = config[association[2]]
+
+    # Check if we need to create `list_hp_periods` from config (ie. legacy options.json format)
+    if params.get('optim_conf',None) is not None and config.get("list_peak_hours_periods_start_hours", None) is not None and config.get("list_peak_hours_periods_end_hours", None) is not None:
+            start_hours_list = [i["peak_hours_periods_start_hours"] for i in config["list_peak_hours_periods_start_hours"]]
+            end_hours_list = [i["peak_hours_periods_end_hours"] for i in config["list_peak_hours_periods_end_hours"]]
             num_peak_hours = len(start_hours_list)
-            list_hp_periods_list = [{'period_hp_'+str(i+1):[{'start':start_hours_list[i]},{'end':end_hours_list[i]}]} for i in range(num_peak_hours)]
-            params['optim_conf']['list_hp_periods'] = list_hp_periods_list
-        params['optim_conf']['load_cost_hp'] = options.get('load_peak_hours_cost', params['optim_conf']['load_cost_hp'])
-        params['optim_conf']['load_cost_hc'] = options.get('load_offpeak_hours_cost', params['optim_conf']['load_cost_hc'])
-        params['optim_conf']['prod_price_forecast_method'] = options.get('production_price_forecast_method', params['optim_conf']['prod_price_forecast_method'])
-        params['optim_conf']['prod_sell_price'] = options.get('photovoltaic_production_sell_price', params['optim_conf']['prod_sell_price'])
-        params['optim_conf']['set_total_pv_sell'] = options.get('set_total_pv_sell', params['optim_conf']['set_total_pv_sell'])
-        params['optim_conf']['lp_solver'] = options.get('lp_solver', params['optim_conf']['lp_solver'])
-        params['optim_conf']['lp_solver_path'] = options.get('lp_solver_path', params['optim_conf']['lp_solver_path'])
-        params['optim_conf']['set_nocharge_from_grid'] = options.get('set_nocharge_from_grid', params['optim_conf']['set_nocharge_from_grid'])
-        params['optim_conf']['set_nodischarge_to_grid'] = options.get('set_nodischarge_to_grid', params['optim_conf']['set_nodischarge_to_grid'])
-        params['optim_conf']['set_battery_dynamic'] = options.get('set_battery_dynamic', params['optim_conf']['set_battery_dynamic'])
-        params['optim_conf']['battery_dynamic_max'] = options.get('battery_dynamic_max', params['optim_conf']['battery_dynamic_max'])
-        params['optim_conf']['battery_dynamic_min'] = options.get('battery_dynamic_min', params['optim_conf']['battery_dynamic_min'])
-        params['optim_conf']['weight_battery_discharge'] = options.get('weight_battery_discharge', params['optim_conf']['weight_battery_discharge'])
-        params['optim_conf']['weight_battery_charge'] = options.get('weight_battery_charge', params['optim_conf']['weight_battery_charge'])
-        if options.get('list_start_timesteps_of_each_deferrable_load',None) != None: 
-            params['optim_conf']['def_start_timestep'] = [i['start_timesteps_of_each_deferrable_load'] for i in options.get('list_start_timesteps_of_each_deferrable_load')]
-        if options.get('list_end_timesteps_of_each_deferrable_load',None) != None: 
-            params['optim_conf']['def_end_timestep'] = [i['end_timesteps_of_each_deferrable_load'] for i in options.get('list_end_timesteps_of_each_deferrable_load')]
-        # Updating variables in plant_conf
-        params['plant_conf']['P_from_grid_max'] = options.get('maximum_power_from_grid', params['plant_conf']['P_from_grid_max'])
-        params['plant_conf']['P_to_grid_max'] = options.get('maximum_power_to_grid', params['plant_conf']['P_to_grid_max'])
-        if options.get('list_pv_module_model',None) != None:         
-            params['plant_conf']['module_model'] = [i['pv_module_model'] for i in options.get('list_pv_module_model')]
-        if options.get('list_pv_inverter_model',None) != None:        
-            params['plant_conf']['inverter_model'] = [i['pv_inverter_model'] for i in options.get('list_pv_inverter_model')]
-        if options.get('list_surface_tilt',None) != None:        
-            params['plant_conf']['surface_tilt'] = [i['surface_tilt'] for i in options.get('list_surface_tilt')]
-        if options.get('list_surface_azimuth',None) != None:         
-            params['plant_conf']['surface_azimuth'] = [i['surface_azimuth'] for i in options.get('list_surface_azimuth')]
-        if options.get('list_modules_per_string',None) != None:         
-            params['plant_conf']['modules_per_string'] = [i['modules_per_string'] for i in options.get('list_modules_per_string')]
-        if options.get('list_strings_per_inverter',None) != None: 
-            params['plant_conf']['strings_per_inverter'] = [i['strings_per_inverter'] for i in options.get('list_strings_per_inverter')]
-        params["plant_conf"]["inverter_is_hybrid"] = options.get("inverter_is_hybrid", params["plant_conf"]["inverter_is_hybrid"])
-        params["plant_conf"]["compute_curtailment"] = options.get("compute_curtailment", params["plant_conf"]["compute_curtailment"])
-        params['plant_conf']['Pd_max'] = options.get('battery_discharge_power_max', params['plant_conf']['Pd_max']) 
-        params['plant_conf']['Pc_max'] = options.get('battery_charge_power_max', params['plant_conf']['Pc_max'])
-        params['plant_conf']['eta_disch'] = options.get('battery_discharge_efficiency', params['plant_conf']['eta_disch'])
-        params['plant_conf']['eta_ch'] = options.get('battery_charge_efficiency', params['plant_conf']['eta_ch'])
-        params['plant_conf']['Enom'] = options.get('battery_nominal_energy_capacity', params['plant_conf']['Enom'])
-        params['plant_conf']['SOCmin'] = options.get('battery_minimum_state_of_charge', params['plant_conf']['SOCmin']) 
-        params['plant_conf']['SOCmax'] = options.get('battery_maximum_state_of_charge', params['plant_conf']['SOCmax']) 
-        params['plant_conf']['SOCtarget'] = options.get('battery_target_state_of_charge', params['plant_conf']['SOCtarget'])
-        # Check parameter lists have the same amounts as deferrable loads
-        # If not, set defaults it fill in gaps
-        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['def_start_timestep']):
-            logger.warning("def_start_timestep / list_start_timesteps_of_each_deferrable_load does not match number in num_def_loads, adding default values to parameter")
-            for x in range(len(params['optim_conf']['def_start_timestep']), params['optim_conf']['num_def_loads']):
-                params['optim_conf']['def_start_timestep'].append(0)
-        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['def_end_timestep']):
-            logger.warning("def_end_timestep / list_end_timesteps_of_each_deferrable_load does not match number in num_def_loads, adding default values to parameter")
-            for x in range(len(params['optim_conf']['def_end_timestep']), params['optim_conf']['num_def_loads']):
-                params['optim_conf']['def_end_timestep'].append(0)
-        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['set_def_constant']):
-            logger.warning("set_def_constant / list_set_deferrable_load_single_constant does not match number in num_def_loads, adding default values to parameter")
-            for x in range(len(params['optim_conf']['set_def_constant']), params['optim_conf']['num_def_loads']):
-                params['optim_conf']['set_def_constant'].append(False)
-        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['treat_def_as_semi_cont']):
-            logger.warning("treat_def_as_semi_cont / list_treat_deferrable_load_as_semi_cont does not match number in num_def_loads, adding default values to parameter")
-            for x in range(len(params['optim_conf']['treat_def_as_semi_cont']), params['optim_conf']['num_def_loads']):
-                params['optim_conf']['treat_def_as_semi_cont'].append(True)   
-        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['def_start_penalty']):
-            logger.warning("def_start_penalty / list_set_deferrable_startup_penalty does not match number in num_def_loads, adding default values to parameter")
-            for x in range(len(params['optim_conf']['def_start_penalty']), params['optim_conf']['num_def_loads']):
-                params['optim_conf']['def_start_penalty'].append(0.0)            
-        # days_to_retrieve should be no less then 2     
-        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['def_total_hours']):
-            logger.warning("def_total_hours / list_operating_hours_of_each_deferrable_load does not match number in num_def_loads, adding default values to parameter")
-            for x in range(len(params['optim_conf']['def_total_hours']), params['optim_conf']['num_def_loads']):
-                params['optim_conf']['def_total_hours'].append(0)                   
-        if params['optim_conf']['num_def_loads'] is not len(params['optim_conf']['P_deferrable_nom']):
-            logger.warning("P_deferrable_nom / list_nominal_power_of_deferrable_loads does not match number in num_def_loads, adding default values to parameter")
-            for x in range(len(params['optim_conf']['P_deferrable_nom']), params['optim_conf']['num_def_loads']):
-                params['optim_conf']['P_deferrable_nom'].append(0)   
-        # days_to_retrieve should be no less then 2
-        if params["retrieve_hass_conf"]["days_to_retrieve"] < 2:
-            params["retrieve_hass_conf"]["days_to_retrieve"] = 2
+            list_hp_periods_list = {'period_hp_'+str(i+1):[{'start':start_hours_list[i]},{'end':end_hours_list[i]}] for i in range(num_peak_hours)}
+            params['optim_conf']['load_peak_hour_periods'] = list_hp_periods_list
+    else:
+        # Else, check param already contains load_peak_hour_periods from config
+        if params['optim_conf'].get('load_peak_hour_periods',None) is None:
+            logger.warning("Unable to detect or create load_peak_hour_periods parameter")
+
+    # Format load_peak_hour_periods list to dict if necessary
+    if params['optim_conf'].get('load_peak_hour_periods',None) is not None and isinstance(params['optim_conf']['load_peak_hour_periods'], list):
+        params['optim_conf']['load_peak_hour_periods'] = dict((key, d[key]) for d in params['optim_conf']['load_peak_hour_periods'] for key in d)
+
+    # Call function to check parameter lists that require the same length as deferrable loads
+    # If not, set defaults it fill in gaps
+    if params['optim_conf'].get('number_of_deferrable_loads',None) is not None:
+        num_def_loads = params['optim_conf']['number_of_deferrable_loads']
+        params['optim_conf']['start_timesteps_of_each_deferrable_load'] = check_def_loads(num_def_loads,params['optim_conf'],0,'start_timesteps_of_each_deferrable_load',logger)
+        params['optim_conf']['end_timesteps_of_each_deferrable_load'] = check_def_loads(num_def_loads,params['optim_conf'],0,'end_timesteps_of_each_deferrable_load',logger)
+        params['optim_conf']['set_deferrable_load_single_constant'] = check_def_loads(num_def_loads,params['optim_conf'],False,'set_deferrable_load_single_constant',logger)
+        params['optim_conf']['treat_deferrable_load_as_semi_cont'] = check_def_loads(num_def_loads,params['optim_conf'],True,'treat_deferrable_load_as_semi_cont',logger)
+        params['optim_conf']['set_deferrable_startup_penalty'] = check_def_loads(num_def_loads,params['optim_conf'],0.0,'set_deferrable_startup_penalty',logger)
+        params['optim_conf']['operating_hours_of_each_deferrable_load'] = check_def_loads(num_def_loads,params['optim_conf'],0,'operating_hours_of_each_deferrable_load',logger)
+        params['optim_conf']['nominal_power_of_deferrable_loads'] = check_def_loads(num_def_loads,params['optim_conf'],0,'nominal_power_of_deferrable_loads',logger)
+    else:
+        logger.warning("unable to obtain parameter: number_of_deferrable_loads")
+    # historic_days_to_retrieve should be no less then 2
+    if params["retrieve_hass_conf"].get('historic_days_to_retrieve',None) is not None:
+        if params["retrieve_hass_conf"]['historic_days_to_retrieve'] < 2:
+            params["retrieve_hass_conf"]['historic_days_to_retrieve'] = 2
             logger.warning("days_to_retrieve should not be lower then 2, setting days_to_retrieve to 2. Make sure your sensors also have at least 2 days of history")
     else:
-        params["params_secrets"] = params_secrets
-    # The params dict
+        logger.warning("unable to obtain parameter: historic_days_to_retrieve")
+
+    # Configure secrets, set params to correct config categorie
+    # retrieve_hass_conf
+    params['retrieve_hass_conf']["hass_url"] = params_secrets.get("hass_url",None)
+    params['retrieve_hass_conf']["long_lived_token"] = params_secrets.get("long_lived_token",None)
+    params['retrieve_hass_conf']["time_zone"] = params_secrets.get("time_zone",None)
+    params['retrieve_hass_conf']['Latitude'] = params_secrets.get('Latitude',None)
+    params['retrieve_hass_conf']['Longitude'] = params_secrets.get('Longitude',None)
+    params['retrieve_hass_conf']['Altitude'] = params_secrets.get('Altitude',None)
+    # Update optional param secrets
+    if params["optim_conf"].get('weather_forecast_method',None) is not None:
+        if params["optim_conf"]['weather_forecast_method'] == "solcast":
+            params["retrieve_hass_conf"]["solcast_api_key"] = params_secrets.get("solcast_api_key", "123456")
+            params["params_secrets"]["solcast_api_key"] = params_secrets.get("solcast_api_key", "123456")
+            params["retrieve_hass_conf"]["solcast_rooftop_id"] = params_secrets.get("solcast_rooftop_id", "123456")
+            params["params_secrets"]["solcast_rooftop_id"] = params_secrets.get("solcast_rooftop_id", "123456")
+        elif params["optim_conf"]['weather_forecast_method'] == "solar.forecast":
+            params["retrieve_hass_conf"]["solar_forecast_kwp"] = params_secrets.get("solar_forecast_kwp", 5)
+            params["params_secrets"]["solar_forecast_kwp"] = params_secrets.get("solar_forecast_kwp", 5)
+    else:
+        logger.warning("Unable to detect weather_forecast_method parameter")   
+    #  Check if secrets parameters still defaults values
+    secret_params = ["https://myhass.duckdns.org/","thatverylongtokenhere",45.83,6.86,4807.8]
+    if any(x in secret_params for x in params['retrieve_hass_conf'].values()):
+        logger.warning("Some secret parameters values are still matching their defaults")
+    
+
+    # Set empty dict objects for params passed_data
+    # To be latter populated with runtime parameters (treat_runtimeparams)
     params["passed_data"] = {
         "pv_power_forecast": None,
         "load_power_forecast": None,
@@ -920,14 +1212,39 @@ def build_params(params: dict, params_secrets: dict, options: dict, addon: int,
         "prediction_horizon": None,
         "soc_init": None,
         "soc_final": None,
-        "def_total_hours": None,
-        "def_start_timestep": None,
-        "def_end_timestep": None,
+        'operating_hours_of_each_deferrable_load': None,
+        'start_timesteps_of_each_deferrable_load': None,
+        'end_timesteps_of_each_deferrable_load': None,
         "alpha": None,
         "beta": None,
     }
+
     return params
 
+def check_def_loads(num_def_loads: int, parameter: list[dict], default, parameter_name: str, logger):
+    """
+    Check parameter lists with deferrable loads number, if they do not match, enlarge to fit.
+
+    :param num_def_loads: Total number deferrable loads
+    :type num_def_loads: int
+    :param parameter: parameter config dict containing paramater
+    :type: list[dict]
+    :param default: default value for parameter to pad missing
+    :type: obj
+    :param parameter_name: name of parameter
+    :type logger: str
+    :param logger: The logger object
+    :type logger: logging.Logger
+    return: parameter list 
+    :rtype: list[dict]
+
+    """
+    if parameter.get(parameter_name,None) is not None and type(parameter[parameter_name]) is list and num_def_loads > len(parameter[parameter_name]):
+        logger.warning(parameter_name + " does not match number in num_def_loads, adding default values ("+ str(default) + ") to parameter")
+        for x in range(len(parameter[parameter_name]), num_def_loads):
+            parameter[parameter_name].append(default)
+    return parameter[parameter_name]
+
 
 def get_days_list(days_to_retrieve: int) -> pd.date_range:
     """