emhass 0.10.6__py3-none-any.whl → 0.15.5__py3-none-any.whl

emhass/forecast.py

@@ -1,110 +1,120 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-import pathlib
-import os
-import pickle
+import asyncio
+import bz2
 import copy
 import logging
-import json
-from typing import Optional
-import bz2
+import os
+import pickle
 import pickle as cPickle
-import pandas as pd
-import numpy as np
+import re
 from datetime import datetime, timedelta
-from requests import get
-from bs4 import BeautifulSoup
-import pvlib
-from pvlib.pvsystem import PVSystem
+from itertools import zip_longest
+from urllib.parse import quote
+
+import aiofiles
+import aiohttp
+import numpy as np
+import orjson
+import pandas as pd
+from pvlib.irradiance import disc
 from pvlib.location import Location
 from pvlib.modelchain import ModelChain
+from pvlib.pvsystem import PVSystem
+from pvlib.solarposition import get_solarposition
 from pvlib.temperature import TEMPERATURE_MODEL_PARAMETERS
-from pvlib.irradiance import disc
+from sklearn.metrics import mean_squared_error, r2_score
+from sklearn.model_selection import GridSearchCV, TimeSeriesSplit
 
-from emhass.retrieve_hass import RetrieveHass
 from emhass.machine_learning_forecaster import MLForecaster
-from emhass.utils import get_days_list, set_df_index_freq
+from emhass.machine_learning_regressor import MLRegressor
+from emhass.retrieve_hass import RetrieveHass
+from emhass.utils import add_date_features, get_days_list, set_df_index_freq
 
+header_accept = "application/json"
+error_msg_list_not_long_enough = "Passed data from passed list is not long enough"
+error_msg_method_not_valid = "Passed method is not valid"
 
-class Forecast(object):
+
+class Forecast:
     r"""
     Generate weather, load and costs forecasts needed as inputs to the optimization.
-    
+
     In EMHASS we have basically 4 forecasts to deal with:
-    
+
     - PV power production forecast (internally based on the weather forecast and the
       characteristics of your PV plant). This is given in Watts.
-    
+
     - Load power forecast: how much power your house will demand on the next 24h. This
       is given in Watts.
-    
+
     - PV production selling price forecast: at what price are you selling your excess
       PV production on the next 24h. This is given in EUR/kWh.
-    
+
     - Load cost forecast: the price of the energy from the grid on the next 24h. This
       is given in EUR/kWh.
-    
+
     There are methods that are generalized to the 4 forecast needed. For all there
     forecasts it is possible to pass the data either as a passed list of values or by
     reading from a CSV file. With these methods it is then possible to use data from
     external forecast providers.
-    
-    Then there are the methods that are specific to each type of forecast and that 
+
+    Then there are the methods that are specific to each type of forecast and that
     proposed forecast treated and generated internally by this EMHASS forecast class.
-    For the weather forecast a first method (`scrapper`) uses a scrapping to the 
-    ClearOutside webpage which proposes detailed forecasts based on Lat/Lon locations. 
-    This method seems stable but as with any scrape method it will fail if any changes 
-    are made to the webpage API. Another method (`solcast`) is using the SolCast PV 
-    production forecast service. A final method (`solar.forecast`) is using another 
-    external service: Solar.Forecast, for which just the nominal PV peak installed 
-    power should be provided. Search the forecast section on the documentation for examples 
+    For the weather forecast a first method (`open-meteo`) uses a open-meteos API
+    proposing detailed forecasts based on Lat/Lon locations.
+    This method seems stable but as with any scrape method it will fail if any changes
+    are made to the webpage API. Another method (`solcast`) is using the SolCast PV
+    production forecast service. A final method (`solar.forecast`) is using another
+    external service: Solar.Forecast, for which just the nominal PV peak installed
+    power should be provided. Search the forecast section on the documentation for examples
     on how to implement these different methods.
-    
+
     The `get_power_from_weather` method is proposed here to convert from irradiance
     data to electrical power. The PVLib module is used to model the PV plant.
-    
-    The specific methods for the load forecast are a first method (`naive`) that uses 
-    a naive approach, also called persistance. It simply assumes that the forecast for 
-    a future period will be equal to the observed values in a past period. The past 
+
+    The specific methods for the load forecast are a first method (`naive`) that uses
+    a naive approach, also called persistance. It simply assumes that the forecast for
+    a future period will be equal to the observed values in a past period. The past
     period is controlled using parameter `delta_forecast`. A second method (`mlforecaster`)
     uses an internal custom forecasting model using machine learning. There is a section
     in the documentation explaining how to use this method.
-    
+
     .. note:: This custom machine learning model is introduced from v0.4.0. EMHASS \
         proposed this new `mlforecaster` class with `fit`, `predict` and `tune` methods. \
         Only the `predict` method is used here to generate new forecasts, but it is \
         necessary to previously fit a forecaster model and it is a good idea to \
         optimize the model hyperparameters using the `tune` method. See the dedicated \
         section in the documentation for more help.
-    
+
     For the PV production selling price and Load cost forecasts the privileged method
     is a direct read from a user provided list of values. The list should be passed
     as a runtime parameter during the `curl` to the EMHASS API.
-    
-    I reading from a CSV file, it should contain no header and the timestamped data 
+
+    I reading from a CSV file, it should contain no header and the timestamped data
     should have the following format:
-    
     2021-04-29 00:00:00+00:00,287.07
-    
     2021-04-29 00:30:00+00:00,274.27
-    
     2021-04-29 01:00:00+00:00,243.38
-    
     ...
-    
+
     The data columns in these files will correspond to the data in the units expected
     for each forecasting method.
-    
+
     """
 
-    def __init__(self, retrieve_hass_conf: dict, optim_conf: dict, plant_conf: dict, 
-                 params: str, emhass_conf: dict, logger: logging.Logger, 
-                 opt_time_delta: Optional[int] = 24,
-                 get_data_from_file: Optional[bool] = False) -> None:
+    def __init__(
+        self,
+        retrieve_hass_conf: dict,
+        optim_conf: dict,
+        plant_conf: dict,
+        params: str,
+        emhass_conf: dict,
+        logger: logging.Logger,
+        opt_time_delta: int | None = 24,
+        get_data_from_file: bool | None = False,
+    ) -> None:
         """
         Define constructor for the forecast class.
-        
+
         :param retrieve_hass_conf: Dictionary containing the needed configuration
             data from the configuration file, specific to retrieve data from HASS
         :type retrieve_hass_conf: dict
@@ -120,10 +130,10 @@ class Forecast(object):
         :type emhass_conf: dict
         :param logger: The passed logger object
         :type logger: logging object
-        :param opt_time_delta: The time delta in hours used to generate forecasts, 
+        :param opt_time_delta: The time delta in hours used to generate forecasts,
             a value of 24 will generate 24 hours of forecast data, defaults to 24
         :type opt_time_delta: int, optional
-        :param get_data_from_file: Select if data should be retrieved from a 
+        :param get_data_from_file: Select if data should be retrieved from a
             previously saved pickle useful for testing or directly from connection to
             hass database
         :type get_data_from_file: bool, optional
@@ -132,279 +142,466 @@ class Forecast(object):
         self.retrieve_hass_conf = retrieve_hass_conf
         self.optim_conf = optim_conf
         self.plant_conf = plant_conf
-        self.freq = self.retrieve_hass_conf['freq']
-        self.time_zone = self.retrieve_hass_conf['time_zone']
-        self.method_ts_round = self.retrieve_hass_conf['method_ts_round']
-        self.timeStep = self.freq.seconds/3600 # in hours
+        self.freq = self.retrieve_hass_conf["optimization_time_step"]
+        self.time_zone = self.retrieve_hass_conf["time_zone"]
+        self.method_ts_round = self.retrieve_hass_conf["method_ts_round"]
         self.time_delta = pd.to_timedelta(opt_time_delta, "hours")
-        self.var_PV = self.retrieve_hass_conf['var_PV']
-        self.var_load = self.retrieve_hass_conf['var_load']
-        self.var_load_new = self.var_load+'_positive'
-        self.lat = self.retrieve_hass_conf['lat'] 
-        self.lon = self.retrieve_hass_conf['lon']
+        self.var_pv = self.retrieve_hass_conf["sensor_power_photovoltaics"]
+        self.var_pv_forecast = self.retrieve_hass_conf["sensor_power_photovoltaics_forecast"]
+        self.var_load = self.retrieve_hass_conf["sensor_power_load_no_var_loads"]
+        self.var_load_new = self.var_load + "_positive"
+        self.lat = self.retrieve_hass_conf["Latitude"]
+        self.lon = self.retrieve_hass_conf["Longitude"]
         self.emhass_conf = emhass_conf
         self.logger = logger
         self.get_data_from_file = get_data_from_file
-        self.var_load_cost = 'unit_load_cost'
-        self.var_prod_price = 'unit_prod_price'
-        if params is None:
+        self.var_load_cost = "unit_load_cost"
+        self.var_prod_price = "unit_prod_price"
+        if (params is None) or (params == "null"):
+            self.params = {}
+        elif type(params) is dict:
             self.params = params
         else:
-            self.params = json.loads(params)
-        if self.method_ts_round == 'nearest':
-            self.start_forecast = pd.Timestamp(datetime.now(), tz=self.time_zone).replace(microsecond=0)
-        elif self.method_ts_round == 'first':
-            self.start_forecast = pd.Timestamp(datetime.now(), tz=self.time_zone).replace(microsecond=0).floor(freq=self.freq)
-        elif self.method_ts_round == 'last':
-            self.start_forecast = pd.Timestamp(datetime.now(), tz=self.time_zone).replace(microsecond=0).ceil(freq=self.freq)
+            self.params = orjson.loads(params)
+
+        if self.method_ts_round == "nearest":
+            self.start_forecast = pd.Timestamp.now(tz=self.time_zone).replace(microsecond=0)
+        elif self.method_ts_round == "first":
+            self.start_forecast = (
+                pd.Timestamp.now(tz=self.time_zone).replace(microsecond=0).floor(freq=self.freq)
+            )
+        elif self.method_ts_round == "last":
+            self.start_forecast = (
+                pd.Timestamp.now(tz=self.time_zone).replace(microsecond=0).ceil(freq=self.freq)
+            )
         else:
             self.logger.error("Wrong method_ts_round passed parameter")
-        self.end_forecast = (self.start_forecast + self.optim_conf['delta_forecast']).replace(microsecond=0)
-        self.forecast_dates = pd.date_range(start=self.start_forecast, 
-                                            end=self.end_forecast-self.freq, 
-                                            freq=self.freq).round(self.freq, ambiguous='infer', nonexistent='shift_forward')
-        if params is not None:
-            if 'prediction_horizon' in list(self.params['passed_data'].keys()):
-                if self.params['passed_data']['prediction_horizon'] is not None:
-                    self.forecast_dates = self.forecast_dates[0:self.params['passed_data']['prediction_horizon']]
-        
-        
-    def get_weather_forecast(self, method: Optional[str] = 'scrapper',
-                             csv_path: Optional[str] = "data_weather_forecast.csv") -> pd.DataFrame:
+        # check if weather_forecast_cache, if so get 2x the amount of forecast
+        if self.params["passed_data"].get("weather_forecast_cache", False):
+            self.end_forecast = (
+                self.start_forecast + (self.optim_conf["delta_forecast_daily"] * 2)
+            ).replace(microsecond=0)
+        else:
+            self.end_forecast = (
+                self.start_forecast + self.optim_conf["delta_forecast_daily"]
+            ).replace(microsecond=0)
+        self.forecast_dates = (
+            pd.date_range(
+                start=self.start_forecast,
+                end=self.end_forecast - self.freq,
+                freq=self.freq,
+                tz=self.time_zone,
+            )
+            .tz_convert("utc")
+            .round(self.freq, ambiguous="infer", nonexistent="shift_forward")
+            .tz_convert(self.time_zone)
+        )
+        if (
+            params is not None
+            and "prediction_horizon" in list(self.params["passed_data"].keys())
+            and self.params["passed_data"]["prediction_horizon"] is not None
+        ):
+            self.forecast_dates = self.forecast_dates[
+                0 : self.params["passed_data"]["prediction_horizon"]
+            ]
+
+    async def get_cached_open_meteo_forecast_json(
+        self, max_age: int | None = 30, forecast_days: int = 3
+    ) -> dict:
         r"""
-        Get and generate weather forecast data.
-        
-        :param method: The desired method, options are 'scrapper', 'csv', 'list', 'solcast' and \
-            'solar.forecast'. Defaults to 'scrapper'.
-        :type method: str, optional
-        :return: The DataFrame containing the forecasted data
-        :rtype: pd.DataFrame
-        
-        """
-        csv_path  = self.emhass_conf['data_path'] / csv_path
-        w_forecast_cache_path = os.path.abspath(self.emhass_conf['data_path'] / "weather_forecast_data.pkl")
-
-        self.logger.info("Retrieving weather forecast data using method = "+method)
-        self.weather_forecast_method = method # Saving this attribute for later use to identify csv method usage
-        if method == 'scrapper':
-            freq_scrap = pd.to_timedelta(60, "minutes") # The scrapping time step is 60min on clearoutside
-            forecast_dates_scrap = pd.date_range(start=self.start_forecast,
-                                                 end=self.end_forecast-freq_scrap, 
-                                                 freq=freq_scrap).round(freq_scrap, ambiguous='infer', nonexistent='shift_forward')
-            # Using the clearoutside webpage
-            response = get("https://clearoutside.com/forecast/"+str(round(self.lat, 2))+"/"+str(round(self.lon, 2))+"?desktop=true")
-            '''import bz2 # Uncomment to save a serialized data for tests
-            import _pickle as cPickle
-            with bz2.BZ2File("data/test_response_scrapper_get_method.pbz2", "w") as f: 
-                cPickle.dump(response.content, f)'''
-            soup = BeautifulSoup(response.content, 'html.parser')
-            table = soup.find_all(id='day_0')[0]
-            list_names = table.find_all(class_='fc_detail_label')
-            list_tables = table.find_all('ul')[1:]
-            selected_cols = [0, 1, 2, 3, 10, 12, 15] # Selected variables
-            col_names = [list_names[i].get_text() for i in selected_cols]
-            list_tables = [list_tables[i] for i in selected_cols]
-            # Building the raw DF container
-            raw_data = pd.DataFrame(index=range(len(forecast_dates_scrap)), columns=col_names, dtype=float)
-            for count_col, col in enumerate(col_names):
-                list_rows = list_tables[count_col].find_all('li')
-                for count_row, row in enumerate(list_rows):
-                    raw_data.loc[count_row, col] = float(row.get_text())
-            # Treating index
-            raw_data.set_index(forecast_dates_scrap, inplace=True)
-            raw_data = raw_data[~raw_data.index.duplicated(keep='first')]
-            raw_data = raw_data.reindex(self.forecast_dates)
-            raw_data.interpolate(method='linear', axis=0, limit=None, 
-                                 limit_direction='both', inplace=True)
-            # Converting the cloud cover into Global Horizontal Irradiance with a PVLib method
-            ghi_est = self.cloud_cover_to_irradiance(raw_data['Total Clouds (% Sky Obscured)'])
-            data = ghi_est
-            data['temp_air'] = raw_data['Temperature (°C)']
-            data['wind_speed'] = raw_data['Wind Speed/Direction (mph)']*1.60934 # conversion to km/h
-            data['relative_humidity'] = raw_data['Relative Humidity (%)']
-            data['precipitable_water'] = pvlib.atmosphere.gueymard94_pw(
-                data['temp_air'], data['relative_humidity'])
-        elif method == 'solcast': # using Solcast API
-            # Check if weather_forecast_cache is true or if forecast_data file does not exist
-            if self.params["passed_data"]["weather_forecast_cache"] or not os.path.isfile(w_forecast_cache_path):
-                # Check if weather_forecast_cache_only is true, if so produce error for not finding cache file
-                if not self.params["passed_data"]["weather_forecast_cache_only"]:
-                    # Retrieve data from the Solcast API
-                    if 'solcast_api_key' not in self.retrieve_hass_conf:
-                        self.logger.error("The solcast_api_key parameter was not defined")
+        Get weather forecast json from Open-Meteo and cache it for re-use.
+        The response json is cached in the local file system and returned
+        on subsequent calls until it is older than max_age, at which point
+        attempts will be made to replace it with a new version.
+        The cached version will not be overwritten until a new version has
+        been successfully fetched from Open-Meteo.
+        In the event of connectivity issues, the cached version will continue
+        to be returned until such time as a new version can be successfully
+        fetched from Open-Meteo.
+        If you want to force reload, pass max_age value of zero.
+
+        :param max_age: The maximum age of the cached json file, in minutes,
+            before it is discarded and a new version fetched from Open-Meteo.
+            Defaults to 30 minutes.
+        :type max_age: int, optional
+        :param forecast_days: The number of days of forecast data required from Open-Meteo.
+            One additional day is always fetched from Open-Meteo so there is an extra data in the cache.
+            Defaults to 2 days (3 days fetched) to match the prior default.
+        :type forecast_days: int, optional
+        :return: The json containing the Open-Meteo forecast data
+        :rtype: dict
+
+        """
+
+        # Ensure at least 3 weather forecast days (and 1 more than requested)
+        if forecast_days is None:
+            self.logger.debug("Open-Meteo forecast_days is missing so defaulting to 3 days")
+            forecast_days = 3
+        elif forecast_days < 3:
+            self.logger.debug(
+                "Open-Meteo forecast_days is low (%s) so defaulting to 3 days",
+                forecast_days,
+            )
+            forecast_days = 3
+        else:
+            forecast_days = forecast_days + 1
+
+        # The addition of -b.json file name suffix is because the time format
+        # has changed, and it avoids any attempt to use the old format file.
+        json_path = os.path.abspath(
+            self.emhass_conf["data_path"] / "cached-open-meteo-forecast-b.json"
+        )
+        # The cached JSON file is always loaded, if it exists, as it is also a fallback
+        # in case the REST API call to Open-Meteo fails - the cached JSON will continue to
+        # be used until it can successfully fetch a new version from Open-Meteo.
+        data = None
+        use_cache = False
+        if os.path.exists(json_path):
+            delta = datetime.now() - datetime.fromtimestamp(os.path.getmtime(json_path))
+            json_age = int(delta / timedelta(seconds=60))
+            use_cache = json_age < max_age
+            self.logger.info("Loading existing cached Open-Meteo JSON file: %s", json_path)
+            async with aiofiles.open(json_path) as json_file:
+                content = await json_file.read()
+                data = orjson.loads(content)
+            if use_cache:
+                self.logger.info(
+                    "The cached Open-Meteo JSON file is recent (age=%.0fm, max_age=%sm)",
+                    json_age,
+                    max_age,
+                )
+            else:
+                self.logger.info(
+                    "The cached Open-Meteo JSON file is old (age=%.0fm, max_age=%sm)",
+                    json_age,
+                    max_age,
+                )
+
+        if not use_cache:
+            self.logger.info("Fetching a new weather forecast from Open-Meteo")
+            headers = {"User-Agent": "EMHASS", "Accept": header_accept}
+            # Open-Meteo has returned non-existent time over DST transitions,
+            # so we now return unix timestamps and convert to date/times locally
+            # instead.
+            url = (
+                "https://api.open-meteo.com/v1/forecast?"
+                + "latitude="
+                + str(round(self.lat, 2))
+                + "&longitude="
+                + str(round(self.lon, 2))
+                + "&minutely_15="
+                + "temperature_2m,"
+                + "relative_humidity_2m,"
+                + "rain,"
+                + "cloud_cover,"
+                + "wind_speed_10m,"
+                + "shortwave_radiation_instant,"
+                + "diffuse_radiation_instant,"
+                + "direct_normal_irradiance_instant"
+                + "&forecast_days="
+                + str(forecast_days)
+                + "&timezone="
+                + quote(str(self.time_zone), safe="")
+                + "&timeformat=unixtime"
+            )
+            try:
+                self.logger.debug("Fetching data from Open-Meteo using URL: %s", url)
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(url, headers=headers) as response:
+                        self.logger.debug("Returned HTTP status code: %s", response.status)
+                        response.raise_for_status()
+                        """import bz2 # Uncomment to save a serialized data for tests
+                        import _pickle as cPickle
+                        with bz2.BZ2File("data/test_response_openmeteo_get_method.pbz2", "w") as f:
+                            cPickle.dump(response, f)"""
+                        data = await response.json()
+                        self.logger.info(
+                            "Saving response in Open-Meteo JSON cache file: %s",
+                            json_path,
+                        )
+                        async with aiofiles.open(json_path, "w") as json_file:
+                            content = orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()
+                            await json_file.write(content)
+            except aiohttp.ClientError:
+                self.logger.error("Failed to fetch weather forecast from Open-Meteo", exc_info=True)
+                if data is not None:
+                    self.logger.warning("Returning old cached data until next Open-Meteo attempt")
+
+        return data
+
+    async def _get_weather_open_meteo(
+        self, w_forecast_cache_path: str, use_legacy_pvlib: bool
+    ) -> pd.DataFrame:
+        """Helper to retrieve weather data from Open-Meteo or cache."""
+        if not os.path.isfile(w_forecast_cache_path):
+            data_raw = await self.get_cached_open_meteo_forecast_json(
+                self.optim_conf["open_meteo_cache_max_age"],
+                self.optim_conf["delta_forecast_daily"].days,
+            )
+            data_15min = pd.DataFrame.from_dict(data_raw["minutely_15"])
+            # Date/times in the Open-Meteo JSON are unix timestamps
+            data_15min["time"] = pd.to_datetime(data_15min["time"], unit="s", utc=True)
+            data_15min["time"] = data_15min["time"].dt.tz_convert(self.time_zone)
+            data_15min.set_index("time", inplace=True)
+            data_15min = data_15min.rename(
+                columns={
+                    "temperature_2m": "temp_air",
+                    "relative_humidity_2m": "relative_humidity",
+                    "rain": "precipitable_water",
+                    "cloud_cover": "cloud_cover",
+                    "wind_speed_10m": "wind_speed",
+                    "shortwave_radiation_instant": "ghi",
+                    "diffuse_radiation_instant": "dhi",
+                    "direct_normal_irradiance_instant": "dni",
+                }
+            )
+            if self.logger.isEnabledFor(logging.DEBUG):
+                data_15min.to_csv(
+                    self.emhass_conf["data_path"] / "debug-weather-forecast-open-meteo.csv"
+                )
+            data = data_15min.reindex(self.forecast_dates)
+            data.interpolate(
+                method="linear",
+                axis=0,
+                limit=None,
+                limit_direction="both",
+                inplace=True,
+            )
+            data = set_df_index_freq(data)
+            index_utc = data.index.tz_convert("utc")
+            index_tz = index_utc.round(
+                freq=data.index.freq, ambiguous="infer", nonexistent="shift_forward"
+            ).tz_convert(self.time_zone)
+            data.index = index_tz
+            data = set_df_index_freq(data)
+            # Convert mm to cm and clip minimum to 0.1 cm
+            data["precipitable_water"] = (data["precipitable_water"] / 10).clip(lower=0.1)
+            if use_legacy_pvlib:
+                data = data.drop(columns=["ghi", "dhi", "dni"])
+                ghi_est = self.cloud_cover_to_irradiance(data["cloud_cover"])
+                data["ghi"] = ghi_est["ghi"]
+                data["dni"] = ghi_est["dni"]
+                data["dhi"] = ghi_est["dhi"]
+            if self.params["passed_data"].get("weather_forecast_cache", False):
+                data = await self.set_cached_forecast_data(w_forecast_cache_path, data)
+        else:
+            data = await self.get_cached_forecast_data(w_forecast_cache_path)
+        return data
+
+    async def _get_weather_solcast(self, w_forecast_cache_path: str) -> pd.DataFrame:
+        """Helper to retrieve weather data from Solcast or cache."""
+        if os.path.isfile(w_forecast_cache_path):
+            return await self.get_cached_forecast_data(w_forecast_cache_path)
+        if self.params["passed_data"].get("weather_forecast_cache_only", False):
+            self.logger.error("Unable to obtain Solcast cache file.")
+            self.logger.error(
+                "Try running optimization again with 'weather_forecast_cache_only': false"
+            )
+            self.logger.error(
+                "Optionally, obtain new Solcast cache with runtime parameter 'weather_forecast_cache': true."
+            )
+            return False
+        if "solcast_api_key" not in self.retrieve_hass_conf:
+            self.logger.error("The solcast_api_key parameter was not defined")
+            return False
+        if "solcast_rooftop_id" not in self.retrieve_hass_conf:
+            self.logger.error("The solcast_rooftop_id parameter was not defined")
+            return False
+        headers = {
+            "User-Agent": "EMHASS",
+            "Authorization": "Bearer " + self.retrieve_hass_conf["solcast_api_key"],
+            "content-type": header_accept,
+        }
+        days_solcast = int(len(self.forecast_dates) * self.freq.seconds / 3600)
+        roof_ids = re.split(r"[,\s]+", self.retrieve_hass_conf["solcast_rooftop_id"].strip())
+        total_data_list = [0] * len(self.forecast_dates)
+
+        async with aiohttp.ClientSession() as session:
+            for roof_id in roof_ids:
+                url = f"https://api.solcast.com.au/rooftop_sites/{roof_id}/forecasts?hours={days_solcast}"
+                async with session.get(url, headers=headers) as response:
+                    if int(response.status) == 200:
+                        data = await response.json()
+                    elif int(response.status) in [402, 429]:
+                        self.logger.error(
+                            "Solcast error: May have exceeded your subscription limit."
+                        )
                         return False
-                    if 'solcast_rooftop_id' not in self.retrieve_hass_conf:
-                        self.logger.error("The solcast_rooftop_id parameter was not defined")
+                    elif int(response.status) >= 400 or (202 <= int(response.status) <= 299):
+                        self.logger.error(
+                            "Solcast error: Issue with request, check API key and rooftop ID."
+                        )
                         return False
-                    headers = {
-                        'User-Agent': 'EMHASS',
-                        "Authorization": "Bearer " + self.retrieve_hass_conf['solcast_api_key'],
-                        "content-type": "application/json",
-                        }
-                    days_solcast = int(len(self.forecast_dates)*self.freq.seconds/3600)
-                    # If weather_forecast_cache, set request days as twice as long to avoid length issues (add a buffer) 
-                    if self.params["passed_data"]["weather_forecast_cache"]:
-                        days_solcast = min((days_solcast * 2), 336)
-                    url = "https://api.solcast.com.au/rooftop_sites/"+self.retrieve_hass_conf['solcast_rooftop_id']+"/forecasts?hours="+str(days_solcast)
-                    response = get(url, headers=headers)
-                    '''import bz2 # Uncomment to save a serialized data for tests
-                    import _pickle as cPickle
-                    with bz2.BZ2File("data/test_response_solcast_get_method.pbz2", "w") as f: 
-                        cPickle.dump(response, f)'''
-                    # Verify the request passed
-                    if int(response.status_code) == 200:
-                        data = response.json()
-                    elif int(response.status_code) == 402 or int(response.status_code) == 429:
-                        self.logger.error("Solcast error: May have exceeded your subscription limit.")
-                        return False  
-                    elif int(response.status_code) >= 400 or int(response.status_code) >= 202:
-                        self.logger.error("Solcast error: There was a issue with the solcast request, check solcast API key and rooftop ID.")
-                        self.logger.error("Solcast error: Check that your subscription is valid and your network can connect to Solcast.")
-                        return False  
                     data_list = []
-                    for elm in data['forecasts']:
-                        data_list.append(elm['pv_estimate']*1000) # Converting kW to W
-                    # Check if the retrieved data has the correct length
+                    for elm in data["forecasts"]:
+                        data_list.append(elm["pv_estimate"] * 1000)
                     if len(data_list) < len(self.forecast_dates):
-                        self.logger.error("Not enough data retried from Solcast service, try increasing the time step or use MPC.")
-                    else:
-                        # If runtime weather_forecast_cache is true save forecast result to file as cache
-                        if self.params["passed_data"]["weather_forecast_cache"]:
-                            # Add x2 forecast periods for cached results. This adds a extra delta_forecast amount of days for a buffer
-                            cached_forecast_dates =  self.forecast_dates.union(pd.date_range(self.forecast_dates[-1], periods=(len(self.forecast_dates) +1), freq=self.freq)[1:])
-                            cache_data_list = data_list[0:len(cached_forecast_dates)]
-                            cache_data_dict = {'ts':cached_forecast_dates, 'yhat':cache_data_list}
-                            data_cache = pd.DataFrame.from_dict(cache_data_dict)
-                            data_cache.set_index('ts', inplace=True)
-                            with open(w_forecast_cache_path, "wb") as file:    
-                                cPickle.dump(data_cache, file)
-                            if not os.path.isfile(w_forecast_cache_path):
-                                self.logger.warning("Solcast forecast data could not be saved to file.")
-                            else:
-                                self.logger.info("Saved the Solcast results to cache, for later reference.")    
-                        # Trim request results to forecast_dates        
-                        data_list = data_list[0:len(self.forecast_dates)]
-                        data_dict = {'ts':self.forecast_dates, 'yhat':data_list}
-                        # Define DataFrame
-                        data = pd.DataFrame.from_dict(data_dict)
-                        # Define index
-                        data.set_index('ts', inplace=True)
-            # Else, notify user to update cache
-                else:
-                    self.logger.error("Unable to obtain Solcast cache file.")
-                    self.logger.error("Try running optimization again with 'weather_forecast_cache_only': false")
-                    self.logger.error("Optionally, obtain new Solcast cache with runtime parameter 'weather_forecast_cache': true in an optimization, or run the `forecast-cache` action, to pull new data from Solcast and cache.")
-                    return False
-            # Else, open stored weather_forecast_data.pkl file for previous forecast data (cached data)
-            else:
-                with open(w_forecast_cache_path, "rb") as file:
-                    data = cPickle.load(file)
-                    if not isinstance(data, pd.DataFrame) or len(data) < len(self.forecast_dates):
-                        self.logger.error("There has been a error obtaining cached Solcast forecast data.")
-                        self.logger.error("Try running optimization again with 'weather_forecast_cache': true, or run action `forecast-cache`, to pull new data from Solcast and cache.")
-                        self.logger.warning("Removing old Solcast cache file. Next optimization will pull data from Solcast, unless 'weather_forecast_cache_only': true")
-                        os.remove(w_forecast_cache_path)
+                        self.logger.error("Not enough data retrieved from Solcast service.")
                         return False
-                    # Filter cached forecast data to match current forecast_dates start-end range (reduce forecast Dataframe size to appropriate length)
-                    if self.forecast_dates[0] in data.index and self.forecast_dates[-1] in data.index:
-                        data = data.loc[self.forecast_dates[0]:self.forecast_dates[-1]]
-                        self.logger.info("Retrieved Solcast data from the previously saved cache.")
+                    total_data_list = [
+                        total + current
+                        for total, current in zip_longest(total_data_list, data_list, fillvalue=0)
+                    ]
+
+        total_data_list = total_data_list[0 : len(self.forecast_dates)]
+        data_dict = {"ts": self.forecast_dates, "yhat": total_data_list}
+        data = pd.DataFrame.from_dict(data_dict)
+        data.set_index("ts", inplace=True)
+        if self.params["passed_data"].get("weather_forecast_cache", False):
+            data = await self.set_cached_forecast_data(w_forecast_cache_path, data)
+        return data
+
+    async def _get_weather_solar_forecast(self, w_forecast_cache_path: str) -> pd.DataFrame:
+        """Helper to retrieve weather data from solar.forecast or cache."""
+        if os.path.isfile(w_forecast_cache_path):
+            return await self.get_cached_forecast_data(w_forecast_cache_path)
+        # Validation and Default Setup
+        if "solar_forecast_kwp" not in self.retrieve_hass_conf:
+            self.logger.warning(
+                "The solar_forecast_kwp parameter was not defined, using dummy values for testing"
+            )
+            self.retrieve_hass_conf["solar_forecast_kwp"] = 5
+        if self.retrieve_hass_conf["solar_forecast_kwp"] == 0:
+            self.logger.warning(
+                "The solar_forecast_kwp parameter is set to zero, setting to default 5"
+            )
+            self.retrieve_hass_conf["solar_forecast_kwp"] = 5
+        if self.optim_conf["delta_forecast_daily"].days > 1:
+            self.logger.warning(
+                "The free public tier for solar.forecast only provides one day forecasts"
+            )
+        headers = {"Accept": header_accept}
+        data = pd.DataFrame()
+
+        async with aiohttp.ClientSession() as session:
+            for i in range(len(self.plant_conf["pv_module_model"])):
+                url = (
+                    "https://api.forecast.solar/estimate/"
+                    + str(round(self.lat, 2))
+                    + "/"
+                    + str(round(self.lon, 2))
+                    + "/"
+                    + str(self.plant_conf["surface_tilt"][i])
+                    + "/"
+                    + str(self.plant_conf["surface_azimuth"][i] - 180)
+                    + "/"
+                    + str(self.retrieve_hass_conf["solar_forecast_kwp"])
+                )
+                async with session.get(url, headers=headers) as response:
+                    data_raw = await response.json()
+                    data_dict = {
+                        "ts": list(data_raw["result"]["watts"].keys()),
+                        "yhat": list(data_raw["result"]["watts"].values()),
+                    }
+                    data_tmp = pd.DataFrame.from_dict(data_dict)
+                    data_tmp.set_index("ts", inplace=True)
+                    data_tmp.index = pd.to_datetime(data_tmp.index)
+                    data_tmp = data_tmp.tz_localize(
+                        self.forecast_dates.tz,
+                        ambiguous="infer",
+                        nonexistent="shift_forward",
+                    )
+                    data_tmp = data_tmp.reindex(index=self.forecast_dates)
+                    # Gap filling
+                    mask_up = data_tmp.copy(deep=True).fillna(method="ffill").isnull()
+                    mask_down = data_tmp.copy(deep=True).fillna(method="bfill").isnull()
+                    data_tmp.loc[mask_up["yhat"], :] = 0.0
+                    data_tmp.loc[mask_down["yhat"], :] = 0.0
+                    data_tmp.interpolate(inplace=True, limit=1)
+                    data_tmp = data_tmp.fillna(0.0)
+                    if len(data) == 0:
+                        data = copy.deepcopy(data_tmp)
                     else:
-                        self.logger.error("Unable to obtain cached Solcast forecast data within the requested timeframe range.")
-                        self.logger.error("Try running optimization again (not using cache). Optionally, add runtime parameter 'weather_forecast_cache': true to pull new data from Solcast and cache.")
-                        self.logger.warning("Removing old Solcast cache file. Next optimization will pull data from Solcast, unless 'weather_forecast_cache_only': true")
-                        os.remove(w_forecast_cache_path)
-                        return False    
-        elif method == 'solar.forecast': # using the solar.forecast API
-            # Retrieve data from the solar.forecast API
-            if 'solar_forecast_kwp' not in self.retrieve_hass_conf:
-                self.logger.warning("The solar_forecast_kwp parameter was not defined, using dummy values for testing")
-                self.retrieve_hass_conf['solar_forecast_kwp'] = 5
-            if  self.retrieve_hass_conf['solar_forecast_kwp'] == 0:
-                self.logger.warning("The solar_forecast_kwp parameter is set to zero, setting to default 5")
-                self.retrieve_hass_conf['solar_forecast_kwp'] = 5
-            if self.optim_conf['delta_forecast'].days > 1:
-                self.logger.warning("The free public tier for solar.forecast only provides one day forecasts")
-                self.logger.warning("Continuing with just the first day of data, the other days are filled with 0.0.")
-                self.logger.warning("Use the other available methods for delta_forecast > 1")
-            headers = {
-                "Accept": "application/json"
-                }
-            data = pd.DataFrame()
-            for i in range(len(self.plant_conf['module_model'])):
-                url = "https://api.forecast.solar/estimate/"+str(round(self.lat, 2))+"/"+str(round(self.lon, 2))+\
-                    "/"+str(self.plant_conf["surface_tilt"][i])+"/"+str(self.plant_conf["surface_azimuth"][i]-180)+\
-                    "/"+str(self.retrieve_hass_conf["solar_forecast_kwp"])
-                response = get(url, headers=headers)
-                '''import bz2 # Uncomment to save a serialized data for tests
-                import _pickle as cPickle
-                with bz2.BZ2File("data/test_response_solarforecast_get_method.pbz2", "w") as f: 
-                    cPickle.dump(response.json(), f)'''
-                data_raw = response.json()
-                data_dict = {'ts':list(data_raw['result']['watts'].keys()), 'yhat':list(data_raw['result']['watts'].values())}
-                # Form the final DataFrame
-                data_tmp = pd.DataFrame.from_dict(data_dict)
-                data_tmp.set_index('ts', inplace=True)
-                data_tmp.index = pd.to_datetime(data_tmp.index)
-                data_tmp = data_tmp.tz_localize(self.forecast_dates.tz)
-                data_tmp = data_tmp.reindex(index=self.forecast_dates)
-                mask_up_data_df = data_tmp.copy(deep=True).fillna(method = "ffill").isnull()
-                mask_down_data_df = data_tmp.copy(deep=True).fillna(method = "bfill").isnull()
-                data_tmp.loc[data_tmp.index[mask_up_data_df['yhat']==True],:] = 0.0
-                data_tmp.loc[data_tmp.index[mask_down_data_df['yhat']==True],:] = 0.0
-                data_tmp.interpolate(inplace=True, limit=1)
-                data_tmp = data_tmp.fillna(0.0)
-                if len(data) == 0:
-                    data = copy.deepcopy(data_tmp)
-                else:
-                    data = data + data_tmp
-        elif method == 'csv': # reading from a csv file
-            weather_csv_file_path = csv_path
-            # Loading the csv file, we will consider that this is the PV power in W
-            data = pd.read_csv(weather_csv_file_path, header=None, names=['ts', 'yhat'])
-            # Check if the passed data has the correct length       
-            if len(data) < len(self.forecast_dates):
-                self.logger.error("Passed data from CSV is not long enough")
-            else:
-                # Ensure correct length
-                data = data.loc[data.index[0:len(self.forecast_dates)],:]
-                # Define index
-                data.index = self.forecast_dates
-                data.drop('ts', axis=1, inplace=True)
-                data = data.copy().loc[self.forecast_dates]
-        elif method == 'list': # reading a list of values
-            # Loading data from passed list
-            data_list = self.params['passed_data']['pv_power_forecast']
-            # Check if the passed data has the correct length
-            if len(data_list) < len(self.forecast_dates) and self.params['passed_data']['prediction_horizon'] is None:
-                self.logger.error("Passed data from passed list is not long enough")
-            else:
-                # Ensure correct length
-                data_list = data_list[0:len(self.forecast_dates)]
-                # Define DataFrame
-                data_dict = {'ts':self.forecast_dates, 'yhat':data_list}
-                data = pd.DataFrame.from_dict(data_dict)
-                # Define index
-                data.set_index('ts', inplace=True)
+                        data = data + data_tmp
+
+        if self.params["passed_data"].get("weather_forecast_cache", False):
+            data = await self.set_cached_forecast_data(w_forecast_cache_path, data)
+        return data
+
+    def _get_weather_csv(self, csv_path: str) -> pd.DataFrame:
+        """Helper to retrieve weather data from CSV."""
+        data = pd.read_csv(csv_path, header=None, names=["ts", "yhat"])
+        if len(data) < len(self.forecast_dates):
+            self.logger.error("Passed data from CSV is not long enough")
+        else:
+            data = data.loc[data.index[0 : len(self.forecast_dates)], :]
+            data.index = self.forecast_dates
+            data.drop("ts", axis=1, inplace=True)
+            data = data.copy().loc[self.forecast_dates]
+        return data
+
+    def _get_weather_list(self) -> pd.DataFrame:
+        """Helper to retrieve weather data from a passed list."""
+        data_list = self.params["passed_data"]["pv_power_forecast"]
+        if (
+            len(data_list) < len(self.forecast_dates)
+            and self.params["passed_data"]["prediction_horizon"] is None
+        ):
+            self.logger.error(error_msg_list_not_long_enough)
+            return None
+        else:
+            data_list = data_list[0 : len(self.forecast_dates)]
+            data_dict = {"ts": self.forecast_dates, "yhat": data_list}
+            data = pd.DataFrame.from_dict(data_dict)
+            data.set_index("ts", inplace=True)
+            return data
+
+    async def get_weather_forecast(
+        self,
+        method: str | None = "open-meteo",
+        csv_path: str | None = "data_weather_forecast.csv",
+        use_legacy_pvlib: bool | None = False,
+    ) -> pd.DataFrame:
+        r"""
+        Get and generate weather forecast data.
+
+        :param method: The desired method, options are 'open-meteo', 'csv', 'list', 'solcast' and \
+            'solar.forecast'. Defaults to 'open-meteo'.
+        :type method: str, optional
+        :return: The DataFrame containing the forecasted data
+        :rtype: pd.DataFrame
+        """
+        csv_path = self.emhass_conf["data_path"] / csv_path
+        w_forecast_cache_path = os.path.abspath(
+            self.emhass_conf["data_path"] / "weather_forecast_data.pkl"
+        )
+        self.logger.info("Retrieving weather forecast data using method = " + method)
+        if method == "scrapper":
+            self.logger.warning(
+                "The scrapper method has been deprecated and the keyword is accepted just for backward compatibility, please change the PV forecast method to open-meteo"
+            )
+        self.weather_forecast_method = method
+        if method in ["open-meteo", "scrapper"]:
+            data = await self._get_weather_open_meteo(w_forecast_cache_path, use_legacy_pvlib)
+        elif method == "solcast":
+            data = await self._get_weather_solcast(w_forecast_cache_path)
+        elif method == "solar.forecast":
+            data = await self._get_weather_solar_forecast(w_forecast_cache_path)
+        elif method == "csv":
+            data = self._get_weather_csv(csv_path)
+        elif method == "list":
+            data = self._get_weather_list()
         else:
             self.logger.error("Method %r is not valid", method)
             data = None
+        self.logger.debug("get_weather_forecast returning:\n%s", data)
         return data
-    
-    def cloud_cover_to_irradiance(self, cloud_cover: pd.Series, 
-                                  offset:Optional[int] = 35) -> pd.DataFrame:
+
+    def cloud_cover_to_irradiance(
+        self, cloud_cover: pd.Series, offset: int | None = 35
+    ) -> pd.DataFrame:
         """
         Estimates irradiance from cloud cover in the following steps.
-        
+
         1. Determine clear sky GHI using Ineichen model and
            climatological turbidity.
-           
+
         2. Estimate cloudy sky GHI using a function of cloud_cover
-           
+
         3. Estimate cloudy sky DNI using the DISC model.
-        
+
         4. Calculate DHI from DNI and GHI.
-        
+
         (This function was copied and modified from PVLib)
 
         :param cloud_cover: Cloud cover in %.
@@ -416,21 +613,26 @@ class Forecast(object):
         """
         location = Location(latitude=self.lat, longitude=self.lon)
         solpos = location.get_solarposition(cloud_cover.index)
-        cs = location.get_clearsky(cloud_cover.index, model='ineichen', 
-                                   solar_position=solpos)
+        cs = location.get_clearsky(cloud_cover.index, model="ineichen", solar_position=solpos)
         # Using only the linear method
-        offset = offset / 100.
-        cloud_cover_unit = copy.deepcopy(cloud_cover) / 100.
-        ghi = (offset + (1 - offset) * (1 - cloud_cover_unit)) * cs['ghi']
+        offset = offset / 100.0
+        cloud_cover_unit = copy.deepcopy(cloud_cover) / 100.0
+        ghi = (offset + (1 - offset) * (1 - cloud_cover_unit)) * cs["ghi"]
         # Using disc model
-        dni = disc(ghi, solpos['zenith'], cloud_cover.index)['dni']
-        dhi = ghi - dni * np.cos(np.radians(solpos['zenith']))
-        irrads = pd.DataFrame({'ghi': ghi, 'dni': dni, 'dhi': dhi}).fillna(0)
+        dni = disc(ghi, solpos["zenith"], cloud_cover.index)["dni"]
+        dhi = ghi - dni * np.cos(np.radians(solpos["zenith"]))
+        irrads = pd.DataFrame({"ghi": ghi, "dni": dni, "dhi": dhi}).fillna(0)
         return irrads
-    
+
     @staticmethod
-    def get_mix_forecast(df_now: pd.DataFrame, df_forecast: pd.DataFrame, 
-                         alpha:float, beta:float, col:str) -> pd.DataFrame:
+    def get_mix_forecast(
+        df_now: pd.DataFrame,
+        df_forecast: pd.DataFrame,
+        alpha: float,
+        beta: float,
+        col: str,
+        ignore_pv_feedback: bool = False,
+    ) -> pd.DataFrame:
         """A simple correction method for forecasted data using the current real values of a variable.
 
         :param df_now: The DataFrame containing the current/real values
@@ -443,127 +645,521 @@ class Forecast(object):
         :type beta: float
         :param col: The column variable name
         :type col: str
+        :param ignore_pv_feedback: If True, bypass mixing and return original forecast (used during curtailment)
+        :type ignore_pv_feedback: bool
         :return: The output DataFrame with the corrected values
         :rtype: pd.DataFrame
         """
-        first_fcst = alpha*df_forecast.iloc[0] + beta*df_now[col].iloc[-1]
-        df_forecast.iloc[0] = first_fcst
+        # If ignoring PV feedback (e.g., during curtailment), return original forecast
+        if ignore_pv_feedback:
+            return df_forecast
+
+        first_fcst = alpha * df_forecast.iloc[0] + beta * df_now[col].iloc[-1]
+        df_forecast.iloc[0] = int(round(first_fcst))
         return df_forecast
-    
-    def get_power_from_weather(self, df_weather: pd.DataFrame, 
-                               set_mix_forecast:Optional[bool] = False,
-                               df_now:Optional[pd.DataFrame] = pd.DataFrame()) -> pd.Series:
+
+    def _get_model_power(self, params, device_type):
+        """
+        Helper to extract power rating based on device type and available parameters.
+        """
+        if device_type == "module":
+            if "STC" in params:
+                return params["STC"]
+            if "I_mp_ref" in params and "V_mp_ref" in params:
+                return params["I_mp_ref"] * params["V_mp_ref"]
+        elif device_type == "inverter":
+            if "Paco" in params:
+                return params["Paco"]
+            if "Pdco" in params:
+                return params["Pdco"]
+        return None
+
+    def _find_closest_model(self, target_power, database, device_type):
+        """
+        Find the model in the database that has a power rating closest to the target_power.
+        """
+        closest_model = None
+        min_diff = float("inf")
+        # Handle DataFrame (columns are models) or Dict (keys are models)
+        iterator = database.items() if hasattr(database, "items") else database.iteritems()
+        for _, params in iterator:
+            power = self._get_model_power(params, device_type)
+            if power is not None:
+                diff = abs(power - target_power)
+                if diff < min_diff:
+                    min_diff = diff
+                    closest_model = params
+        if closest_model is not None:
+            # Safely get name if it exists (DataFrame Series usually have a .name attribute)
+            model_name = getattr(closest_model, "name", "unknown")
+            self.logger.info(f"Closest {device_type} model to {target_power}W found: {model_name}")
+        else:
+            self.logger.warning(f"No suitable {device_type} model found close to {target_power}W")
+        return closest_model
+
+    def _get_model(self, model_spec, database, device_type):
+        """
+        Retrieve a model from the database by name or by power rating.
+        """
+        # If it's a string, try to find it by name
+        if isinstance(model_spec, str):
+            if model_spec in database:
+                return database[model_spec]
+            # If not found by name, check if it is a number string (e.g., "300")
+            try:
+                target_power = float(model_spec)
+                return self._find_closest_model(target_power, database, device_type)
+            except ValueError:
+                # Not a number, fallback to original behavior (will likely raise KeyError later)
+                self.logger.warning(f"{device_type} model '{model_spec}' not found in database.")
+                return database[model_spec]
+        # If it's a number (int or float), find closest by power
+        elif isinstance(model_spec, int | float):
+            return self._find_closest_model(model_spec, database, device_type)
+        else:
+            self.logger.error(f"Invalid type for {device_type} model: {type(model_spec)}")
+            return None
+
+    def _calculate_pvlib_power(self, df_weather: pd.DataFrame) -> pd.Series:
+        """
+        Helper to simulate PV power generation using PVLib when no direct forecast is available.
+        """
+        # Setting the main parameters of the PV plant
+        location = Location(latitude=self.lat, longitude=self.lon)
+        temp_params = TEMPERATURE_MODEL_PARAMETERS["sapm"]["close_mount_glass_glass"]
+        # Load CEC databases
+        cec_modules_path = self.emhass_conf["root_path"] / "data" / "cec_modules.pbz2"
+        cec_inverters_path = self.emhass_conf["root_path"] / "data" / "cec_inverters.pbz2"
+        with bz2.BZ2File(cec_modules_path, "rb") as f:
+            cec_modules = cPickle.load(f)
+        with bz2.BZ2File(cec_inverters_path, "rb") as f:
+            cec_inverters = cPickle.load(f)
+
+        # Inner helper to run a single simulation configuration
+        def run_single_config(mod_spec, inv_spec, tilt, azimuth, mod_per_str, str_per_inv):
+            module = self._get_model(mod_spec, cec_modules, "module")
+            inverter = self._get_model(inv_spec, cec_inverters, "inverter")
+            system = PVSystem(
+                surface_tilt=tilt,
+                surface_azimuth=azimuth,
+                module_parameters=module,
+                inverter_parameters=inverter,
+                temperature_model_parameters=temp_params,
+                modules_per_string=mod_per_str,
+                strings_per_inverter=str_per_inv,
+            )
+            mc = ModelChain(system, location, aoi_model="physical")
+            mc.run_model(df_weather)
+            return mc.results.ac
+
+        # Handle list (mixed orientation) vs single configuration
+        if isinstance(self.plant_conf["pv_module_model"], list):
+            p_pv_forecast = pd.Series(0, index=df_weather.index)
+            for i in range(len(self.plant_conf["pv_module_model"])):
+                result = run_single_config(
+                    self.plant_conf["pv_module_model"][i],
+                    self.plant_conf["pv_inverter_model"][i],
+                    self.plant_conf["surface_tilt"][i],
+                    self.plant_conf["surface_azimuth"][i],
+                    self.plant_conf["modules_per_string"][i],
+                    self.plant_conf["strings_per_inverter"][i],
+                )
+                p_pv_forecast = p_pv_forecast + result
+        else:
+            p_pv_forecast = run_single_config(
+                self.plant_conf["pv_module_model"],
+                self.plant_conf["pv_inverter_model"],
+                self.plant_conf["surface_tilt"],
+                self.plant_conf["surface_azimuth"],
+                self.plant_conf["modules_per_string"],
+                self.plant_conf["strings_per_inverter"],
+            )
+        return p_pv_forecast
+
+    def get_power_from_weather(
+        self,
+        df_weather: pd.DataFrame,
+        set_mix_forecast: bool | None = False,
+        df_now: pd.DataFrame | None = pd.DataFrame(),
+    ) -> pd.Series:
         r"""
-        Convert wheater forecast data into electrical power.
-        
+        Convert weather forecast data into electrical power.
+
         :param df_weather: The DataFrame containing the weather forecasted data. \
             This DF should be generated by the 'get_weather_forecast' method or at \
             least contain the same columns names filled with proper data.
         :type df_weather: pd.DataFrame
-        :param set_mix_forecast: Use a mixed forcast strategy to integra now/current values.
+        :param set_mix_forecast: Use a mixed forecast strategy to integrate now/current values.
         :type set_mix_forecast: Bool, optional
         :param df_now: The DataFrame containing the now/current data.
         :type df_now: pd.DataFrame
         :return: The DataFrame containing the electrical power in Watts
         :rtype: pd.DataFrame
-
         """
         # If using csv method we consider that yhat is the PV power in W
-        if "solar_forecast_kwp" in self.retrieve_hass_conf.keys() and self.retrieve_hass_conf["solar_forecast_kwp"] == 0:
-            P_PV_forecast = pd.Series(0, index=df_weather.index)
+        if (
+            "solar_forecast_kwp" in self.retrieve_hass_conf.keys()
+            and self.retrieve_hass_conf["solar_forecast_kwp"] == 0
+        ):
+            p_pv_forecast = pd.Series(0, index=df_weather.index)
+        elif self.weather_forecast_method in [
+            "solcast",
+            "solar.forecast",
+            "csv",
+            "list",
+        ]:
+            p_pv_forecast = df_weather["yhat"]
+            p_pv_forecast.name = None
         else:
-            if self.weather_forecast_method == 'solcast' or self.weather_forecast_method == 'solar.forecast' or \
-                self.weather_forecast_method == 'csv' or self.weather_forecast_method == 'list':
-                P_PV_forecast = df_weather['yhat']
-                P_PV_forecast.name = None
-            else: # We will transform the weather data into electrical power
-                # Transform to power (Watts)
-                # Setting the main parameters of the PV plant
-                location = Location(latitude=self.lat, longitude=self.lon)
-                temp_params = TEMPERATURE_MODEL_PARAMETERS['sapm']['close_mount_glass_glass']
-                cec_modules = bz2.BZ2File(self.emhass_conf['root_path'] / 'data' / 'cec_modules.pbz2', "rb")
-                cec_modules = cPickle.load(cec_modules)
-                cec_inverters = bz2.BZ2File(self.emhass_conf['root_path'] / 'data' /  'cec_inverters.pbz2', "rb")
-                cec_inverters = cPickle.load(cec_inverters)
-                if type(self.plant_conf['module_model']) == list:
-                    P_PV_forecast = pd.Series(0, index=df_weather.index)
-                    for i in range(len(self.plant_conf['module_model'])):
-                        # Selecting correct module and inverter
-                        module = cec_modules[self.plant_conf['module_model'][i]]
-                        inverter = cec_inverters[self.plant_conf['inverter_model'][i]]
-                        # Building the PV system in PVLib
-                        system = PVSystem(surface_tilt=self.plant_conf['surface_tilt'][i], 
-                                        surface_azimuth=self.plant_conf['surface_azimuth'][i],
-                                        module_parameters=module,
-                                        inverter_parameters=inverter,
-                                        temperature_model_parameters=temp_params,
-                                        modules_per_string=self.plant_conf['modules_per_string'][i],
-                                        strings_per_inverter=self.plant_conf['strings_per_inverter'][i])
-                        mc = ModelChain(system, location, aoi_model="physical")
-                        # Run the model on the weather DF indexes
-                        mc.run_model(df_weather)
-                        # Extracting results for AC power
-                        P_PV_forecast = P_PV_forecast + mc.results.ac
-                else:
-                    # Selecting correct module and inverter
-                    module = cec_modules[self.plant_conf['module_model']]
-                    inverter = cec_inverters[self.plant_conf['inverter_model']]
-                    # Building the PV system in PVLib
-                    system = PVSystem(surface_tilt=self.plant_conf['surface_tilt'], 
-                                    surface_azimuth=self.plant_conf['surface_azimuth'],
-                                    module_parameters=module,
-                                    inverter_parameters=inverter,
-                                    temperature_model_parameters=temp_params,
-                                    modules_per_string=self.plant_conf['modules_per_string'],
-                                    strings_per_inverter=self.plant_conf['strings_per_inverter'])
-                    mc = ModelChain(system, location, aoi_model="physical")
-                    # Run the model on the weather DF indexes
-                    mc.run_model(df_weather)
-                    # Extracting results for AC power
-                    P_PV_forecast = mc.results.ac
+            # We will transform the weather data into electrical power
+            p_pv_forecast = self._calculate_pvlib_power(df_weather)
         if set_mix_forecast:
-            P_PV_forecast = Forecast.get_mix_forecast(
-                df_now, P_PV_forecast, 
-                self.params['passed_data']['alpha'], self.params['passed_data']['beta'], self.var_PV)
-        return P_PV_forecast
-    
-    def get_forecast_days_csv(self, timedelta_days: Optional[int] = 1) -> pd.date_range:
+            ignore_pv_feedback = self.params["passed_data"].get(
+                "ignore_pv_feedback_during_curtailment", False
+            )
+            p_pv_forecast = Forecast.get_mix_forecast(
+                df_now,
+                p_pv_forecast,
+                self.params["passed_data"]["alpha"],
+                self.params["passed_data"]["beta"],
+                self.var_pv,
+                ignore_pv_feedback,
+            )
+        p_pv_forecast[p_pv_forecast < 0] = 0  # replace any negative PV values with zero
+        self.logger.debug("get_power_from_weather returning:\n%s", p_pv_forecast)
+        return p_pv_forecast
+
+    @staticmethod
+    def compute_solar_angles(df: pd.DataFrame, latitude: float, longitude: float) -> pd.DataFrame:
+        """
+        Compute solar angles (elevation, azimuth) based on timestamps and location.
+
+        :param df: DataFrame with a DateTime index.
+        :param latitude: Latitude of the PV system.
+        :param longitude: Longitude of the PV system.
+        :return: DataFrame with added solar elevation and azimuth.
+        """
+        df = df.copy()
+        solpos = get_solarposition(df.index, latitude, longitude)
+        df["solar_elevation"] = solpos["elevation"]
+        df["solar_azimuth"] = solpos["azimuth"]
+        return df
+
+    def adjust_pv_forecast_data_prep(self, data: pd.DataFrame) -> pd.DataFrame:
+        """
+        Prepare data for adjusting the photovoltaic (PV) forecast.
+
+        This method aligns the actual PV production data with the forecasted data,
+        adds additional features for analysis, and separates the predictors (X)
+        from the target variable (y).
+
+        :param data: A DataFrame containing the actual PV production data and the
+            forecasted PV production data.
+        :type data: pd.DataFrame
+        :return: DataFrame with data for adjusted PV model train.
+        """
+        # Extract target and predictor
+        self.logger.debug("adjust_pv_forecast_data_prep using data:\n%s", data)
+        if self.logger.isEnabledFor(logging.DEBUG):
+            data.to_csv(
+                self.emhass_conf["data_path"] / "debug-adjust-pv-forecast-data-prep-input-data.csv"
+            )
+        P_PV = data[self.var_pv]  # Actual PV production
+        p_pv_forecast = data[self.var_pv_forecast]  # Forecasted PV production
+        # Define time ranges
+        last_day = data.index.max().normalize()  # Last available day
+        three_months_ago = last_day - pd.DateOffset(
+            days=self.retrieve_hass_conf["historic_days_to_retrieve"]
+        )
+        # Train/Test: Last historic_days_to_retrieve days (excluding the last day)
+        train_test_mask = (data.index >= three_months_ago) & (data.index < last_day)
+        self.p_pv_train_test = P_PV[train_test_mask]
+        self.p_pv_forecast_train_test = p_pv_forecast[train_test_mask]
+        # Validation: Last day only
+        validation_mask = data.index >= last_day
+        self.p_pv_validation = P_PV[validation_mask]
+        self.p_pv_forecast_validation = p_pv_forecast[validation_mask]
+        # Ensure data is aligned
+        self.data_adjust_pv = pd.concat(
+            [P_PV.rename("actual"), p_pv_forecast.rename("forecast")], axis=1
+        ).dropna()
+        # Add more features
+        self.data_adjust_pv = add_date_features(self.data_adjust_pv)
+        self.data_adjust_pv = Forecast.compute_solar_angles(self.data_adjust_pv, self.lat, self.lon)
+        # Features (X) and target (y)
+        self.x_adjust_pv = self.data_adjust_pv.drop(columns=["actual"])  # Predictors
+        self.y_adjust_pv = self.data_adjust_pv["actual"]  # Target: actual PV production
+        self.logger.debug("adjust_pv_forecast_data_prep output data:\n%s", self.data_adjust_pv)
+        if self.logger.isEnabledFor(logging.DEBUG):
+            self.data_adjust_pv.to_csv(
+                self.emhass_conf["data_path"] / "debug-adjust-pv-forecast-data-prep-output-data.csv"
+            )
+
+    async def adjust_pv_forecast_fit(
+        self,
+        n_splits: int = 5,
+        regression_model: str = "LassoRegression",
+        debug: bool | None = False,
+    ) -> pd.DataFrame:
+        """
+        Fit a regression model to adjust the photovoltaic (PV) forecast.
+
+        This method uses historical actual and forecasted PV production data, along with
+        additional solar and date features, to train a regression model. The model is
+        optimized using a grid search with time-series cross-validation.
+
+        :param n_splits: The number of splits for time-series cross-validation, defaults to 5.
+        :type n_splits: int, optional
+        :param regression_model: The type of regression model to use. See REGRESSION_METHODS \
+            in machine_learning_regressor.py for the authoritative list of supported models. \
+            Currently: 'LinearRegression', 'RidgeRegression', 'LassoRegression', 'ElasticNet', \
+            'KNeighborsRegressor', 'DecisionTreeRegressor', 'SVR', 'RandomForestRegressor', \
+            'ExtraTreesRegressor', 'GradientBoostingRegressor', 'AdaBoostRegressor', \
+            'MLPRegressor'. Defaults to "LassoRegression".
+        :type regression_model: str, optional
+        :param debug: If True, the model is not saved to disk, useful for debugging, defaults to False.
+        :type debug: bool, optional
+        :return: A DataFrame containing the adjusted PV forecast.
+        :rtype: pd.DataFrame
+        """
+        # Get regression model and hyperparameter grid
+        mlr = MLRegressor(
+            self.data_adjust_pv,
+            "adjusted_pv_forecast",
+            regression_model,
+            list(self.x_adjust_pv.columns),
+            list(self.y_adjust_pv.name),
+            None,
+            self.logger,
+        )
+        pipeline, param_grid = mlr._get_model_and_params()
+        # Time-series split
+        tscv = TimeSeriesSplit(n_splits=n_splits)
+        grid_search = GridSearchCV(
+            pipeline, param_grid, cv=tscv, scoring="neg_mean_squared_error", verbose=0
+        )
+        # Train model
+        await asyncio.to_thread(grid_search.fit, self.x_adjust_pv, self.y_adjust_pv)
+        self.model_adjust_pv = grid_search.best_estimator_
+        # Calculate training metrics
+        y_pred_train = self.model_adjust_pv.predict(self.x_adjust_pv)
+        self.rmse = np.sqrt(mean_squared_error(self.y_adjust_pv, y_pred_train))
+        self.r2 = r2_score(self.y_adjust_pv, y_pred_train)
+        # Log the metrics
+        self.logger.info(f"PV adjust Training metrics: RMSE = {self.rmse}, R2 = {self.r2}")
+        # Save model
+        if not debug:
+            filename = "adjust_pv_regressor.pkl"
+            filename_path = self.emhass_conf["data_path"] / filename
+            async with aiofiles.open(filename_path, "wb") as outp:
+                await outp.write(pickle.dumps(self.model_adjust_pv, pickle.HIGHEST_PROTOCOL))
+
+    def adjust_pv_forecast_predict(self, forecasted_pv: pd.DataFrame | None = None) -> pd.DataFrame:
+        """
+        Predict the adjusted photovoltaic (PV) forecast.
+
+        This method uses the trained regression model to predict the adjusted PV forecast
+        based on either the validation data stored in `self` or a new forecasted PV data
+        passed as input. It applies additional features such as date and solar angles to
+        the forecasted PV production data before making predictions. The solar elevation
+        is used to avoid negative values and to fix values at the beginning and end of the day.
+
+        :param forecasted_pv: Optional. A DataFrame containing the forecasted PV production data.
+                            It must have a DateTime index and a column named "forecast".
+                            If not provided, the method will use `self.p_pv_forecast_validation`.
+        :type forecasted_pv: pd.DataFrame, optional
+        :return: A DataFrame containing the adjusted PV forecast with additional features.
+        :rtype: pd.DataFrame
+        """
+        # Use the provided forecasted PV data or fall back to the validation data in `self`
+        if forecasted_pv is not None:
+            # Ensure the input DataFrame has the required structure
+            if "forecast" not in forecasted_pv.columns:
+                raise ValueError("The input DataFrame must contain a 'forecast' column.")
+            forecast_data = forecasted_pv.copy()
+        else:
+            # Use the validation data stored in `self`
+            forecast_data = self.p_pv_forecast_validation.rename("forecast").to_frame()
+        # Prepare the forecasted PV data
+        forecast_data = add_date_features(forecast_data)
+        forecast_data = Forecast.compute_solar_angles(forecast_data, self.lat, self.lon)
+        # Predict the adjusted forecast
+        forecast_data["adjusted_forecast"] = self.model_adjust_pv.predict(forecast_data)
+
+        # Apply solar elevation weighting only for specific cases
+        def apply_weighting(row):
+            if row["solar_elevation"] <= 0:  # Nighttime or negative solar elevation
+                return 0
+            elif (
+                row["solar_elevation"] < self.optim_conf["adjusted_pv_solar_elevation_threshold"]
+            ):  # Early morning or late evening
+                return max(
+                    row["adjusted_forecast"]
+                    * (
+                        row["solar_elevation"]
+                        / self.optim_conf["adjusted_pv_solar_elevation_threshold"]
+                    ),
+                    0,
+                )
+            else:  # Daytime with sufficient solar elevation
+                return row["adjusted_forecast"]
+
+        forecast_data["adjusted_forecast"] = forecast_data.apply(apply_weighting, axis=1)
+        # If using validation data, calculate validation metrics
+        if forecasted_pv is None:
+            y_true = self.p_pv_validation.values
+            y_pred = forecast_data["adjusted_forecast"].values
+            self.validation_rmse = np.sqrt(mean_squared_error(y_true, y_pred))
+            self.validation_r2 = r2_score(y_true, y_pred)
+            # Log the validation metrics
+            self.logger.info(
+                f"PV adjust Validation metrics: RMSE = {self.validation_rmse}, R2 = {self.validation_r2}"
+            )
+        self.logger.debug("adjust_pv_forecast_predict forecast data:\n%s", forecast_data)
+        if self.logger.isEnabledFor(logging.DEBUG):
+            forecast_data.to_csv(
+                self.emhass_conf["data_path"] / "debug-adjust-pv-forecast-predict-forecast-data.csv"
+            )
+        # Return the DataFrame with the adjusted forecast
+        return forecast_data
+
+    def get_forecast_days_csv(self, timedelta_days: int | None = 1) -> pd.date_range:
         r"""
         Get the date range vector of forecast dates that will be used when loading a CSV file.
-        
+
         :return: The forecast dates vector
         :rtype: pd.date_range
 
         """
         start_forecast_csv = pd.Timestamp(datetime.now(), tz=self.time_zone).replace(microsecond=0)
-        if self.method_ts_round == 'nearest':
-            start_forecast_csv = pd.Timestamp(datetime.now(), tz=self.time_zone).replace(microsecond=0)
-        elif self.method_ts_round == 'first':
-            start_forecast_csv = pd.Timestamp(datetime.now(), tz=self.time_zone).replace(microsecond=0).floor(freq=self.freq)
-        elif self.method_ts_round == 'last':
-            start_forecast_csv = pd.Timestamp(datetime.now(), tz=self.time_zone).replace(microsecond=0).ceil(freq=self.freq)
+        if self.method_ts_round == "nearest":
+            start_forecast_csv = pd.Timestamp.now(tz=self.time_zone).replace(microsecond=0)
+        elif self.method_ts_round == "first":
+            start_forecast_csv = (
+                pd.Timestamp.now(tz=self.time_zone).replace(microsecond=0).floor(freq=self.freq)
+            )
+        elif self.method_ts_round == "last":
+            start_forecast_csv = (
+                pd.Timestamp.now(tz=self.time_zone).replace(microsecond=0).ceil(freq=self.freq)
+            )
         else:
             self.logger.error("Wrong method_ts_round passed parameter")
-        end_forecast_csv = (start_forecast_csv + self.optim_conf['delta_forecast']).replace(microsecond=0)
-        forecast_dates_csv = pd.date_range(start=start_forecast_csv, 
-                                           end=end_forecast_csv+timedelta(days=timedelta_days)-self.freq, 
-                                           freq=self.freq).round(self.freq, ambiguous='infer', nonexistent='shift_forward')
-        if self.params is not None:
-            if 'prediction_horizon' in list(self.params['passed_data'].keys()):
-                if self.params['passed_data']['prediction_horizon'] is not None:
-                    forecast_dates_csv = forecast_dates_csv[0:self.params['passed_data']['prediction_horizon']]
+        end_forecast_csv = (start_forecast_csv + self.optim_conf["delta_forecast_daily"]).replace(
+            microsecond=0
+        )
+        forecast_dates_csv = (
+            pd.date_range(
+                start=start_forecast_csv,
+                end=end_forecast_csv + timedelta(days=timedelta_days) - self.freq,
+                freq=self.freq,
+                tz=self.time_zone,
+            )
+            .tz_convert("utc")
+            .round(self.freq, ambiguous="infer", nonexistent="shift_forward")
+            .tz_convert(self.time_zone)
+        )
+        if (
+            self.params is not None
+            and "prediction_horizon" in list(self.params["passed_data"].keys())
+            and self.params["passed_data"]["prediction_horizon"] is not None
+        ):
+            forecast_dates_csv = forecast_dates_csv[
+                0 : self.params["passed_data"]["prediction_horizon"]
+            ]
         return forecast_dates_csv
-    
-    def get_forecast_out_from_csv_or_list(self, df_final: pd.DataFrame, forecast_dates_csv: pd.date_range,
-                                          csv_path: str, data_list: Optional[list] = None, 
-                                          list_and_perfect: Optional[bool] = False) -> pd.DataFrame:
+
+    def _load_forecast_data(
+        self,
+        csv_path: str,
+        data_list: list | None,
+        forecast_dates_csv: pd.date_range,
+    ) -> pd.DataFrame:
+        """
+        Helper to load and format forecast data from a CSV file or a list.
+        """
+        if csv_path is None:
+            data_dict = {"ts": forecast_dates_csv, "yhat": data_list}
+            df_csv = pd.DataFrame.from_dict(data_dict)
+            df_csv.index = forecast_dates_csv
+            df_csv = df_csv.drop(["ts"], axis=1)
+            df_csv = set_df_index_freq(df_csv)
+        else:
+            if not os.path.exists(csv_path):
+                csv_path = self.emhass_conf["data_path"] / csv_path
+            df_csv = pd.read_csv(csv_path, header=None, names=["ts", "yhat"])
+            # Check if first column is a valid datetime
+            first_col = df_csv.iloc[:, 0]
+            if pd.to_datetime(first_col, errors="coerce").notna().all():
+                df_csv["ts"] = pd.to_datetime(df_csv["ts"], utc=True)
+                df_csv.set_index("ts", inplace=True)
+                df_csv.index = df_csv.index.tz_convert(self.time_zone)
+            else:
+                df_csv.index = forecast_dates_csv
+                df_csv = df_csv.drop(["ts"], axis=1)
+            df_csv = set_df_index_freq(df_csv)
+        return df_csv
+
+    def _extract_daily_forecast(
+        self,
+        day: int,
+        df_timing: pd.DataFrame,
+        df_csv: pd.DataFrame,
+        csv_path: str,
+        list_and_perfect: bool,
+    ) -> pd.DataFrame:
+        """
+        Helper to extract a specific day's forecast data based on timing configuration.
+        """
+        # Find the start and end indices for the specific day in the timing DataFrame
+        day_mask = df_timing.index.day == day
+        day_indices = [i for i, x in enumerate(day_mask) if x]
+        first_elm_index = day_indices[0]
+        last_elm_index = day_indices[-1]
+        # Define the target forecast index based on the timing DataFrame
+        fcst_index = pd.date_range(
+            start=df_timing.index[first_elm_index],
+            end=df_timing.index[last_elm_index],
+            freq=df_timing.index.freq,
+        )
+        first_hour = f"{df_timing.index[first_elm_index].hour:02d}:{df_timing.index[first_elm_index].minute:02d}"
+        last_hour = f"{df_timing.index[last_elm_index].hour:02d}:{df_timing.index[last_elm_index].minute:02d}"
+        # Extract data
+        if csv_path is None:
+            if list_and_perfect:
+                values_array = df_csv.between_time(first_hour, last_hour).values
+                # Adjust index length if necessary
+                fcst_index = fcst_index[0 : len(values_array)]
+                return pd.DataFrame(values_array, index=fcst_index)
+            else:
+                return pd.DataFrame(
+                    df_csv.loc[fcst_index, :].between_time(first_hour, last_hour).values,
+                    index=fcst_index,
+                )
+        else:
+            # For CSV path, filter by date string first
+            df_csv_filtered_date = df_csv.loc[
+                df_csv.index.strftime("%Y-%m-%d") == fcst_index[0].date().strftime("%Y-%m-%d")
+            ]
+            return pd.DataFrame(
+                df_csv_filtered_date.between_time(first_hour, last_hour).values,
+                index=fcst_index,
+            )
+
+    def get_forecast_out_from_csv_or_list(
+        self,
+        df_final: pd.DataFrame,
+        forecast_dates_csv: pd.date_range,
+        csv_path: str,
+        data_list: list | None = None,
+        list_and_perfect: bool | None = False,
+    ) -> pd.DataFrame:
         r"""
-        Get the forecast data as a DataFrame from a CSV file. 
-        
-        The data contained in the CSV file should be a 24h forecast with the same frequency as 
-        the main 'freq' parameter in the configuration file. The timestamp will not be used and 
+        Get the forecast data as a DataFrame from a CSV file.
+
+        The data contained in the CSV file should be a 24h forecast with the same frequency as
+        the main 'optimization_time_step' parameter in the configuration file. The timestamp will not be used and
         a new DateTimeIndex is generated to fit the timestamp index of the input data in 'df_final'.
-        
+
         :param df_final: The DataFrame containing the input data.
         :type df_final: pd.DataFrame
         :param forecast_dates_csv: The forecast dates vector
@@ -574,93 +1170,294 @@ class Forecast(object):
         :rtype: pd.DataFrame
 
         """
-        if csv_path is None:
-            data_dict = {'ts':forecast_dates_csv, 'yhat':data_list}
-            df_csv = pd.DataFrame.from_dict(data_dict)
-            df_csv.index = forecast_dates_csv
-            df_csv.drop(['ts'], axis=1, inplace=True)
-            df_csv = set_df_index_freq(df_csv)
-            if list_and_perfect:
-                days_list = df_final.index.day.unique().tolist()
-            else:
-                days_list = df_csv.index.day.unique().tolist()
-        else:
-            if not os.path.exists(csv_path):
-                csv_path = self.emhass_conf['data_path'] / csv_path
-            load_csv_file_path = csv_path    
-            df_csv = pd.read_csv(load_csv_file_path, header=None, names=['ts', 'yhat'])
-            df_csv.index = forecast_dates_csv
-            df_csv.drop(['ts'], axis=1, inplace=True)
-            df_csv = set_df_index_freq(df_csv)
+        # Load the source data (df_csv)
+        df_csv = self._load_forecast_data(csv_path, data_list, forecast_dates_csv)
+        # Configure timing source (df_timing) and iteration list
+        if csv_path is None or list_and_perfect:
+            df_final = set_df_index_freq(df_final)
+            df_timing = copy.deepcopy(df_final)
             days_list = df_final.index.day.unique().tolist()
-        forecast_out = pd.DataFrame()
+        else:
+            df_timing = copy.deepcopy(df_csv)
+            days_list = df_csv.index.day.unique().tolist()
+        # Iterate over days and collect forecast parts
+        forecast_parts = []
         for day in days_list:
-            if csv_path is None:
-                if list_and_perfect:
-                    df_tmp = copy.deepcopy(df_final)
-                else:
-                    df_tmp = copy.deepcopy(df_csv)
-            else:
-                df_tmp = copy.deepcopy(df_final)
-            first_elm_index = [i for i, x in enumerate(df_tmp.index.day == day) if x][0]
-            last_elm_index = [i for i, x in enumerate(df_tmp.index.day == day) if x][-1]
-            fcst_index = pd.date_range(start=df_tmp.index[first_elm_index],
-                                    end=df_tmp.index[last_elm_index], 
-                                    freq=df_tmp.index.freq)
-            first_hour = str(df_tmp.index[first_elm_index].hour)+":"+str(df_tmp.index[first_elm_index].minute)
-            last_hour = str(df_tmp.index[last_elm_index].hour)+":"+str(df_tmp.index[last_elm_index].minute)
-            if len(forecast_out) == 0:
-                if csv_path is None:
-                    if list_and_perfect:
-                        forecast_out = pd.DataFrame(
-                            df_csv.between_time(first_hour, last_hour).values,
-                            index=fcst_index)
-                    else:
-                        forecast_out = pd.DataFrame(
-                            df_csv.loc[fcst_index,:].between_time(first_hour, last_hour).values,
-                            index=fcst_index)
-                else:
-                    forecast_out = pd.DataFrame(
-                        df_csv.between_time(first_hour, last_hour).values,
-                        index=fcst_index)
-            else:
-                if csv_path is None:
-                    if list_and_perfect:
-                        forecast_tp = pd.DataFrame(
-                            df_csv.between_time(first_hour, last_hour).values,
-                            index=fcst_index)
-                    else:
-                        forecast_tp = pd.DataFrame(
-                            df_csv.loc[fcst_index,:].between_time(first_hour, last_hour).values,
-                            index=fcst_index)
-                else:
-                    forecast_tp = pd.DataFrame(
-                        df_csv.between_time(first_hour, last_hour).values,
-                        index=fcst_index)
-                forecast_out = pd.concat([forecast_out, forecast_tp], axis=0)
+            daily_df = self._extract_daily_forecast(
+                day, df_timing, df_csv, csv_path, list_and_perfect
+            )
+            forecast_parts.append(daily_df)
+        if forecast_parts:
+            forecast_out = pd.concat(forecast_parts, axis=0)
+        else:
+            forecast_out = pd.DataFrame()
+        # Merge with final DataFrame to align indices
+        merged = pd.merge_asof(
+            df_final.sort_index(),
+            forecast_out.sort_index(),
+            left_index=True,
+            right_index=True,
+            direction="nearest",
+        )
+        # Keep only forecast_out columns
+        forecast_out = merged[forecast_out.columns]
         return forecast_out
-    
-    def get_load_forecast(self, days_min_load_forecast: Optional[int] = 3, method: Optional[str] = 'naive',
-                          csv_path: Optional[str] = "data_load_forecast.csv",
-                          set_mix_forecast:Optional[bool] = False, df_now:Optional[pd.DataFrame] = pd.DataFrame(),
-                          use_last_window: Optional[bool] = True, mlf: Optional[MLForecaster] = None,
-                          debug: Optional[bool] = False) -> pd.Series:
+
+    @staticmethod
+    def resample_data(data, freq, current_freq):
+        r"""
+        Resample a DataFrame with a custom frequency.
+
+        :param data: Original time series data with a DateTimeIndex.
+        :type data: pd.DataFrame
+        :param freq: Desired frequency for resampling (e.g., pd.Timedelta("10min")).
+        :type freq: pd.Timedelta
+        :return: Resampled data at the specified frequency.
+        :rtype: pd.DataFrame
+        """
+        if freq > current_freq:
+            # Downsampling
+            # Use 'mean' to aggregate or choose other options ('sum', 'max', etc.)
+            resampled_data = data.resample(freq).mean()
+        elif freq < current_freq:
+            # Upsampling
+            # Use 'asfreq' to create empty slots, then interpolate
+            resampled_data = data.resample(freq).asfreq()
+            resampled_data = resampled_data.interpolate(method="time")
+        else:
+            # No resampling needed
+            resampled_data = data.copy()
+        return resampled_data
+
+    @staticmethod
+    def get_typical_load_forecast(data, forecast_date):
         r"""
+        Forecast the load profile for the next day based on historic data.
+
+        :param data: A DataFrame with a DateTimeIndex containing the historic load data.
+                    Must include a 'load' column.
+        :type data: pd.DataFrame
+        :param forecast_date: The date for which the forecast will be generated.
+        :type forecast_date: pd.Timestamp
+        :return: A Series with the forecasted load profile for the next day and a list of days used
+                to calculate the forecast.
+        :rtype: tuple (pd.Series, list)
+        """
+        # Ensure the 'load' column exists
+        if "load" not in data.columns:
+            raise ValueError("Data must have a 'load' column.")
+        # Filter historic data for the same month and day of the week
+        month = forecast_date.month
+        day_of_week = forecast_date.dayofweek
+        historic_data = data[(data.index.month == month) & (data.index.dayofweek == day_of_week)]
+        used_days = np.unique(historic_data.index.date)
+        # Align all historic data to the forecast day
+        aligned_data = []
+        for day in used_days:
+            daily_data = data[data.index.date == pd.Timestamp(day).date()]
+            aligned_daily_data = daily_data.copy()
+            aligned_daily_data.index = aligned_daily_data.index.map(
+                lambda x: x.replace(
+                    year=forecast_date.year,
+                    month=forecast_date.month,
+                    day=forecast_date.day,
+                )
+            )
+            aligned_data.append(aligned_daily_data)
+        # Combine all aligned historic data into a single DataFrame
+        combined_data = pd.concat(aligned_data)
+        # Compute the mean load for each timestamp
+        forecast = combined_data.groupby(combined_data.index).mean()
+        return forecast, used_days
+
+    async def _prepare_hass_load_data(
+        self, days_min_load_forecast: int, method: str
+    ) -> pd.DataFrame | bool:
+        """Helper to retrieve and prepare load data from Home Assistant."""
+        self.logger.info(f"Retrieving data from hass for load forecast using method = {method}")
+        var_list = [self.var_load]
+        var_replace_zero = None
+        var_interp = [self.var_load]
+        time_zone_load_forecast = None
+        rh = RetrieveHass(
+            self.retrieve_hass_conf["hass_url"],
+            self.retrieve_hass_conf["long_lived_token"],
+            self.freq,
+            time_zone_load_forecast,
+            self.params,
+            self.emhass_conf,
+            self.logger,
+        )
+        if self.get_data_from_file:
+            filename_path = self.emhass_conf["data_path"] / "test_df_final.pkl"
+            async with aiofiles.open(filename_path, "rb") as inp:
+                content = await inp.read()
+                rh.df_final, days_list, var_list, rh.ha_config = pickle.loads(content)
+                self.var_load = var_list[0]
+                self.retrieve_hass_conf["sensor_power_load_no_var_loads"] = self.var_load
+                var_interp = [var_list[0]]
+                self.var_list = [var_list[0]]
+                rh.var_list = self.var_list
+                self.var_load_new = self.var_load + "_positive"
+        else:
+            days_list = get_days_list(days_min_load_forecast)
+            if not await rh.get_data(days_list, var_list):
+                return False
+        if not rh.prepare_data(
+            self.retrieve_hass_conf["sensor_power_load_no_var_loads"],
+            load_negative=self.retrieve_hass_conf["load_negative"],
+            set_zero_min=self.retrieve_hass_conf["set_zero_min"],
+            var_replace_zero=var_replace_zero,
+            var_interp=var_interp,
+        ):
+            return False
+        return rh.df_final.copy()[[self.var_load_new]]
+
+    async def _get_load_forecast_typical(self) -> pd.DataFrame:
+        """Helper to generate typical load forecast."""
+        model_type = "long_train_data"
+        data_path = self.emhass_conf["data_path"] / str(model_type + ".pkl")
+        async with aiofiles.open(data_path, "rb") as fid:
+            content = await fid.read()
+            data, _, _, _ = pickle.loads(content)
+        # Ensure the data index is timezone-aware
+        data.index = (
+            data.index.tz_localize(
+                self.forecast_dates.tz,
+                ambiguous="infer",
+                nonexistent="shift_forward",
+            )
+            if data.index.tz is None
+            else data.index.tz_convert(self.forecast_dates.tz)
+        )
+        data = data[[self.var_load]]
+        current_freq = pd.Timedelta("30min")
+        if self.freq != current_freq:
+            data = Forecast.resample_data(data, self.freq, current_freq)
+        dates_list = np.unique(self.forecast_dates.date).tolist()
+        forecast = pd.DataFrame()
+        for date in dates_list:
+            forecast_date = pd.Timestamp(date)
+            data.columns = ["load"]
+            forecast_tmp, used_days = Forecast.get_typical_load_forecast(data, forecast_date)
+            self.logger.debug(f"Using {len(used_days)} days of data to generate the forecast.")
+            forecast_tmp = forecast_tmp * self.plant_conf["maximum_power_from_grid"] / 9000
+            if len(forecast) == 0:
+                forecast = forecast_tmp
+            else:
+                forecast = pd.concat([forecast, forecast_tmp], axis=0)
+        forecast_out = forecast.loc[forecast.index.intersection(self.forecast_dates)]
+        forecast_out.index = self.forecast_dates
+        forecast_out.index.name = "ts"
+        return forecast_out.rename(columns={"load": "yhat"})
+
+    def _get_load_forecast_naive(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Helper for naive forecast."""
+        forecast_horizon = len(self.forecast_dates)
+        historical_values = df.iloc[-forecast_horizon:]
+        return pd.DataFrame(historical_values.values, index=self.forecast_dates, columns=["yhat"])
+
+    async def _get_load_forecast_ml(
+        self, df: pd.DataFrame, use_last_window: bool, mlf, debug: bool
+    ) -> pd.DataFrame | bool:
+        """Helper for ML forecast."""
+        model_type = self.params["passed_data"]["model_type"]
+        filename = model_type + "_mlf.pkl"
+        filename_path = self.emhass_conf["data_path"] / filename
+        if not debug:
+            if filename_path.is_file():
+                async with aiofiles.open(filename_path, "rb") as inp:
+                    content = await inp.read()
+                    mlf = pickle.loads(content)
+            else:
+                self.logger.error(
+                    "The ML forecaster file was not found, please run a model fit method before this predict method"
+                )
+                return False
+        data_last_window = None
+        if use_last_window:
+            data_last_window = copy.deepcopy(df)
+            data_last_window = data_last_window.rename(columns={self.var_load_new: self.var_load})
+        forecast_out = await mlf.predict(data_last_window)
+        self.logger.debug(
+            "Number of ML predict forcast data generated (lags_opt): "
+            + str(len(forecast_out.index))
+        )
+        self.logger.debug(
+            "Number of forcast dates obtained (prediction_horizon): "
+            + str(len(self.forecast_dates))
+        )
+        if len(self.forecast_dates) < len(forecast_out.index):
+            forecast_out = forecast_out.iloc[0 : len(self.forecast_dates)]
+        elif len(self.forecast_dates) > len(forecast_out.index):
+            self.logger.error(
+                "Unable to obtain: "
+                + str(len(self.forecast_dates))
+                + " lags_opt values from sensor: power load no var loads, check optimization_time_step/freq and historic_days_to_retrieve/days_to_retrieve parameters"
+            )
+            return False
+        data_dict = {
+            "ts": self.forecast_dates,
+            "yhat": forecast_out.values.tolist(),
+        }
+        data = pd.DataFrame.from_dict(data_dict)
+        data.set_index("ts", inplace=True)
+        return data.copy().loc[self.forecast_dates]
+
+    def _get_load_forecast_csv(self, csv_path: str) -> pd.DataFrame:
+        """Helper to retrieve load data from CSV."""
+        df_csv = pd.read_csv(csv_path, header=None, names=["ts", "yhat"])
+        if len(df_csv) < len(self.forecast_dates):
+            self.logger.error("Passed data from CSV is not long enough")
+            return None
+        df_csv = df_csv.loc[df_csv.index[0 : len(self.forecast_dates)], :]
+        df_csv.index = self.forecast_dates
+        df_csv = df_csv.drop(["ts"], axis=1)
+        return df_csv.copy().loc[self.forecast_dates]
+
+    def _get_load_forecast_list(self) -> pd.DataFrame:
+        """Helper to retrieve load data from a passed list."""
+        data_list = self.params["passed_data"]["load_power_forecast"]
+        if (
+            len(data_list) < len(self.forecast_dates)
+            and self.params["passed_data"]["prediction_horizon"] is None
+        ):
+            self.logger.error(error_msg_list_not_long_enough)
+            return False
+        data_list = data_list[0 : len(self.forecast_dates)]
+        data_dict = {"ts": self.forecast_dates, "yhat": data_list}
+        data = pd.DataFrame.from_dict(data_dict)
+        data.set_index("ts", inplace=True)
+        return data.copy().loc[self.forecast_dates]
+
+    async def get_load_forecast(
+        self,
+        days_min_load_forecast: int | None = 3,
+        method: str | None = "typical",
+        csv_path: str | None = "data_load_forecast.csv",
+        set_mix_forecast: bool | None = False,
+        df_now: pd.DataFrame | None = pd.DataFrame(),
+        use_last_window: bool | None = True,
+        mlf: MLForecaster | None = None,
+        debug: bool | None = False,
+    ) -> pd.Series:
+        """
         Get and generate the load forecast data.
-        
+
         :param days_min_load_forecast: The number of last days to retrieve that \
             will be used to generate a naive forecast, defaults to 3
         :type days_min_load_forecast: int, optional
         :param method: The method to be used to generate load forecast, the options \
-            are 'naive' for a persistance model, 'mlforecaster' for using a custom \
+            are 'typical' for a typical household load consumption curve, \
+            are 'naive' for a persistence model, 'mlforecaster' for using a custom \
             previously fitted machine learning model, 'csv' to read the forecast from \
             a CSV file and 'list' to use data directly passed at runtime as a list of \
-            values. Defaults to 'naive'.
+            values. Defaults to 'typical'.
         :type method: str, optional
         :param csv_path: The path to the CSV file used when method = 'csv', \
             defaults to "/data/data_load_forecast.csv"
         :type csv_path: str, optional
-        :param set_mix_forecast: Use a mixed forcast strategy to integra now/current values.
+        :param set_mix_forecast: Use a mixed forecast strategy to integrate now/current values.
         :type set_mix_forecast: Bool, optional
         :param df_now: The DataFrame containing the now/current data.
         :type df_now: pd.DataFrame, optional
@@ -679,123 +1476,60 @@ class Forecast(object):
         :rtype: pd.DataFrame
 
         """
-        csv_path  = self.emhass_conf['data_path'] / csv_path
-        
-        if method == 'naive' or method == 'mlforecaster': # retrieving needed data for these methods
-            self.logger.info("Retrieving data from hass for load forecast using method = "+method)
-            var_list = [self.var_load]
-            var_replace_zero = None
-            var_interp = [self.var_load]
-            time_zone_load_foreacast = None
-            # We will need to retrieve a new set of load data according to the days_min_load_forecast parameter
-            rh = RetrieveHass(self.retrieve_hass_conf['hass_url'], self.retrieve_hass_conf['long_lived_token'], 
-                               self.freq, time_zone_load_foreacast, self.params, self.emhass_conf, self.logger)
-            if self.get_data_from_file:
-                filename_path = self.emhass_conf['data_path'] / 'test_df_final.pkl'
-                with open(filename_path, 'rb') as inp:
-                    rh.df_final, days_list, var_list = pickle.load(inp)
-                    self.var_load = var_list[0]
-                    self.retrieve_hass_conf['var_load'] = self.var_load
-                    var_interp = [var_list[0]]
-                    self.var_list = [var_list[0]]
-                    self.var_load_new = self.var_load+'_positive'
-            else:
-                days_list = get_days_list(days_min_load_forecast) 
-                if not rh.get_data(days_list, var_list):
-                    return False
-            if  not rh.prepare_data(
-                self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'],
-                set_zero_min = self.retrieve_hass_conf['set_zero_min'], 
-                var_replace_zero = var_replace_zero, var_interp = var_interp):
+        csv_path = self.emhass_conf["data_path"] / csv_path
+        # Retrieve Data from Home Assistant if needed
+        df = None
+        if method in ["naive", "mlforecaster"]:
+            df = await self._prepare_hass_load_data(days_min_load_forecast, method)
+            if df is False:
                 return False
-            df = rh.df_final.copy()[[self.var_load_new]]
-        if method == 'naive': # using a naive approach
-            mask_forecast_out = (df.index > days_list[-1] - self.optim_conf['delta_forecast'])
-            forecast_out = df.copy().loc[mask_forecast_out]
-            forecast_out = forecast_out.rename(columns={self.var_load_new: 'yhat'})
-            # Force forecast_out length to avoid mismatches
-            forecast_out = forecast_out.iloc[0:len(self.forecast_dates)]
-            forecast_out.index = self.forecast_dates
-        elif method == 'mlforecaster': # using a custom forecast model with machine learning
-            # Load model
-            model_type = self.params['passed_data']['model_type']
-            filename = model_type+'_mlf.pkl'
-            filename_path = self.emhass_conf['data_path'] / filename
-            if not debug:
-                if filename_path.is_file():
-                    with open(filename_path, 'rb') as inp:
-                        mlf = pickle.load(inp)
-                else:
-                    self.logger.error("The ML forecaster file was not found, please run a model fit method before this predict method")
-                    return False
-            # Make predictions
-            if use_last_window:
-                data_last_window = copy.deepcopy(df)
-                data_last_window = data_last_window.rename(columns={self.var_load_new: self.var_load})
-            else:
-                data_last_window = None
-            forecast_out = mlf.predict(data_last_window)
-            # Force forecast length to avoid mismatches
-            self.logger.debug("Number of ML predict forcast data generated (lags_opt): " + str(len(forecast_out.index)))
-            self.logger.debug("Number of forcast dates obtained: " + str(len(self.forecast_dates)))
-            if len(self.forecast_dates) < len(forecast_out.index):
-                forecast_out = forecast_out.iloc[0:len(self.forecast_dates)]
-            # To be removed once bug is fixed
-            elif len(self.forecast_dates) > len(forecast_out.index):
-                self.logger.error("Unable to obtain: " + str(len(self.forecast_dates))  + " lags_opt values from sensor: power load no var loads, check optimization_time_step/freq and historic_days_to_retrieve/days_to_retrieve parameters")
+        # Generate Forecast based on Method
+        if method == "typical":
+            forecast_out = await self._get_load_forecast_typical()
+        elif method == "naive":
+            forecast_out = self._get_load_forecast_naive(df)
+        elif method == "mlforecaster":
+            forecast_out = await self._get_load_forecast_ml(df, use_last_window, mlf, debug)
+            if forecast_out is False:
                 return False
-            # Define DataFrame
-            data_dict = {'ts':self.forecast_dates, 'yhat':forecast_out.values.tolist()}
-            data = pd.DataFrame.from_dict(data_dict)
-            # Define index
-            data.set_index('ts', inplace=True)
-            forecast_out = data.copy().loc[self.forecast_dates]
-        elif method == 'csv': # reading from a csv file
-            load_csv_file_path = csv_path
-            df_csv = pd.read_csv(load_csv_file_path, header=None, names=['ts', 'yhat'])
-            if len(df_csv) < len(self.forecast_dates):
-                self.logger.error("Passed data from CSV is not long enough")
-            else:
-                # Ensure correct length
-                df_csv = df_csv.loc[df_csv.index[0:len(self.forecast_dates)],:]
-                # Define index
-                df_csv.index = self.forecast_dates
-                df_csv.drop(['ts'], axis=1, inplace=True)
-                forecast_out = df_csv.copy().loc[self.forecast_dates]
-        elif method == 'list': # reading a list of values
-            # Loading data from passed list
-            data_list = self.params['passed_data']['load_power_forecast']
-            # Check if the passed data has the correct length
-            if len(data_list) < len(self.forecast_dates) and self.params['passed_data']['prediction_horizon'] is None:
-                self.logger.error("Passed data from passed list is not long enough")
+        elif method == "csv":
+            forecast_out = self._get_load_forecast_csv(csv_path)
+            if forecast_out is None:
+                return False
+        elif method == "list":
+            forecast_out = self._get_load_forecast_list()
+            if forecast_out is False:
                 return False
-            else:
-                # Ensure correct length
-                data_list = data_list[0:len(self.forecast_dates)]
-                # Define DataFrame
-                data_dict = {'ts':self.forecast_dates, 'yhat':data_list}
-                data = pd.DataFrame.from_dict(data_dict)
-                # Define index
-                data.set_index('ts', inplace=True)
-                forecast_out = data.copy().loc[self.forecast_dates]
         else:
-            self.logger.error("Passed method is not valid")
+            self.logger.error(error_msg_method_not_valid)
             return False
-        P_Load_forecast = copy.deepcopy(forecast_out['yhat'])
+        # Post-processing (Mix Forecast)
+        p_load_forecast = copy.deepcopy(forecast_out["yhat"])
         if set_mix_forecast:
-            P_Load_forecast = Forecast.get_mix_forecast(
-                df_now, P_Load_forecast, 
-                self.params['passed_data']['alpha'], self.params['passed_data']['beta'], self.var_load_new)
-        return P_Load_forecast
-    
-    def get_load_cost_forecast(self, df_final: pd.DataFrame, method: Optional[str] = 'hp_hc_periods',
-                               csv_path: Optional[str] = "data_load_cost_forecast.csv",
-                               list_and_perfect: Optional[bool] = False) -> pd.DataFrame:
+            # Load forecasts don't need curtailment protection - always use feedback
+            p_load_forecast = Forecast.get_mix_forecast(
+                df_now,
+                p_load_forecast,
+                self.params["passed_data"]["alpha"],
+                self.params["passed_data"]["beta"],
+                self.var_load_new,
+                False,  # Never ignore feedback for load forecasts
+            )
+        self.logger.debug("get_load_forecast returning:\n%s", p_load_forecast)
+        return p_load_forecast
+
+    def get_load_cost_forecast(
+        self,
+        df_final: pd.DataFrame,
+        method: str | None = "hp_hc_periods",
+        csv_path: str | None = "data_load_cost_forecast.csv",
+        list_and_perfect: bool | None = False,
+    ) -> pd.DataFrame:
         r"""
         Get the unit cost for the load consumption based on multiple tariff \
         periods. This is the cost of the energy from the utility in a vector \
         sampled at the fixed freq value.
-        
+
         :param df_final: The DataFrame containing the input data.
         :type df_final: pd.DataFrame
         :param method: The method to be used to generate load cost forecast, \
@@ -810,50 +1544,77 @@ class Forecast(object):
         :rtype: pd.DataFrame
 
         """
-        csv_path  = self.emhass_conf['data_path'] / csv_path
-        if method == 'hp_hc_periods':
-            df_final[self.var_load_cost] = self.optim_conf['load_cost_hc']
+        csv_path = self.emhass_conf["data_path"] / csv_path
+        if method == "hp_hc_periods":
+            df_final[self.var_load_cost] = self.optim_conf["load_offpeak_hours_cost"]
             list_df_hp = []
-            for key, period_hp in self.optim_conf['list_hp_periods'].items():
-                list_df_hp.append(df_final[self.var_load_cost].between_time(
-                    period_hp[0]['start'], period_hp[1]['end']))
+            for _key, period_hp in self.optim_conf["load_peak_hour_periods"].items():
+                list_df_hp.append(
+                    df_final[self.var_load_cost].between_time(
+                        period_hp[0]["start"], period_hp[1]["end"]
+                    )
+                )
             for df_hp in list_df_hp:
-                df_final.loc[df_hp.index, self.var_load_cost] = self.optim_conf['load_cost_hp']
-        elif method == 'csv':
+                df_final.loc[df_hp.index, self.var_load_cost] = self.optim_conf[
+                    "load_peak_hours_cost"
+                ]
+        elif method == "csv":
             forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0)
             forecast_out = self.get_forecast_out_from_csv_or_list(
-                df_final, forecast_dates_csv, csv_path)
-            df_final[self.var_load_cost] = forecast_out
-        elif method == 'list': # reading a list of values
+                df_final, forecast_dates_csv, csv_path
+            )
+            # Ensure correct length
+            if not list_and_perfect:
+                forecast_out = forecast_out[0 : len(self.forecast_dates)]
+                df_final = df_final[0 : len(self.forecast_dates)].copy()
+            # Convert to Series if needed and align index
+            if not isinstance(forecast_out, pd.Series):
+                forecast_out = pd.Series(np.ravel(forecast_out), index=df_final.index)
+            df_final.loc[:, self.var_load_cost] = forecast_out
+        elif method == "list":  # reading a list of values
             # Loading data from passed list
-            data_list = self.params['passed_data']['load_cost_forecast']
+            data_list = self.params["passed_data"]["load_cost_forecast"]
             # Check if the passed data has the correct length
-            if len(data_list) < len(self.forecast_dates) and self.params['passed_data']['prediction_horizon'] is None:
-                self.logger.error("Passed data from passed list is not long enough")
+            if (
+                len(data_list) < len(self.forecast_dates)
+                and self.params["passed_data"]["prediction_horizon"] is None
+            ):
+                self.logger.error(error_msg_list_not_long_enough)
                 return False
             else:
                 # Ensure correct length
-                data_list = data_list[0:len(self.forecast_dates)]
+                data_list = data_list[0 : len(self.forecast_dates)]
+                if not list_and_perfect:
+                    df_final = df_final.iloc[0 : len(self.forecast_dates)]
                 # Define the correct dates
                 forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0)
                 forecast_out = self.get_forecast_out_from_csv_or_list(
-                    df_final, forecast_dates_csv, None, data_list=data_list, list_and_perfect=list_and_perfect)
-                # Fill the final DF
+                    df_final,
+                    forecast_dates_csv,
+                    None,
+                    data_list=data_list,
+                    list_and_perfect=list_and_perfect,
+                )
+                df_final = df_final.copy()
                 df_final[self.var_load_cost] = forecast_out
         else:
-            self.logger.error("Passed method is not valid")
+            self.logger.error(error_msg_method_not_valid)
             return False
+        self.logger.debug("get_load_cost_forecast returning:\n%s", df_final)
         return df_final
-    
-    def get_prod_price_forecast(self, df_final: pd.DataFrame, method: Optional[str] = 'constant',
-                                csv_path: Optional[str] = "data_prod_price_forecast.csv", 
-                                list_and_perfect: Optional[bool] = False) -> pd.DataFrame:
 
+    def get_prod_price_forecast(
+        self,
+        df_final: pd.DataFrame,
+        method: str | None = "constant",
+        csv_path: str | None = "data_prod_price_forecast.csv",
+        list_and_perfect: bool | None = False,
+    ) -> pd.DataFrame:
         r"""
         Get the unit power production price for the energy injected to the grid.\
         This is the price of the energy injected to the utility in a vector \
         sampled at the fixed freq value.
-        
+
         :param df_input_data: The DataFrame containing all the input data retrieved
             from hass
         :type df_input_data: pd.DataFrame
@@ -869,31 +1630,130 @@ class Forecast(object):
         :rtype: pd.DataFrame
 
         """
-        csv_path  = self.emhass_conf['data_path'] / csv_path
-        if method == 'constant':
-            df_final[self.var_prod_price] = self.optim_conf['prod_sell_price']
-        elif method == 'csv':
+        csv_path = self.emhass_conf["data_path"] / csv_path
+        if method == "constant":
+            df_final[self.var_prod_price] = self.optim_conf["photovoltaic_production_sell_price"]
+        elif method == "csv":
             forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0)
             forecast_out = self.get_forecast_out_from_csv_or_list(
-                df_final, forecast_dates_csv, csv_path)
-            df_final[self.var_prod_price] = forecast_out
-        elif method == 'list': # reading a list of values
+                df_final, forecast_dates_csv, csv_path
+            )
+            # Ensure correct length
+            if not list_and_perfect:
+                forecast_out = forecast_out[0 : len(self.forecast_dates)]
+                df_final = df_final[0 : len(self.forecast_dates)].copy()
+            # Convert to Series if needed and align index
+            if not isinstance(forecast_out, pd.Series):
+                forecast_out = pd.Series(np.ravel(forecast_out), index=df_final.index)
+            df_final.loc[:, self.var_prod_price] = forecast_out
+        elif method == "list":  # reading a list of values
             # Loading data from passed list
-            data_list = self.params['passed_data']['prod_price_forecast']
+            data_list = self.params["passed_data"]["prod_price_forecast"]
             # Check if the passed data has the correct length
-            if len(data_list) < len(self.forecast_dates) and self.params['passed_data']['prediction_horizon'] is None:
-                self.logger.error("Passed data from passed list is not long enough")
+            if (
+                len(data_list) < len(self.forecast_dates)
+                and self.params["passed_data"]["prediction_horizon"] is None
+            ):
+                self.logger.error(error_msg_list_not_long_enough)
                 return False
             else:
                 # Ensure correct length
-                data_list = data_list[0:len(self.forecast_dates)]
+                data_list = data_list[0 : len(self.forecast_dates)]
+                if not list_and_perfect:
+                    df_final = df_final.iloc[0 : len(self.forecast_dates)]
                 # Define the correct dates
                 forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0)
                 forecast_out = self.get_forecast_out_from_csv_or_list(
-                    df_final, forecast_dates_csv, None, data_list=data_list, list_and_perfect=list_and_perfect)
-                # Fill the final DF
+                    df_final,
+                    forecast_dates_csv,
+                    None,
+                    data_list=data_list,
+                    list_and_perfect=list_and_perfect,
+                )
+                df_final = df_final.copy()
                 df_final[self.var_prod_price] = forecast_out
         else:
-            self.logger.error("Passed method is not valid")
+            self.logger.error(error_msg_method_not_valid)
             return False
-        return df_final
+        self.logger.debug("get_prod_price_forecast returning:\n%s", df_final)
+        return df_final
+
+    async def get_cached_forecast_data(self, w_forecast_cache_path) -> pd.DataFrame:
+        r"""
+        Get cached weather forecast data from file.
+
+        :param w_forecast_cache_path: the path to file.
+        :type method: Any
+        :return: The DataFrame containing the forecasted data
+        :rtype: pd.DataFrame
+
+        """
+        async with aiofiles.open(w_forecast_cache_path, "rb") as file:
+            content = await file.read()
+            data = pickle.loads(content)
+            if not isinstance(data, pd.DataFrame) or len(data) < len(self.forecast_dates):
+                self.logger.error("There has been a error obtaining cached forecast data.")
+                self.logger.error(
+                    "Try running optimization again with 'weather_forecast_cache': true, or run action `weather-forecast-cache`, to pull new data from forecast API and cache."
+                )
+                self.logger.warning(
+                    "Removing old forecast cache file. Next optimization will pull data from forecast API, unless 'weather_forecast_cache_only': true"
+                )
+                os.remove(w_forecast_cache_path)
+                return False
+            # Filter cached forecast data to match current forecast_dates start-end range (reduce forecast Dataframe size to appropriate length)
+            if self.forecast_dates[0] in data.index and self.forecast_dates[-1] in data.index:
+                data = data.loc[self.forecast_dates[0] : self.forecast_dates[-1]]
+                self.logger.info("Retrieved forecast data from the previously saved cache.")
+            else:
+                self.logger.error(
+                    "Unable to obtain cached forecast data within the requested timeframe range."
+                )
+                self.logger.error(
+                    "Try running optimization again (not using cache). Optionally, add runtime parameter 'weather_forecast_cache': true to pull new data from forecast API and cache."
+                )
+                self.logger.warning(
+                    "Removing old forecast cache file. Next optimization will pull data from forecast API, unless 'weather_forecast_cache_only': true"
+                )
+                os.remove(w_forecast_cache_path)
+                return False
+            return data
+
+    async def set_cached_forecast_data(self, w_forecast_cache_path, data) -> pd.DataFrame:
+        r"""
+        Set generated weather forecast data to file.
+        Trim data to match the original requested forecast dates
+
+        :param w_forecast_cache_path: the path to file.
+        :type method: Any
+        :param: The DataFrame containing the forecasted data
+        :type: pd.DataFrame
+        :return: The DataFrame containing the forecasted data
+        :rtype: pd.DataFrame
+
+        """
+        async with aiofiles.open(w_forecast_cache_path, "wb") as file:
+            content = pickle.dumps(data)
+            await file.write(content)
+            if not os.path.isfile(w_forecast_cache_path):
+                self.logger.warning("forecast data could not be saved to file.")
+            else:
+                self.logger.info("Saved the forecast results to cache, for later reference.")
+
+        # Trim cached data to match requested dates
+        end_forecast = (self.start_forecast + self.optim_conf["delta_forecast_daily"]).replace(
+            microsecond=0
+        )
+        forecast_dates = (
+            pd.date_range(
+                start=self.start_forecast,
+                end=end_forecast - self.freq,
+                freq=self.freq,
+                tz=self.time_zone,
+            )
+            .tz_convert("utc")
+            .round(self.freq, ambiguous="infer", nonexistent="shift_forward")
+            .tz_convert(self.time_zone)
+        )
+        data = data.loc[forecast_dates[0] : forecast_dates[-1]]
+        return data