openstef 3.4.56__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

openstef/postprocessing/postprocessing.py

@@ -1,275 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-import logging
-from enum import Enum
-
-import numpy as np
-import pandas as pd
-import structlog
-
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import ForecastType
-from openstef.feature_engineering import weather_features
-from openstef.settings import Settings
-
-# this is the default for "Lagerwey100"
-TURBINE_DATA = {
-    "rated_power": 1,
-    "slope_center": 8.07,
-    "steepness": 0.664,
-}
-
-# Set value to define precission of power, this is needed because comparing to zero sometimes leads to issues for very small values.
-SMALLEST_POWER_UNIT: float = 0.000001
-
-
-def normalize_and_convert_weather_data_for_splitting(
-    weather_data: pd.DataFrame,
-) -> pd.DataFrame:
-    """Normalize and converts weather data for use in energy splitting.
-
-    Args:
-        weather_data: Weather data with "windspeed_100m" and "radiation".
-
-    Returns:
-         Dataframe with "windpower" and "radiation" columns.
-
-    """
-    # Check we have "windspeed_100m" and "radiation" available
-    if not all(
-        elem in weather_data.columns for elem in ["windspeed_100m", "radiation"]
-    ):
-        raise ValueError("weather data does not contain required data!")
-
-    # Prepare output dataframe
-    output_dataframe = pd.DataFrame()
-
-    # Normalize weather data
-    output_dataframe["radiation"] = (
-        weather_data["radiation"]
-        / np.percentile(weather_data["radiation"].dropna(), 99.0)
-        * -1
-    )
-    wind_ref_series = weather_features.calculate_windspeed_at_hubheight(
-        weather_data["windspeed_100m"], fromheight=100
-    )
-    wind_ref = wind_ref_series.to_frame()
-    wind_ref = calculate_wind_power(wind_ref)
-    wind_ref *= -1
-
-    output_dataframe["windpower"] = wind_ref
-    return output_dataframe
-
-
-def calculate_wind_power(
-    windspeed_100m: pd.DataFrame,
-) -> pd.DataFrame:
-    """Calculate the generated wind power based on the wind speed.
-
-    Values are related through the power curve, which is
-    described by turbine_data. Default values are used and are normalized to 1MWp.
-
-    Args:
-        windspeed_100m: Example: ``pd.DataFrame (index = datetime, columns = ["windspeed_100m"])``
-
-    Returns:
-        Example output ``pd.DataFrame(index = datetime, columns = ["windenergy"])``
-
-    """
-    generated_power = TURBINE_DATA["rated_power"] / (
-        1
-        + np.exp(
-            -TURBINE_DATA["steepness"] * (windspeed_100m - TURBINE_DATA["slope_center"])
-        )
-    )
-    return generated_power["windspeed_100m"].rename("windenergy").to_frame()
-
-
-def split_forecast_in_components(
-    forecast: pd.DataFrame, weather_data: pd.DataFrame, split_coefs: dict
-) -> dict[str, pd.DataFrame]:
-    """Make estimates of energy components based on given forecast.
-
-    Args:
-        forecast: KTP load forecast
-        weather_data: Weather data for energy splitting, at least; "windspeed_100m" and "radiation"
-        split_coefs: Previously determined splitting coefs for prediction job
-
-    Returns:
-        Forecast dataframe for each component
-
-    """
-    # Normalize weather data
-    weather_ref_profiles = normalize_and_convert_weather_data_for_splitting(
-        weather_data
-    )
-
-    # Check input
-    if not all(
-        elem in ["windpower", "radiation"]
-        for elem in list(weather_ref_profiles.columns)
-    ):
-        raise ValueError("weather data does not contain required data!")
-
-    # Merge to ensure datetime index is the same
-    weather_ref_profiles = forecast.merge(
-        weather_ref_profiles, how="outer", right_index=True, left_index=True
-    )
-    # Drop rows with duplicate indices
-    weather_ref_profiles = weather_ref_profiles[
-        ~weather_ref_profiles.index.duplicated()
-    ]
-    weather_ref_profiles.replace([np.inf, -np.inf], np.nan).dropna(inplace=True)
-
-    # Prepare output dictionary and list of forecast types
-    components = forecast.copy(deep=True)
-
-    # Calculate profiles of estimated components
-    components["forecast_wind_on_shore"] = (
-        split_coefs["wind_ref"] * weather_ref_profiles["windpower"]
-    )
-    components["forecast_solar"] = (
-        split_coefs["pv_ref"] * weather_ref_profiles["radiation"]
-    )
-    components["forecast_other"] = (
-        weather_ref_profiles["forecast"]
-        - components["forecast_solar"]
-        - components["forecast_wind_on_shore"]
-    )
-
-    # Check that sign of production components is negative and not positive, change if sign is wrong
-    if components["forecast_wind_on_shore"].sum() > 0:
-        raise ValueError("Sign of estimated wind_on_shore component is positive!")
-    if components["forecast_solar"].sum() > 0:
-        raise ValueError("Sign of estimated solar component is positive!")
-
-    # Post process predictions to ensure realistic values
-    components["forecast_solar"] = post_process_wind_solar(
-        components["forecast_solar"], ForecastType.SOLAR
-    )
-    components["forecast_wind_on_shore"] = post_process_wind_solar(
-        components["forecast_wind_on_shore"], ForecastType.WIND
-    )
-
-    return components.drop("forecast", axis=1).drop("stdev", axis=1).dropna()
-
-
-def post_process_wind_solar(
-    forecast: pd.Series, forecast_type: ForecastType
-) -> pd.DataFrame:
-    """Function that caries out postprocessing for wind and solar power generators.
-
-        As these points will always produce energy, predicted energy consumption is
-        set to zero. This function enforces the assumption that production is negative
-        and consuption positive.
-
-    Args:
-        forecast: Series with forecast data.
-        forecast_type: Specifies the type of forecast. This can be retrieved
-            from the prediction job as pj['forecast_type']
-
-    Returns:
-        Post-processed forecast.
-
-    """
-    if forecast_type not in [ForecastType.WIND, ForecastType.SOLAR]:
-        return forecast
-
-    # For wind and solar forecasted value should always be negative.
-    forecast.loc[forecast > (-1 * SMALLEST_POWER_UNIT)] = 0
-
-    # write changed back to forecast
-    return forecast
-
-
-def add_components_base_case_forecast(basecase_forecast: pd.DataFrame) -> pd.DataFrame:
-    """Makes a basecase forecast for the forecast_other component.
-
-    This will make a simple basecase components forecast
-    available and ensures that the sum of the components (other, wind and solar) is equal to the normal basecase
-    forecast This is important for sending GLMD messages correctly to TenneT!
-
-    Args:
-        basecase_forecast: pd.DataFrame with basecase forecast
-
-    Returns:
-        basecase_forecast: pd.DataFrame with extra "forecast_other component"
-
-    """
-    basecase_forecast["forecast_other"] = basecase_forecast["forecast"]
-    return basecase_forecast
-
-
-def add_prediction_job_properties_to_forecast(
-    pj: PredictionJobDataClass,
-    forecast: pd.DataFrame,
-    algorithm_type: str,
-    forecast_type: Enum = None,
-    forecast_quality: str = None,
-) -> pd.DataFrame:
-    """Adds prediciton job meta data to a forecast dataframe.
-
-    Args:
-        pj: Prediciton job.
-        forecast: Forecast dataframe
-        algorithm_type: Type of algirithm used for making the forecast.
-        forecast_type: Type of the forecast. Defaults to None.
-        forecast_quality: Quality of the forecast. Defaults to None.
-
-    Returns:
-        Dataframe with added metadata.
-
-    """
-    structlog.configure(
-        wrapper_class=structlog.make_filtering_bound_logger(
-            logging.getLevelName(Settings.log_level)
-        )
-    )
-    logger = structlog.get_logger(__name__)
-
-    logger.info("Postproces in preparation of storing")
-    if forecast_type is None:
-        forecast_type = pj["forecast_type"]
-    else:
-        # get the value from the enum
-        forecast_type = forecast_type.value
-
-    # NOTE this field is only used when making the babasecase forecast and fallback
-    if forecast_quality is not None:
-        forecast["quality"] = forecast_quality
-
-    # TODO rename prediction job typ to type
-    # TODO algtype = model_file_path, perhaps we can find a more logical name
-    # TODO perhaps better to make a forecast its own class!
-    # TODO double check and sync this with make_basecase_forecast (other fields are added)
-    # !!!!! TODO fix the requirement for customer
-    forecast["pid"] = pj["id"]
-    forecast["customer"] = pj["name"]
-    forecast["description"] = pj["description"]
-    forecast["type"] = forecast_type
-    forecast["algtype"] = algorithm_type
-
-    return forecast
-
-
-def sort_quantiles(
-    forecast: pd.DataFrame, quantile_col_start="quantile_P"
-) -> pd.DataFrame:
-    """Sort quantile values so quantiles do not cross.
-
-    This function assumes that all quantile columns start with 'quantile_P' For more academic details on why this is
-    mathematically sounds, please refer to Quantile and Probability Curves Without Crossing (Chernozhukov, 2010)
-
-    """
-    p_columns = [col for col in forecast.columns if col.startswith(quantile_col_start)]
-
-    if len(p_columns) == 0:
-        return forecast
-
-    # sort the columns
-    p_columns = np.sort(p_columns)
-
-    forecast.loc[:, p_columns] = forecast[p_columns].apply(sorted, axis=1).to_list()
-
-    return forecast

openstef/preprocessing/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0

openstef/preprocessing/preprocessing.py

@@ -1,42 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-
-import numpy as np
-import pandas as pd
-
-
-def replace_repeated_values_with_nan(
-    df: pd.DataFrame, threshold: int, column_name: str
-) -> pd.DataFrame:
-    """Replace sequentially repeated values with NaN.
-
-    Args:
-        df: Data with potential repeating values.
-        threshold: The minimum number of squentially repeated values needed to trigger the replacement with NaN.
-        column_name: Column name of input dataframe with repeating values.
-
-    Returns:
-        DataFrame, similar to df, with the desired values set to NaN.
-
-    """
-    data = df.copy()
-
-    # Add a boolean column to mark sequential duplicates
-    data["temp_is_duplicate"] = data[column_name].eq(data[column_name].shift(1))
-
-    # Create an unique identifier for each sequence with the same value, so we can easily remove the correct sequences
-    data["temp_repeated_group"] = (~data["temp_is_duplicate"]).cumsum()
-
-    # Create mask of sequences larger than or equal to the threshold value
-    mask = (
-        data.groupby("temp_repeated_group")[column_name].transform("count") >= threshold
-    )
-
-    # Replace the masked values with NaN
-    data.loc[mask, column_name] = np.nan
-
-    # Drop temporary columns
-    data = data.drop(["temp_is_duplicate", "temp_repeated_group"], axis=1)
-
-    return data

openstef/settings.py

@@ -1,15 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-
-from functools import lru_cache
-
-from openstef.app_settings import AppSettings
-
-
-@lru_cache
-def _get_app_settings() -> AppSettings:
-    return AppSettings()
-
-
-Settings = _get_app_settings()

openstef/tasks/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0

openstef/tasks/calculate_kpi.py

@@ -1,324 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-
-"""This module contains the CRON job that is periodically executed to calculate key performance indicators (KPIs).
-
-This code assumes prognoses are available from the persistent storage.
-If these are not available run create_forecast.py to train all models.
-
-The folowing tasks are caried out:
-  1: Calculate the KPI for a given pid. Ignore SplitEnergy
-  2: Create figures
-  3: Write KPI to database
-
-Example:
-    This module is meant to be called directly from a CRON job.
-    Alternatively this code can be run directly by running::
-        $ python calculate_kpi.py
-
-"""
-import logging
-
-# Import builtins
-from datetime import datetime, timedelta
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-import structlog
-
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import ModelType
-from openstef.exceptions import NoPredictedLoadError, NoRealisedLoadError
-from openstef.metrics import metrics
-from openstef.settings import Settings
-from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
-from openstef.tasks.utils.taskcontext import TaskContext
-from openstef.validation import validation
-
-# Thresholds for retraining and optimizing
-THRESHOLD_RETRAINING = 0.25
-THRESHOLD_OPTIMIZING = 0.50
-
-
-def main(model_type: ModelType = None, config=None, database=None) -> None:
-    taskname = Path(__file__).name.replace(".py", "")
-
-    if database is None or config is None:
-        raise RuntimeError(
-            "Please specifiy a config object and/or database connection object. These"
-            " can be found in the openstef-dbc package."
-        )
-
-    if model_type is None:
-        model_type = [ml.value for ml in ModelType]
-
-    with TaskContext(taskname, config, database) as context:
-        # Set start and end time
-        start_time = datetime.utcnow() - timedelta(days=1)
-        end_time = datetime.utcnow()
-
-        PredictionJobLoop(context, model_type=model_type).map(
-            check_kpi_task,
-            context,
-            start_time=start_time,
-            end_time=end_time,
-        )
-
-
-def check_kpi_task(
-    pj: PredictionJobDataClass,
-    context: TaskContext,
-    start_time: datetime,
-    end_time: datetime,
-    threshold_optimizing=THRESHOLD_OPTIMIZING,
-    threshold_retraining=THRESHOLD_RETRAINING,
-) -> None:
-    # Apply default parameters if none are provided
-    if start_time is None:
-        start_time = datetime.utcnow() - timedelta(days=1)
-    if end_time is None:
-        end_time = datetime.utcnow()
-
-    # Get realised load data
-    realised = context.database.get_load_pid(pj["id"], start_time, end_time, "15T")
-
-    # Get predicted load
-    predicted_load = context.database.get_predicted_load_tahead(
-        pj, start_time, end_time
-    )
-
-    # Get basecase prediction
-    load_1_week_before = context.database.get_load_pid(
-        pj["id"], start_time - timedelta(days=7), end_time - timedelta(days=7), "15T"
-    )
-    if len(load_1_week_before) > 0:
-        basecase = load_1_week_before.shift(periods=7, freq="d")
-    else:
-        basecase = pd.DataFrame()
-
-    kpis = calc_kpi_for_specific_pid(pj["id"], realised, predicted_load, basecase)
-    # Write KPI's to database
-    context.database.write_kpi(pj, kpis)
-
-    # Add pid to the list of pids that should be retrained or optimized if
-    # performance is insufficient
-    if kpis["47.0h"]["rMAE"] > threshold_retraining:
-        context.logger.warning(
-            "Need to retrain model, retraining threshold rMAE 47h exceeded",
-            t_ahead="47.0h",
-            rMAE=kpis["47.0h"]["rMAE"],
-            retraining_threshold=threshold_retraining,
-        )
-
-    if kpis["47.0h"]["rMAE"] > threshold_optimizing:
-        context.logger.warning(
-            "Need to optimize hyperparameters, optimizing threshold rMAE 47h exceeded",
-            t_ahead="47.0h",
-            rMAE=kpis["47.0h"]["rMAE"],
-            optimizing_threshold=threshold_optimizing,
-        )
-
-
-def calc_kpi_for_specific_pid(
-    pid: int,
-    realised: pd.DataFrame,
-    predicted_load: pd.DataFrame,
-    basecase: pd.DataFrame,
-) -> dict:
-    """Function that checks the model performance based on a pid. This function.
-
-    - loads and combines forecast and realised data
-    - calculated several key performance indicators (KPIs)
-    These metric include:
-        - RMSE,
-        - bias,
-        - NSME (model efficiency, between -inf and 1)
-        - Mean absolute Error
-
-    Args:
-        pid: Prediction ID for a given prediction job
-        realised: Realised load.
-        predicted_load: Predicted load.
-        basecase: Basecase predicted load.
-
-    Returns:
-        - Dictionary that includes a dictonary for each t_ahead.
-        - Dict includes enddate en window (in days) for clarification
-
-    Raises:
-        NoPredictedLoadError: When no predicted load for given datatime range.
-        NoRealisedLoadError: When no realised load for given datetime range.
-
-    Example:
-        To get the rMAE for the 24 hours ahead prediction: kpis['24h']['rMAE']
-
-    """
-    COMPLETENESS_REALISED_THRESHOLDS = 0.7
-    COMPLETENESS_PREDICTED_LOAD_THRESHOLD = 0.7
-
-    structlog.configure(
-        wrapper_class=structlog.make_filtering_bound_logger(
-            logging.getLevelName(Settings.log_level)
-        )
-    )
-    logger = structlog.get_logger(__name__)
-
-    # If predicted is empty
-    if len(predicted_load) == 0:
-        raise NoPredictedLoadError(pid)
-
-    # If realised is empty
-    if len(realised) == 0:
-        raise NoRealisedLoadError(pid)
-
-    # Define start and end time
-    start_time = realised.index.min().to_pydatetime()
-    end_time = realised.index.max().to_pydatetime()
-
-    completeness_realised = validation.calc_completeness_dataframe(realised)[0]
-
-    # Interpolate missing data if needed
-    realised = realised.resample("15T").interpolate(limit=3)
-
-    completeness_predicted_load = validation.calc_completeness_dataframe(predicted_load)
-
-    # Combine the forecast and the realised to make sure indices are matched nicely
-    combined = pd.merge(realised, predicted_load, left_index=True, right_index=True)
-
-    # Add basecase (load in same time period 7 days ago)
-    # Check if basecase is not empty, else make a dummy dataframe
-    if len(basecase) == 0:
-        basecase = pd.DataFrame(columns=["load"])
-    basecase = basecase.rename(columns=dict(load="basecase"))
-
-    combined = combined.merge(basecase, how="left", left_index=True, right_index=True)
-
-    # Raise exception in case of constant load
-    if combined.load.nunique() == 1:
-        logger.warning(
-            "The load is constant! KPIs will still be calculated, but relative metrics"
-            " will be nan."
-        )
-
-    # Define output dictonary
-    kpis = dict()
-
-    # Extract t_aheads from predicted_load,
-    # Make a list of tuples with [(forecast_xh, stdev_xh),(..,..),..]
-    hor_list = [
-        ("forecast_" + t_ahead, "stdev_" + t_ahead)
-        for t_ahead in set(col.split("_")[1] for col in predicted_load.columns)
-    ]
-
-    # cast date to int
-    date = pd.to_datetime(end_time)
-
-    # Calculate model metrics and add them to the output dictionary
-    logger.info("Start calculating kpis")
-    for hor_cols in hor_list:
-        t_ahead_h = hor_cols[0].split("_")[1]
-        fc = combined[hor_cols[0]]  # load predictions
-        st = combined[hor_cols[1]]  # standard deviations of load predictions
-
-        completeness_predicted_load_specific_hor = (
-            validation.calc_completeness_dataframe(fc.to_frame(name=t_ahead_h))[0]
-        )
-        kpis.update(
-            {
-                t_ahead_h: {
-                    "RMSE": metrics.rmse(combined["load"], fc),
-                    "bias": metrics.bias(combined["load"], fc),
-                    "NSME": metrics.nsme(combined["load"], fc),
-                    "MAE": metrics.mae(combined["load"], fc),
-                    "rMAE": metrics.r_mae(combined["load"], fc),
-                    "rMAE_highest": metrics.r_mae_highest(combined["load"], fc),
-                    "rMNE_highest": metrics.r_mne_highest(combined["load"], fc),
-                    "rMPE_highest": metrics.r_mpe_highest(combined["load"], fc),
-                    "rMAE_lowest": metrics.r_mae_lowest(combined["load"], fc),
-                    "skill_score_basecase": metrics.skill_score(
-                        combined["load"],
-                        combined["basecase"],
-                        np.mean(combined["basecase"]),
-                    ),
-                    "skill_score": metrics.skill_score(
-                        combined["load"], fc, np.mean(combined["basecase"])
-                    ),
-                    "skill_score_positive_peaks": metrics.skill_score_positive_peaks(
-                        combined["load"], fc, np.mean(combined["basecase"])
-                    ),
-                    "skill_score_positive_peaks_basecase": metrics.skill_score_positive_peaks(
-                        combined["load"],
-                        combined["basecase"],
-                        np.mean(combined["basecase"]),
-                    ),
-                    "franks_skill_score": metrics.franks_skill_score(
-                        combined["load"], fc, combined["basecase"]
-                    ),
-                    "franks_skill_score_peaks": metrics.franks_skill_score_peaks(
-                        combined["load"], fc, combined["basecase"]
-                    ),
-                    "load_range": combined["load"].max() - combined["load"].min(),
-                    "frac_in_1sdev": metrics.frac_in_stdev(combined["load"], fc, st),
-                    "frac_in_2sdev": metrics.frac_in_stdev(
-                        combined["load"], fc, 2 * st
-                    ),
-                    "completeness_realised": completeness_realised,
-                    "completeness_predicted": completeness_predicted_load_specific_hor,
-                    "date": date,  # cast to date
-                    "window_days": np.round(
-                        (end_time - start_time).total_seconds() / 60.0 / 60.0 / 24.0
-                    ),
-                }
-            }
-        )
-
-        if completeness_realised < COMPLETENESS_REALISED_THRESHOLDS:
-            logger.warning(
-                "Completeness realised load too low",
-                prediction_id=pid,
-                start_time=start_time,
-                end_time=end_time,
-                completeness=completeness_realised,
-                completeness_threshold=COMPLETENESS_REALISED_THRESHOLDS,
-            )
-            set_incomplete_kpi_to_nan(kpis, t_ahead_h)
-        if completeness_predicted_load.any() < COMPLETENESS_PREDICTED_LOAD_THRESHOLD:
-            logger.warning(
-                "Completeness predicted load of specific horizon too low",
-                prediction_id=pid,
-                horizon=t_ahead_h,
-                start_time=start_time,
-                end_time=end_time,
-                completeness=completeness_predicted_load,
-                completeness_threshold=COMPLETENESS_PREDICTED_LOAD_THRESHOLD,
-            )
-            set_incomplete_kpi_to_nan(kpis, t_ahead_h)
-
-    # Return output dictionary
-    return kpis
-
-
-def set_incomplete_kpi_to_nan(kpis: dict, t_ahead_h: str) -> None:
-    """Checks the given kpis for completeness and sets to nan if this not true.
-
-    Args:
-        kpis: the kpis
-        t_ahead_h: t_ahead_h
-
-    """
-    kpi_metrics = list(kpis[t_ahead_h].keys())
-    # Set to nan
-    for kpi in kpi_metrics:
-        if kpi not in [
-            "completeness_realised",
-            "completeness_predicted",
-            "date",
-            "window_days",
-        ]:
-            kpis[t_ahead_h].update({kpi: np.nan})
-
-
-if __name__ == "__main__":
-    main()