openstef 3.4.56__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

openstef/metrics/metrics.py

@@ -1,486 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-
-# This file contains the Objective loss functions for quantile regression from:
-# https://gist.github.com/Nikolay-Lysenko/06769d701c1d9c9acb9a66f2f9d7a6c7
-#
-# SPDX-FileCopyrightText: 2017 Nikolay Lysenko
-#
-# SPDX-License-Identifier: MIT
-"""This module contains all metrics to assess forecast quality."""
-from typing import Callable
-
-import numpy as np
-import pandas as pd
-import xgboost
-
-
-def get_eval_metric_function(metric_name: str) -> Callable:
-    """Gets a metric if it is available.
-
-    Args:
-        metric_name: Name of the metric.
-
-    Returns:
-        Function to calculate the metric.
-
-    Raises:
-        KeyError: If the metric is not available.
-
-    """
-    evaluation_function = {
-        "rmse": rmse,
-        "bias": bias,
-        "nsme": nsme,
-        "mae": mae,
-        "r_mae": r_mae,
-        "frac_in_stdev": frac_in_stdev,
-        "r_mae_highest": r_mae_highest,
-        "r_mne_highest": r_mne_highest,
-        "r_mpe_highest": r_mpe_highest,
-        "r_mae_lowest": r_mae_lowest,
-        "skill_score": skill_score,
-        "skill_score_positive_peaks": skill_score_positive_peaks,
-        "franks_skill_score": franks_skill_score,
-        "franks_skill_score_peaks": franks_skill_score_peaks,
-    }.get(metric_name, None)
-
-    if evaluation_function is None:
-        raise KeyError(f"Unknown evaluation metric function {metric_name}")
-
-    return evaluation_function
-
-
-def rmse(realised: pd.Series, forecast: pd.Series) -> float:
-    """Function that calculates the Root Mean Square Error based on the true and prediciton.
-
-    Args:
-        realised: Realised load.
-        forecast: Forecasted load.
-
-    Returns:
-        Root Mean Square Error
-
-    """
-    return np.sqrt(((realised - forecast) ** 2).mean())
-
-
-def bias(realised: pd.Series, forecast: pd.Series) -> float:
-    """Function that calculates the absolute bias in % based on the true and prediciton.
-
-    Args:
-        realised: Realised load.
-        forecast: Forecasted load.
-
-    Returns:
-        Bias
-
-    """
-    return np.mean(forecast - realised)
-
-
-def nsme(realised: pd.Series, forecast: pd.Series) -> float:
-    """Function that calculates the Nash-sutcliffe model efficiency based on the true and prediciton.
-
-    Args:
-        realised: Realised load.
-        forecast: Forecasted load.
-
-    Returns:
-        Nash-sutcliffe model efficiency
-
-    """
-    try:
-        return 1 - sum((forecast - realised) ** 2) / sum(
-            (realised - np.mean(realised)) ** 2
-        )
-    except ZeroDivisionError:  # means the error is 0
-        return 1
-
-
-def mae(realised: pd.Series, forecast: pd.Series) -> float:
-    """Function that calculates the mean absolute error based on the true and prediction."""
-    return np.mean(np.abs(forecast - realised))
-
-
-def r_mae(realised: pd.Series, forecast: pd.Series) -> float:
-    """Function that calculates the relative mean absolute error based on the true and prediction.
-
-    The range is based on the load range of the previous two weeks
-
-    """
-    # Determine load range on entire dataset
-    range_ = (
-        realised.max() - realised.min()
-        if (realised.max() - realised.min()) != 0
-        else np.nan
-    )
-
-    return mae(realised, forecast) / range_
-
-
-def frac_in_stdev(realised: pd.Series, forecast: pd.Series, stdev: pd.Series) -> float:
-    """Function that calculates the amount of measurements that are within one stdev of our predictions."""
-    outside_stdev = forecast[(forecast - realised).abs() > stdev]
-    return round((1 - (len(outside_stdev) / len(forecast))), 2)
-
-
-def r_mae_highest(
-    realised: pd.Series, forecast: pd.Series, percentile: float = 0.95
-) -> float:
-    """Function that calculates the relative mean absolute error for the 5 percent highest realised values.
-
-    The range is based on the load range of the previous two weeks.
-
-    Raises:
-        ValueError: If the length of the realised and forecast arrays are not equal.
-
-    """
-    # Check if length of both arrays is equal
-    if len(np.array(realised)) != len(np.array(forecast)):
-        raise ValueError(
-            "Error metric can only be calculated for arrays of equal length!"
-        )
-
-    # Determine load range on entire dataset
-    range_ = (
-        realised.max() - realised.min()
-        if (realised.max() - realised.min()) != 0
-        else np.nan
-    )
-
-    # Get highest percentile of values
-    highest_values = realised > np.percentile(realised, percentile)
-
-    # Calculate mae
-    r_mae_highest = mae(realised[highest_values], forecast[highest_values]) / range_
-
-    return r_mae_highest
-
-
-def r_mne_highest(realised: pd.Series, forecast: pd.Series) -> float:
-    """Function that calculates the relative mean negative error for the 5 percent highest realised values.
-
-    The range is based on the load range of the previous two weeks, this measure quantifies how much we underestimate
-    peaks.
-
-    """
-    # Combine series in one DataFrame
-    combined = pd.concat([realised, forecast], axis=1)
-
-    # Determine load range on entire dataset
-    range_ = (
-        combined[realised.name].max() - combined[realised.name].min()
-        if (combined[realised.name].max() - combined[realised.name].min()) != 0
-        else np.nan
-    )
-
-    # Select 5 percent highest realised load values
-    combined["highest"] = combined[realised.name][
-        combined[realised.name] > combined[realised.name].quantile(0.95)
-    ]
-    combined = combined[np.invert(np.isnan(combined["highest"]))]
-
-    # Calculate rMNE for the selected points
-    diff = combined[forecast.name] - combined[realised.name]
-
-    if len(diff[diff < 0]) < 2:
-        return 0.0
-
-    r_mne_highest = np.mean(diff[diff < 0]) / range_
-
-    if np.isnan(r_mne_highest):
-        return 99999.0
-
-    return r_mne_highest
-
-
-def r_mpe_highest(realised: pd.Series, forecast: pd.Series) -> float:
-    """Function that calculates the relative mean positive error for the 5 percent highest realised values.
-
-    The range is based on the load range of the previous two weeks, this measure quantifies how much we overestimate
-    peaks.
-
-    """
-    # Combine series in one DataFrame
-    combined = pd.concat([realised, forecast], axis=1)
-
-    # Determine load range on entire dataset
-    range_ = (
-        combined[realised.name].max() - combined[realised.name].min()
-        if (combined[realised.name].max() - combined[realised.name].min()) != 0
-        else np.nan
-    )
-
-    # Select 5 percent highest realised load values
-    combined["highest"] = combined[realised.name][
-        combined[realised.name] > combined[realised.name].quantile(0.95)
-    ]
-    combined = combined[np.invert(np.isnan(combined["highest"]))]
-
-    # Calculate rMPE for the selected points
-
-    diff = combined[forecast.name] - combined[realised.name]
-
-    if len(diff[diff > 0]) < 2:
-        return 0.0
-
-    r_mpe_highest = np.mean(diff[diff > 0]) / range_
-
-    if np.isnan(r_mpe_highest):
-        return 99999.0
-    return r_mpe_highest
-
-
-def r_mae_lowest(
-    realised: pd.Series, forecast: pd.Series, quantile: float = 0.05
-) -> float:
-    """Function that calculates the relative mean absolute error for the 5 percent lowest realised values.
-
-    The range is based on the load range of the previous two weeks.
-
-    """
-    # Determine load range on entire dataset
-    range_ = (
-        realised.max() - realised.min()
-        if (realised.max() - realised.min()) != 0
-        else np.nan
-    )
-
-    # Get lowest percentile of values
-    lowest_values = realised < np.quantile(realised, quantile)
-    # Calculate mae
-    r_mae_lowest = mae(realised[lowest_values], forecast[lowest_values]) / range_
-
-    return r_mae_lowest
-
-
-def skill_score(realised: pd.Series, forecast: pd.Series, mean: pd.Series) -> float:
-    """Function that calculates the skill score.
-
-    Thise indicates model performance relative to a reference, in this case the mean of the realised values. The range
-    is based on the load range of the previous two weeks.
-
-    """
-    combined = pd.concat([realised, forecast], axis=1)
-    combined["mean"] = mean
-
-    skill_score = 1 - (mae(realised, forecast) / mae(realised, combined["mean"]))
-
-    if np.isnan(skill_score):
-        return 0
-
-    return skill_score
-
-
-def skill_score_positive_peaks(
-    realised: pd.Series, forecast: pd.Series, mean: pd.Series
-) -> float:
-    """Calculates skill score on positive peaks."""
-    # Combine series in one DataFrame
-    combined = pd.concat([realised, forecast], axis=1)
-
-    # Select 5 percent highest realised load values
-    combined["highest"] = combined[realised.name][
-        combined[realised.name] > combined[realised.name].quantile(0.95)
-    ]
-    combined = combined[np.invert(np.isnan(combined["highest"]))]
-
-    # Calculate rMAE for the selected points
-    skill_score_highest = skill_score(
-        combined[realised.name], combined[forecast.name], mean
-    )
-
-    if np.isnan(skill_score_highest):
-        return 0
-
-    return skill_score_highest
-
-
-def franks_skill_score(
-    realised: pd.Series, forecast: pd.Series, basecase: pd.Series, range_: float = 1.0
-) -> float:
-    """Calculate Franks skill score."""
-    # Combine series in one DataFrame
-    combined = pd.concat([realised, forecast], axis=1)
-    if range_ == 1.0:
-        range_ = (
-            combined[realised.name].max() - combined[realised.name].min()
-            if (combined[realised.name].max() - combined[realised.name].min()) != 0
-            else np.nan
-        )
-
-    franks_skill_score = (mae(realised, basecase) - mae(realised, forecast)) / range_
-
-    if np.isnan(franks_skill_score):
-        return 0
-
-    return franks_skill_score
-
-
-def franks_skill_score_peaks(
-    realised: pd.Series, forecast: pd.Series, basecase: pd.Series
-) -> float:
-    """Calculate Franks skill score on positive peaks."""
-    # Combine series in one DataFrame
-    combined = pd.concat([realised, forecast, basecase], axis=1)
-
-    range_ = (
-        combined[realised.name].max() - combined[realised.name].min()
-        if (combined[realised.name].max() - combined[realised.name].min()) != 0
-        else np.nan
-    )
-    # Select 5 percent highest realised load values
-    combined["highest"] = combined[realised.name][
-        combined[realised.name] > combined[realised.name].quantile(0.95)
-    ]
-    combined = combined[np.invert(np.isnan(combined["highest"]))]
-
-    # Calculate rMAE for the selected points
-    franks_skill_score_highest = franks_skill_score(
-        combined[realised.name],
-        combined[forecast.name],
-        combined[basecase.name],
-        range_=range_,
-    )
-
-    if np.isnan(franks_skill_score_highest):
-        return 0
-
-    return franks_skill_score_highest
-
-
-# Objective loss functions for quantile regression, from: https://gist.github.com/Nikolay-Lysenko/06769d701c1d9c9acb9a66f2f9d7a6c7
-
-# SPDX-FileCopyrightText: 2017 Nikolay Lysenko
-#
-# SPDX-License-Identifier: MIT
-
-
-def xgb_quantile_eval(
-    preds: np.ndarray, dmatrix: xgboost.DMatrix, quantile: float = 0.2
-) -> str:
-    """Customized evaluational metric that equals to quantile regression loss (also known as pinball loss).
-
-    Quantile regression is regression that estimates a specified quantile of target's distribution conditional on given features.
-
-    Args:
-        preds: Predicted values
-        dmatrix: xgboost.DMatrix of the input data.
-        quantile: Target quantile.
-
-    Returns:
-        Loss information
-
-
-    # See also:
-    https://gist.github.com/Nikolay-Lysenko/06769d701c1d9c9acb9a66f2f9d7a6c7
-
-    """
-    labels = dmatrix.get_label()
-    return (
-        "q{}_loss".format(quantile),
-        np.nanmean(
-            (preds >= labels) * (1 - quantile) * (preds - labels)
-            + (preds < labels) * quantile * (labels - preds)
-        ),
-    )
-
-
-def xgb_quantile_obj(
-    preds: np.ndarray, dmatrix: xgboost.DMatrix, quantile: float = 0.2
-) -> tuple[np.ndarray, np.ndarray]:
-    """Quantile regression objective fucntion.
-
-    Computes first-order derivative of quantile regression loss and a non-degenerate substitute for second-order
-    derivative.
-
-    Substitute is returned instead of zeros, because XGBoost requires non-zero second-order derivatives. See
-    this page: https://github.com/dmlc/xgboost/issues/1825 to see why it is possible to use this trick. However, be sure
-    that hyperparameter named `max_delta_step` is small enough to satisfy:``0.5 * max_delta_step <= min(quantile, 1 - quantile)``.
-
-    Args:
-        preds: numpy.ndarray
-        dmatrix: xgboost.DMatrix
-        quantile: float between 0 and 1
-
-    Returns:
-        Gradient and Hessian
-
-    # See also:
-    https://gist.github.com/Nikolay-Lysenko/06769d701c1d9c9acb9a66f2f9d7a6c7
-
-    Reasoning for the hessian:
-    https://gist.github.com/Nikolay-Lysenko/06769d701c1d9c9acb9a66f2f9d7a6c7#gistcomment-2322558
-
-    """
-    try:
-        assert 0 <= quantile <= 1
-    except AssertionError:
-        raise ValueError("Quantile value must be float between 0 and 1.")
-
-    labels = dmatrix.get_label()
-    errors = preds - labels
-
-    left_mask = errors < 0
-    right_mask = errors > 0
-
-    # The factor `* errors` is different from the original implementation, however
-    # this addition makes the objective function scalable with the size of the error.
-    # This solves issues with regression on large (>100) input data.
-    grad = (quantile * left_mask + (1 - quantile) * right_mask) * errors
-    hess = np.ones_like(preds)
-
-    return grad, hess
-
-
-def arctan_loss(y_true, y_pred, taus, s=0.1):
-    """Compute the arctan pinball loss.
-
-    Note that XGBoost outputs the predictions in a slightly peculiar manner.
-    Suppose we have 100 data points and we predict 10 quantiles. The predictions
-    will be an array of size (1000 x 1). We first resize this to a (100x10) array
-    where each row corresponds to the 10 predicted quantile for a single data
-    point. We then use a for-loop (over the 10 columns) to calculate the gradients
-    and second derivatives. Legibility was chosen over efficiency. This part
-    can be made more efficient.
-
-    Args:
-        y_true: An array containing the true observations.
-        y_pred: An array containing the predicted quantiles.
-        taus: A list containing the true desired coverage of the quantiles.
-        s: A smoothing parameter.
-
-    Returns:
-        grad: An array containing the (negative) gradients with respect to y_pred.
-        hess: An array containing the second derivative with respect to y_pred.
-
-    """
-    size = len(y_true)
-    n_dim = len(taus)  # The number of columns
-    n_rows = size // n_dim
-
-    # Resize the predictions and targets.
-    # Each column corresponds to a quantile, each row to a data point.
-    y_pred = np.reshape(y_pred, (n_rows, n_dim))
-    y_true = np.reshape(y_true, (n_rows, n_dim))
-
-    # Calculate the differences
-    u = y_true - y_pred
-
-    # Calculate the gradient and second derivatives
-    grad = np.zeros_like(y_pred)
-    hess = np.zeros_like(y_pred)
-    z = u / s
-    for i, tau in enumerate(taus):
-        x = 1 + z[:, i] ** 2
-        grad[:, i] = (
-            tau - 0.5 + 1 / np.pi * np.arctan(z[:, i]) + z[:, i] / (np.pi) * x**-1
-        )
-        hess[:, i] = 2 / (np.pi * s) * x ** (-2)
-
-    # Reshape back to the original shape.
-    grad = grad.reshape(size)
-    hess = hess.reshape(size)
-    return -grad / n_dim, hess / n_dim

openstef/metrics/reporter.py

@@ -1,222 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""Defines reporter class."""
-import logging
-import os
-import warnings
-from dataclasses import dataclass
-
-import numpy as np
-import pandas as pd
-import sklearn
-import structlog
-from mlflow.models import ModelSignature, infer_signature
-from plotly.graph_objects import Figure
-
-from openstef.metrics import figure
-from openstef.metrics.metrics import bias, mae, nsme, r_mae, rmse
-from openstef.model.regressors.regressor import OpenstfRegressor
-from openstef.settings import Settings
-
-
-@dataclass
-class Report:
-    """Dataclass to hold a report describing the training process."""
-
-    def __init__(
-        self,
-        feature_importance_figure: Figure,
-        data_series_figures: dict[str, Figure],
-        metrics: dict,
-        signature: ModelSignature,
-    ):
-        """Initialize training report.
-
-        Args:
-            feature_importance_figure: Figure with feature importance
-            data_series_figures: Figure with input data time series.
-            metrics: Dict with metrics
-            signature: Model signature
-
-        """
-        self.feature_importance_figure = feature_importance_figure
-        self.data_series_figures = data_series_figures
-        self.metrics = metrics
-        self.signature = signature
-
-
-class Reporter:
-    """Reporter class that generates reports describing the training process."""
-
-    def __init__(
-        self,
-        train_data: pd.DataFrame = None,
-        validation_data: pd.DataFrame = None,
-        test_data: pd.DataFrame = None,
-        quantiles: list[float] = None,
-    ) -> None:
-        """Initializes reporter.
-
-        Args:
-            train_data: Dataframe with training data
-            validation_data: Dataframe with validation data
-            test_data: Dataframe with test data
-            quantiles: List of predicted quantiles that have to be plotted.
-
-        """
-        self.horizons = train_data.horizon.unique()
-        self.predicted_data_list = []
-        self.input_data_list = [train_data, validation_data, test_data]
-        self.quantiles = [] if quantiles is None else sorted(quantiles)
-
-    def generate_report(
-        self,
-        model: OpenstfRegressor,
-    ) -> Report:
-        """Generate a report on a given model.
-
-        Args:
-            model: the model to create a report on
-
-        Returns:
-            Reporter object containing info about the model
-
-        """
-        # Get training (input_data_list[0]) and validation (input_data_list[1]) set
-        train_x, train_y = (
-            self.input_data_list[0].iloc[:, 1:-1],
-            self.input_data_list[0].iloc[:, 0],
-        )
-        valid_x, valid_y = (
-            self.input_data_list[1].iloc[:, 1:-1],
-            self.input_data_list[1].iloc[:, 0],
-        )
-
-        data_series_figures = self._make_data_series_figures(model)
-
-        # feature_importance_dataframe should be a dataframe, to create a figure
-        # can be None if we have no feature importance
-        if isinstance(model.feature_importance_dataframe, pd.DataFrame):
-            feature_importance_figure = figure.plot_feature_importance(
-                model.feature_importance_dataframe
-            )
-        # If it isn't a dataframe we will set feature_importance_figure, so it will not create the figure
-        else:
-            feature_importance_figure = None
-
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-
-            if model.can_predict_quantiles:
-                fiabilities = self.get_fiabilities(
-                    {q: model.predict(valid_x, quantile=q) for q in self.quantiles},
-                    valid_y,
-                )
-            else:
-                fiabilities = {}
-
-            report = Report(
-                data_series_figures=data_series_figures,
-                feature_importance_figure=feature_importance_figure,
-                metrics={
-                    **self.get_metrics(model.predict(valid_x), valid_y),
-                    **fiabilities,
-                },
-                signature=infer_signature(train_x, train_y),
-            )
-
-        return report
-
-    @staticmethod
-    def get_fiabilities(quantiles: dict[float, np.array], y_true: np.array) -> dict:
-        fiabilities_dict = {}
-        for alpha, qhat in quantiles.items():
-            fiabilities_dict[f"fiability_at_q{alpha}"] = np.mean(qhat >= y_true)
-        return fiabilities_dict
-
-    @staticmethod
-    def get_metrics(y_pred: np.array, y_true: np.array) -> dict:
-        """Calculate the metrics for a prediction.
-
-        Args:
-            y_pred: np.array
-            y_true: np.array
-
-        Returns:
-            Metrics for the prediction
-
-        """
-        metric_dict = {
-            "bias": bias,
-            "NSME": nsme,
-            "MAE": mae,
-            "R_MAE": r_mae,
-            "RMSE": rmse,
-            "explained_variance": sklearn.metrics.explained_variance_score,
-            "MSE": sklearn.metrics.mean_squared_error,
-            "r2": sklearn.metrics.r2_score,
-        }
-        results = {}
-        for name, metric in metric_dict.items():
-            try:
-                results[name] = metric(y_true, y_pred)
-            except ValueError:
-                continue
-        return results
-
-    @staticmethod
-    def write_report_to_disk(report: Report, report_folder: str):
-        """Write report to disk; e.g. for viewing report of latest models using grafana."""
-        # Initialize logger and serializer
-        structlog.configure(
-            wrapper_class=structlog.make_filtering_bound_logger(
-                logging.getLevelName(Settings.log_level)
-            )
-        )
-        logger = structlog.get_logger(__name__)
-        if report_folder:
-            # create path if does not exist
-            if not os.path.exists(report_folder):
-                os.makedirs(report_folder)
-            logger.info(f"Writing reports to {report_folder}")
-            # write feature importance figure
-            if report.feature_importance_figure:  # only write if figure is not none
-                report.feature_importance_figure.write_html(
-                    os.path.join(report_folder, "weight_plot.html")
-                )
-            # write predictors
-            for name, figure in report.data_series_figures.items():
-                if figure:  # only write if figure is not none
-                    figure.write_html(os.path.join(report_folder, f"{name}.html"))
-
-    def _make_data_series_figures(self, model: OpenstfRegressor) -> dict:
-        """Make data series figures."""
-        # Make model predictions
-        for data_set in self.input_data_list:
-            # First ("load") and last ("horizon") are removed here
-            # as they are not expected by the model as prediction input
-            model_forecast = model.predict(data_set.iloc[:, 1:-1])
-            forecast = pd.DataFrame(
-                index=data_set.index, data={"forecast": model_forecast}
-            )
-
-            if (model.can_predict_quantiles) & (len(self.quantiles) >= 2):
-                forecast.loc[:, f"q{100 * self.quantiles[0]}"] = model.predict(
-                    data_set.iloc[:, 1:-1], quantile=self.quantiles[0]
-                )
-                forecast.loc[:, f"q{100 * self.quantiles[-1]}"] = model.predict(
-                    data_set.iloc[:, 1:-1], quantile=self.quantiles[-1]
-                )
-
-            self.predicted_data_list.append(forecast)
-
-        # Make cufflinks plots for the data series
-        return {
-            f"Predictor{horizon}": figure.plot_data_series(
-                data=self.input_data_list,
-                predict_data=self.predicted_data_list,
-                horizon=horizon,
-            )
-            for horizon in self.horizons
-        }

openstef/model/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0