openstef 3.2.69__py3-none-any.whl → 3.2.71__py3-none-any.whl

openstef/metrics/figure.py

@@ -106,16 +106,29 @@ def plot_data_series(
         # Filter data on given horizon
         actuals = []
         predictions = []
+        q_low = []
+        q_high = []
 
         for series, predict_series in zip(data, predict_data):
             mask = series["horizon"] == horizon
             actuals.append(series[mask]["load"])
             predictions.append(predict_series[mask]["forecast"])
+            if len(predict_series[mask].columns) > 1:
+                q_low.append(predict_series[mask].iloc[:, -2])
+                q_high.append(predict_series[mask].iloc[:, -1])
+
     else:
         actuals = data
         predictions = predict_data
+        if len(predictions.columns) > 1:
+            q_low = predict_data.iloc[:, -2]
+            q_high = predict_data.iloc[:, -1]
+        else:
+            q_low = None
+            q_high = None
 
-    fig = _plot_data_and_predictions(names, actuals, predictions)
+    quantiles = [q_low, q_high] if (q_low is not None) and (len(q_low) != 0) else None
+    fig = _plot_data_and_predictions(names, actuals, predictions, quantiles)
     fig.update_layout(
         title=f"Predictor in action for horizon: {horizon}",
     )
@@ -167,7 +180,10 @@ def _plot_data(names: list[str], series: list[pd.Series]) -> go.Figure:
 
 
 def _plot_data_and_predictions(
-    names: list[str], actuals: list[pd.Series], predictions: list[pd.Series]
+    names: list[str],
+    actuals: list[pd.Series],
+    predictions: list[pd.Series],
+    quantiles: list[float] = None,
 ) -> go.Figure:
     """Create plot of different data and prediction splits.
 
@@ -178,6 +194,7 @@ def _plot_data_and_predictions(
         names: Name of each seperate split. The passed names will be suffixed with _actual and _predict for data and predictions respectively.
         actuals: Each data split as a seperate series.
         predictions: Each prediction split as a seperate series.
+        quantiles: List of predicted quantiles that have to be plotted.
 
     Returns:
         A line plot of each passed series.
@@ -186,13 +203,29 @@ def _plot_data_and_predictions(
     # Build a combined DataFrame with all data.
     # This step is important to create forced NaNs to create gaps in the plot.
     combined = []
-    for name, actual, prediction in zip(names, actuals, predictions):
-        combined.extend(
-            [
-                actual.rename(f"{name}_actual"),
-                prediction.rename(f"{name}_predict"),
-            ]
-        )
+    if quantiles is None:
+        for name, actual, prediction in zip(names, actuals, predictions):
+            combined.extend(
+                [
+                    actual.rename(f"{name}_actual"),
+                    prediction.rename(f"{name}_predict"),
+                ]
+            )
+    else:
+        for name, actual, prediction, q_low, q_high in zip(
+            names, actuals, predictions, quantiles[0], quantiles[-1]
+        ):
+            q_low_name = q_low.name
+            q_high_name = q_high.name
+            combined.extend(
+                [
+                    actual.rename(f"{name}_actual"),
+                    prediction.rename(f"{name}_predict"),
+                    q_low.rename(f"{name}_{q_low_name}"),
+                    q_high.rename(f"{name}_{q_high_name}"),
+                ]
+            )
+
     df_plot = pd.concat(combined, axis=1)
 
     fig = go.Figure()
@@ -200,7 +233,6 @@ def _plot_data_and_predictions(
     # Add a trace for every data series
     for i, name in enumerate(names):
         actual, predict = f"{name}_actual", f"{name}_predict"
-
         fig.add_trace(
             go.Scatter(
                 x=df_plot.index,
@@ -217,6 +249,32 @@ def _plot_data_and_predictions(
                 line=dict(dash="dot", color=px.colors.qualitative.Dark2[i]),
             )
         )
+        if quantiles is not None:
+            q_low, q_high = f"{name}_{q_low_name}", f"{name}_{q_high_name}"
+            fig.add_trace(
+                go.Scatter(
+                    x=df_plot.index,
+                    y=df_plot[q_low],
+                    mode="lines",
+                    line=dict(
+                        color=px.colors.qualitative.Dark2[i], width=0.5, dash="dash"
+                    ),
+                    name=q_low,
+                )
+            )
+            fig.add_trace(
+                go.Scatter(
+                    x=df_plot.index,
+                    y=df_plot[q_high],
+                    fill="tonexty",
+                    fillcolor=f"rgba({px.colors.qualitative.Dark2[i][4:-1]}, 0.3)",
+                    mode="lines",
+                    line=dict(
+                        color=px.colors.qualitative.Dark2[i], width=0.5, dash="dash"
+                    ),
+                    name=q_high,
+                )
+            )
 
     fig.update_layout(yaxis_title="Load (MW)")
 

openstef/metrics/reporter.py

@@ -52,6 +52,7 @@ class Reporter:
         train_data: pd.DataFrame = None,
         validation_data: pd.DataFrame = None,
         test_data: pd.DataFrame = None,
+        quantiles: list[float] = None,
     ) -> None:
         """Initializes reporter.
 
@@ -59,11 +60,13 @@ class Reporter:
             train_data: Dataframe with training data
             validation_data: Dataframe with validation data
             test_data: Dataframe with test data
+            quantiles: List of predicted quantiles that have to be plotted.
 
         """
         self.horizons = train_data.horizon.unique()
         self.predicted_data_list = []
         self.input_data_list = [train_data, validation_data, test_data]
+        self.quantiles = [] if quantiles is None else sorted(quantiles)
 
     def generate_report(
         self,
@@ -102,15 +105,34 @@ class Reporter:
 
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
+
+            if model.can_predict_quantiles:
+                fiabilities = self.get_fiabilities(
+                    {q: model.predict(valid_x, quantile=q) for q in self.quantiles},
+                    valid_y,
+                )
+            else:
+                fiabilities = {}
+
             report = Report(
                 data_series_figures=data_series_figures,
                 feature_importance_figure=feature_importance_figure,
-                metrics=self.get_metrics(model.predict(valid_x), valid_y),
+                metrics={
+                    **self.get_metrics(model.predict(valid_x), valid_y),
+                    **fiabilities,
+                },
                 signature=infer_signature(train_x, train_y),
             )
 
         return report
 
+    @staticmethod
+    def get_fiabilities(quantiles: dict[float, np.array], y_true: np.array) -> dict:
+        fiabilities_dict = {}
+        for alpha, qhat in quantiles.items():
+            fiabilities_dict[f"fiability_at_q{alpha}"] = np.mean(qhat >= y_true)
+        return fiabilities_dict
+
     @staticmethod
     def get_metrics(y_pred: np.array, y_true: np.array) -> dict:
         """Calculate the metrics for a prediction.
@@ -171,6 +193,15 @@ class Reporter:
             forecast = pd.DataFrame(
                 index=data_set.index, data={"forecast": model_forecast}
             )
+
+            if (model.can_predict_quantiles) & (len(self.quantiles) >= 2):
+                forecast.loc[:, f"q{100 * self.quantiles[0]}"] = model.predict(
+                    data_set.iloc[:, 1:-1], quantile=self.quantiles[0]
+                )
+                forecast.loc[:, f"q{100 * self.quantiles[-1]}"] = model.predict(
+                    data_set.iloc[:, 1:-1], quantile=self.quantiles[-1]
+                )
+
             self.predicted_data_list.append(forecast)
 
         # Make cufflinks plots for the data series

openstef/model/objective.py

@@ -97,7 +97,12 @@ class RegressorObjective:
                 "stratification_min_max": self.model_type != MLModelType.ProLoaf,
                 "back_test": True,
             }
-        (self.train_data, self.validation_data, self.test_data,) = self.split_func(
+        (
+            self.train_data,
+            self.validation_data,
+            self.test_data,
+            self.operational_score_data,
+        ) = self.split_func(
             self.input_data,
             test_fraction=self.test_fraction,
             validation_fraction=self.validation_fraction,

openstef/model_selection/model_selection.py

@@ -113,7 +113,7 @@ def split_data_train_validation_test(
     validation dataset. In an operational setting the following sequence is
     returned (when using stratification):
 
-    Test >> Train >> Validation
+    Train >> Validation (and the test is the Train and Validation combined.)
 
     For a back test (indicated with argument "back_test") the following sequence
     is returned:
@@ -141,6 +141,7 @@ def split_data_train_validation_test(
         - Test data.
 
     """
+    test_fraction = test_fraction if back_test else 0
     train_fraction = 1 - (test_fraction + validation_fraction)
     if train_fraction < 0:
         raise ValueError(
@@ -172,10 +173,18 @@ def split_data_train_validation_test(
         start_date_test = end_date - np.round(number_indices * test_fraction) * delta
         test_data = data_[start_date_test:]
         train_val_data = data_[:start_date_test]
+        operational_score_data = (
+            pd.DataFrame()
+        )  # Empty because a backtest is no operational setting.
     else:
         start_date_val = start_date + np.round(number_indices * test_fraction) * delta
-        test_data = data_[:start_date_val]
+        test_data = data_[
+            :start_date_val
+        ]  # Empty as all data is used for training in an operational setting.
         train_val_data = data_[start_date_val:]
+        operational_score_data = data_.copy(deep=True).reset_index(
+            drop=True
+        )  # Used to check wether a new operationally train model is better than the old one.
 
     if stratification_min_max and (
         len(set(train_val_data.index.date)) >= min_days_for_stratification
@@ -248,11 +257,7 @@ def split_data_train_validation_test(
     validation_data = validation_data.sort_index()
     test_data = test_data.sort_index()
 
-    return (
-        train_data,
-        validation_data,
-        test_data,
-    )
+    return (train_data, validation_data, test_data, operational_score_data)
 
 
 def backtest_split_default(
@@ -286,14 +291,14 @@ def backtest_split_default(
         for ifold in range(n_folds):
             test_data = data[data["random_fold"] == ifold].sort_index()
 
-            (train_data, validation_data, _,) = split_data_train_validation_test(
+            (train_data, validation_data, _, _) = split_data_train_validation_test(
                 data[data["random_fold"] != ifold].iloc[:, :-2],
                 test_fraction=0,
                 back_test=True,
                 stratification_min_max=stratification_min_max,
             )
 
-            yield train_data, validation_data, test_data.iloc[:, :-2]
+            yield train_data, validation_data, test_data.iloc[:, :-2], pd.DataFrame()
     else:
         yield split_data_train_validation_test(
             data,

openstef/pipeline/train_create_forecast_backtest.py

@@ -82,10 +82,14 @@ def train_model_and_forecast_back_test(
     ) = zip(
         *(
             train_model_and_forecast_test_core(
-                pj, modelspecs, train_data, validation_data, test_data
+                pj,
+                modelspecs,
+                train_data,
+                validation_data,
+                test_data,
             )
             + (train_data, validation_data, test_data)
-            for train_data, validation_data, test_data in backtest_split_func(
+            for train_data, validation_data, test_data, _ in backtest_split_func(
                 data_with_features, n_folds, **backtest_split_args
             )
         )

openstef/pipeline/train_model.py

@@ -176,7 +176,14 @@ def train_model_pipeline_core(
     logger = structlog.get_logger(__name__)
 
     # Call common pipeline
-    model, report, train_data, validation_data, test_data = train_pipeline_common(
+    (
+        model,
+        report,
+        train_data,
+        validation_data,
+        test_data,
+        operational_score_data,
+    ) = train_pipeline_common(
         pj,
         model_specs,
         input_data,
@@ -192,8 +199,8 @@ def train_model_pipeline_core(
             combined = combined.iloc[:, :-1]
 
         x_data, y_data = (
-            combined.iloc[:, 1:-1],
-            combined.iloc[:, 0],
+            operational_score_data.iloc[:, 1:-1],
+            operational_score_data.iloc[:, 0],
         )
 
         # Score method always returns R^2
@@ -260,7 +267,12 @@ def train_pipeline_common(
         horizons=horizons,
     )
 
-    train_data, validation_data, test_data = train_pipeline_step_split_data(
+    (
+        train_data,
+        validation_data,
+        test_data,
+        operational_score_data,
+    ) = train_pipeline_step_split_data(
         data_with_features=data_with_features,
         pj=pj,
         test_fraction=test_fraction,
@@ -276,7 +288,7 @@ def train_pipeline_common(
     )
 
     # Report about the training process
-    reporter = Reporter(train_data, validation_data, test_data)
+    reporter = Reporter(train_data, validation_data, test_data, pj.quantiles)
     report = reporter.generate_report(model)
 
     if pj.save_train_forecasts:
@@ -284,7 +296,7 @@ def train_pipeline_common(
         validation_data["forecast"] = model.predict(validation_data.iloc[:, 1:-1])
         test_data["forecast"] = model.predict(test_data.iloc[:, 1:-1])
 
-    return model, report, train_data, validation_data, test_data
+    return model, report, train_data, validation_data, test_data, operational_score_data
 
 
 def train_pipeline_step_load_model(
@@ -515,7 +527,7 @@ def train_pipeline_step_split_data(
             required_arguments=["data", "test_fraction"]
         )
 
-    train_data, validation_data, test_data = split_func(
+    train_data, validation_data, test_data, operational_score_data = split_func(
         data_with_features, test_fraction, **split_args
     )
 
@@ -523,4 +535,4 @@ def train_pipeline_step_split_data(
     if not test_data_predefined.empty:
         test_data = test_data_predefined
 
-    return train_data, validation_data, test_data
+    return train_data, validation_data, test_data, operational_score_data

{openstef-3.2.69.dist-info → openstef-3.2.71.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openstef
-Version: 3.2.69
+Version: 3.2.71
 Summary: Open short term energy forecaster
 Home-page: https://github.com/OpenSTEF/openstef
 Author: Alliander N.V

{openstef-3.2.69.dist-info → openstef-3.2.71.dist-info}/RECORD

@@ -24,15 +24,15 @@ openstef/feature_engineering/holiday_features.py,sha256=J24CURDmQOlYTFh9ffnuWc7k
 openstef/feature_engineering/lag_features.py,sha256=cMAZ5ekhNSKg7J9shoEjSa3VPrT0Z4ZjatMRsOfdeh4,5639
 openstef/feature_engineering/weather_features.py,sha256=wy3KFXUIIwSydFJZpiejsJMwURtDpv9l0HBHu-uLAGQ,15561
 openstef/metrics/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
-openstef/metrics/figure.py,sha256=n2n5YaRZODTJ5oIuN1AwAkFHgFFbj_qtbxTQB4BbvH4,7473
+openstef/metrics/figure.py,sha256=NPJGI4FygjSnOQuL8qCbB87-T31q6EkewkbVmpLwmnk,9657
 openstef/metrics/metrics.py,sha256=c6HGQubArT5G4YxF0KY9HCP19PRHaVfXQ8KEkSwrt0w,13164
-openstef/metrics/reporter.py,sha256=uuHWWtrYBpbw7gWgdBgb5VSHO11pkZluZ-YYEcrVAUM,6412
+openstef/metrics/reporter.py,sha256=V6pa4IUOzVcZ8OY632g5KoF8hr2MT2ySexrjZCjnuwY,7668
 openstef/model/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/model/basecase.py,sha256=caI6Q-8y0ymlxGK9Js_H3Vh0q6ruNHlGD5RG0_kE5M0,2878
 openstef/model/confidence_interval_applicator.py,sha256=7E1_JFLZ4-hyEhleacMvp5szdmYZS4tpKAjfhGvXXvg,8602
 openstef/model/fallback.py,sha256=VV9ehgnoMZtWzqKk9H1t8wnERFh5CyC4TvDIuRP_ZDI,2861
 openstef/model/model_creator.py,sha256=U1Lw4HFyajfxQ2o5lEnCxnmRC62DEu5PBHXrm_jnEJU,5582
-openstef/model/objective.py,sha256=1v8ghCqEY2-Fku5NApQBVN52hIqzoYLvw9uWVKzmkE4,15347
+openstef/model/objective.py,sha256=85CWxLOw8eDe1Waj81H_f8Rm5YaS-AlhCfzcMT4yFyM,15434
 openstef/model/objective_creator.py,sha256=OiPPFSiSu7z9K_983ib5iqhhu6_9tt7iyTyKNZ2Iz68,2057
 openstef/model/serializer.py,sha256=mfa8VZOXKK05zhDKBpXFw4E_UKndHl-fUuMQJiGPGjI,16909
 openstef/model/standard_deviation_generator.py,sha256=WCgZwerAEURUnSNW-DzpvJHC-3piD8TMZiOI60-HfZ8,2913
@@ -50,7 +50,7 @@ openstef/model/regressors/regressor.py,sha256=uJcx59AyCPE9f_yPcAQ59h2ZS7eNsDpIHJ
 openstef/model/regressors/xgb.py,sha256=HggA1U10srzdysjV560BMMX66kfaxCKAnOZB3JyyT_Y,808
 openstef/model/regressors/xgb_quantile.py,sha256=pjtG0WxEEPnKlL63iDHhUqydx_UVK_9w49uhlp0WS6c,7458
 openstef/model_selection/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
-openstef/model_selection/model_selection.py,sha256=3wbFVJY_fI7_t-7ZAky3VR4hCZ_z9O9x5dqzDK-5Jpk,10569
+openstef/model_selection/model_selection.py,sha256=oGloQBP_FPdNyCs9wzS3l8zFNJxMs1P5XPjVN9qUOsw,11081
 openstef/monitoring/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/monitoring/performance_meter.py,sha256=mMQKpDNv_-RcNYdEvEFPvB76lkG8V9gJOKYQqnH5BX4,2851
 openstef/monitoring/teams.py,sha256=fnZScPD55z9yC0q3YavWj40GEZmL7tsSGhWzG_sMPws,6401
@@ -59,8 +59,8 @@ openstef/pipeline/create_basecase_forecast.py,sha256=BPxf2MSvJyfbNCQGCr1Rol5ShqC
 openstef/pipeline/create_component_forecast.py,sha256=HgByae6ruVhy6TuGIJEuPyLyx7g4zSvJfk6Dynlqjl4,5030
 openstef/pipeline/create_forecast.py,sha256=2vK2cH_VeRcoDWPXR06zFmwQ043FPA9uPvg5_OyxUfU,5008
 openstef/pipeline/optimize_hyperparameters.py,sha256=qptTlg6v0hvHt1ocA7nueLwYRXAB82VI6bQ9ATmBVKQ,10824
-openstef/pipeline/train_create_forecast_backtest.py,sha256=BTJKH_VNu-ZLgy7UuIzEOHK8a_eK-o6JCSpcG-uUIQo,5444
-openstef/pipeline/train_model.py,sha256=rGTZ8ZztLLMAnrojI_tawYCW-8M6W_P4nP9wuN1llBQ,18307
+openstef/pipeline/train_create_forecast_backtest.py,sha256=upuoiE01vjjxUu_sY0tANPqdOtpGKrQQ3azhVDnBJdc,5512
+openstef/pipeline/train_model.py,sha256=tYC8xh6eKpea0CutHNoSGYvt6hoQt4vDWqXmZs6ejbk,18567
 openstef/pipeline/utils.py,sha256=fkc-oNirJ-JiyuOAL08RFrnPYPwudWal_N-BO6Cw980,2086
 openstef/postprocessing/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/postprocessing/postprocessing.py,sha256=nehd0tDpkdIaWFJggQ-fDizIKdfmqJ3IOGfk0sDnrzk,8409
@@ -83,8 +83,8 @@ openstef/tasks/utils/predictionjobloop.py,sha256=u4WQjvqBM6z9T7VFUZ-9JqgdepNJO0Z
 openstef/tasks/utils/taskcontext.py,sha256=yI6TntOkZcW8JiNVuw4uJIigEBL0_iIrkPklF4ZeCX4,5401
 openstef/validation/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/validation/validation.py,sha256=AYQJBXwbFhpq34bqEhybw0lTIJ8Td4vr2-AbWxGxm3M,16917
-openstef-3.2.69.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
-openstef-3.2.69.dist-info/METADATA,sha256=RuxSKGf7C7DTHEMCdxcfzrkComwDDI1TxoLXSw0R0Fg,6972
-openstef-3.2.69.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-openstef-3.2.69.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
-openstef-3.2.69.dist-info/RECORD,,
+openstef-3.2.71.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
+openstef-3.2.71.dist-info/METADATA,sha256=Uou71qcVT-bsF0YLEHWasre0rbhX68vvkfqF-W0OBxM,6972
+openstef-3.2.71.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+openstef-3.2.71.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
+openstef-3.2.71.dist-info/RECORD,,