openstef 3.4.64__tar.gz → 3.4.65__tar.gz

{openstef-3.4.64 → openstef-3.4.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: openstef
-Version: 3.4.64
+Version: 3.4.65
 Summary: Open short term energy forecaster
 Home-page: https://github.com/OpenSTEF/openstef
 Author: Alliander N.V

{openstef-3.4.64 → openstef-3.4.65}/openstef/data_classes/prediction_job.py

@@ -96,6 +96,10 @@ class PredictionJobDataClass(BaseModel):
         1440,
         description="Number of minutes that the load has to be constant to detect a flatliner.",
     )
+    detect_non_zero_flatliner: bool = Field(
+        False,
+        description="If True, flatliners are also detected on non-zero values (median of the load).",
+    )
     data_balancing_ratio: Optional[float] = Field(
         None,
         description="If data balancing is enabled, the data will be balanced with data from 1 year ago in the future.",

{openstef-3.4.64 → openstef-3.4.65}/openstef/exceptions.py

@@ -44,8 +44,8 @@ class InputDataWrongColumnOrderError(InputDataInvalidError):
     """Wrong column order input data."""
 
 
-class InputDataOngoingZeroFlatlinerError(InputDataInvalidError):
-    """All recent load measurements are zero."""
+class InputDataOngoingFlatlinerError(InputDataInvalidError):
+    """All recent load measurements are constant."""
 
 
 class OldModelHigherScoreError(Exception):

{openstef-3.4.64 → openstef-3.4.65}/openstef/pipeline/create_basecase_forecast.py

@@ -8,7 +8,7 @@ import pandas as pd
 import structlog
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.exceptions import InputDataOngoingZeroFlatlinerError, NoRealisedLoadError
+from openstef.exceptions import NoRealisedLoadError
 from openstef.feature_engineering.feature_applicator import (
     OperationalPredictFeatureApplicator,
 )
@@ -58,12 +58,12 @@ def create_basecase_forecast_pipeline(
     if not isinstance(input_data.index, pd.DatetimeIndex):
         raise ValueError("Input dataframe does not have a datetime index.")
 
-    zero_flatliner_ongoing = validation.detect_ongoing_zero_flatliner(
+    flatliner_ongoing = validation.detect_ongoing_flatliner(
         load=input_data.iloc[:, 0],
         duration_threshold_minutes=pj.flatliner_threshold_minutes,
     )
 
-    if zero_flatliner_ongoing:
+    if flatliner_ongoing:
         # Set historic load to zero to force the basecase forecasts to be zero.
         input_data.loc[input_data.index < forecast_start, "load"] = 0
 

{openstef-3.4.64 → openstef-3.4.65}/openstef/pipeline/create_forecast.py

@@ -45,7 +45,7 @@ def create_forecast_pipeline(
         DataFrame with the forecast
 
     Raises:
-        InputDataOngoingZeroFlatlinerError: When all recent load measurements are zero.
+        InputDataOngoingFlatlinerError: When all recent load measurements are constant.
         LookupError: When no model is found for the given prediction job in MLflow.
 
     """
@@ -85,7 +85,7 @@ def create_forecast_pipeline_core(
         Forecast
 
     Raises:
-        InputDataOngoingZeroFlatlinerError: When all recent load measurements are zero.
+        InputDataOngoingFlatlinerError: When all recent load measurements are constant.
 
     """
     structlog.configure(
@@ -103,6 +103,7 @@ def create_forecast_pipeline_core(
         input_data,
         pj["flatliner_threshold_minutes"],
         pj["resolution_minutes"],
+        detect_non_zero_flatliner=pj["detect_non_zero_flatliner"],
     )
 
     # Custom data prep or legacy behavior

{openstef-3.4.64 → openstef-3.4.65}/openstef/pipeline/optimize_hyperparameters.py

@@ -132,7 +132,7 @@ def optimize_hyperparameters_pipeline_core(
         InputDataInsufficientError: If the input dataframe is empty.
         InputDataWrongColumnOrderError: If the load column is missing in the input dataframe.
         OldModelHigherScoreError: When old model is better than new model.
-        InputDataOngoingZeroFlatlinerError: When all recent load measurements are zero.
+        InputDataOngoingFlatlinerError: If all recent load measurements are constant.
 
     Returns:
         - Best model,
@@ -157,6 +157,7 @@ def optimize_hyperparameters_pipeline_core(
             input_data,
             pj["flatliner_threshold_minutes"],
             pj["resolution_minutes"],
+            detect_non_zero_flatliner=pj["detect_non_zero_flatliner"],
         )
     )
 

{openstef-3.4.64 → openstef-3.4.65}/openstef/pipeline/train_create_forecast_backtest.py

@@ -60,7 +60,7 @@ def train_model_and_forecast_back_test(
         InputDataInsufficientError: when input data is insufficient.
         InputDataWrongColumnOrderError: when input data has a invalid column order.
         ValueError: when the horizon is a string and the corresponding column in not in the input data
-        InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
+        InputDataOngoingFlatlinerError: If all recent load measurements are constant.
 
     """
     if pj.backtest_split_func is None:

{openstef-3.4.64 → openstef-3.4.65}/openstef/pipeline/train_model.py

@@ -177,7 +177,7 @@ def train_model_pipeline_core(
         InputDataInsufficientError: when input data is insufficient.
         InputDataWrongColumnOrderError: when input data has a invalid column order.
         OldModelHigherScoreError: When old model is better than new model.
-        InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
+        InputDataOngoingFlatlinerError: If all recent load measurements are constant.
 
     Returns:
         - Fitted_model (OpenstfRegressor)
@@ -272,7 +272,7 @@ def train_pipeline_common(
         InputDataInsufficientError: when input data is insufficient.
         InputDataWrongColumnOrderError: when input data has a invalid column order.
             'load' column should be first and 'horizon' column last.
-        InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
+        InputDataOngoingFlatlinerError: If all recent load measurements are constant.
 
     """
     data_with_features = train_pipeline_step_compute_features(
@@ -363,7 +363,7 @@ def train_pipeline_step_compute_features(
         InputDataInsufficientError: when input data is insufficient.
         InputDataWrongColumnOrderError: when input data has a invalid column order.
         ValueError: when the horizon is a string and the corresponding column in not in the input data
-        InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
+        InputDataOngoingFlatlinerError: If all recent load measurements are constant.
 
     """
     if input_data.empty:
@@ -389,6 +389,7 @@ def train_pipeline_step_compute_features(
             input_data,
             pj["flatliner_threshold_minutes"],
             pj["resolution_minutes"],
+            detect_non_zero_flatliner=pj["detect_non_zero_flatliner"],
         )
     )
     # Check if sufficient data is left after cleaning

{openstef-3.4.64 → openstef-3.4.65}/openstef/tasks/create_forecast.py

@@ -25,11 +25,11 @@ from pathlib import Path
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
 from openstef.enums import BiddingZone, ModelType, PipelineType
-from openstef.exceptions import InputDataOngoingZeroFlatlinerError
+from openstef.exceptions import InputDataOngoingFlatlinerError
 from openstef.pipeline.create_forecast import create_forecast_pipeline
 from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
 from openstef.tasks.utils.taskcontext import TaskContext
-from openstef.validation.validation import detect_ongoing_zero_flatliner
+from openstef.validation.validation import detect_ongoing_flatliner
 
 T_BEHIND_DAYS: int = 14
 
@@ -94,7 +94,7 @@ def create_forecast_task(
         forecast = create_forecast_pipeline(
             pj, input_data, mlflow_tracking_uri=mlflow_tracking_uri
         )
-    except (InputDataOngoingZeroFlatlinerError, LookupError) as e:
+    except (InputDataOngoingFlatlinerError, LookupError) as e:
         if (
             context.config.known_zero_flatliners
             and pj.id in context.config.known_zero_flatliners
@@ -103,18 +103,18 @@ def create_forecast_task(
                 "No forecasts were made for this known zero flatliner prediction job. No forecasts need to be made either, since the fallback forecasts are sufficient."
             )
             return
-        elif isinstance(e, InputDataOngoingZeroFlatlinerError):
-            raise InputDataOngoingZeroFlatlinerError(
-                'All recent load measurements are zero. Check the load profile of this pid as well as related/neighbouring prediction jobs. Afterwards, consider adding this pid to the "known_zero_flatliners" app_setting and possibly removing other pids from the same app_setting.'
+        elif isinstance(e, InputDataOngoingFlatlinerError):
+            raise InputDataOngoingFlatlinerError(
+                'All recent load measurements are constant. Check the load profile of this pid as well as related/neighbouring prediction jobs. Afterwards, consider adding this pid to the "known_zero_flatliners" app_setting and possibly removing other pids from the same app_setting.'
             ) from e
         elif isinstance(e, LookupError):
-            zero_flatliner_ongoing = detect_ongoing_zero_flatliner(
+            zero_flatliner_ongoing = detect_ongoing_flatliner(
                 load=input_data.iloc[:, 0],
                 duration_threshold_minutes=pj.flatliner_threshold_minutes,
             )
             if zero_flatliner_ongoing:
                 raise LookupError(
-                    'Model not found. Consider checking for a zero flatliner and adding this pid to the "known_zero_flatliners" app_setting. For zero flatliners, no model can be trained.'
+                    'Model not found. Consider checking for a flatliner and adding this pid to the "known_zero_flatliners" app_setting. For flatliners, no model can be trained.'
                 ) from e
             else:
                 raise e

{openstef-3.4.64 → openstef-3.4.65}/openstef/tasks/train_model.py

@@ -27,7 +27,7 @@ import pandas as pd
 from openstef.data_classes.prediction_job import PredictionJobDataClass
 from openstef.enums import ModelType, PipelineType
 from openstef.exceptions import (
-    InputDataOngoingZeroFlatlinerError,
+    InputDataOngoingFlatlinerError,
     SkipSaveTrainingForecasts,
 )
 from openstef.model.serializer import MLflowSerializer
@@ -67,7 +67,7 @@ def train_model_task(
 
     Raises:
         SkipSaveTrainingForecasts: If old model is better or too young, you don't need to save the traing forcast.
-        InputDataOngoingZeroFlatlinerError: If all recent load measurements are zero.
+        InputDataOngoingFlatlinerError: If all recent load measurements are constant.
 
     """
     # Check pipeline types
@@ -187,18 +187,18 @@ def train_model_task(
             context.logger.debug("Saved Forecasts from trained model on datasets")
     except SkipSaveTrainingForecasts:
         context.logger.debug("Skip saving forecasts")
-    except InputDataOngoingZeroFlatlinerError:
+    except InputDataOngoingFlatlinerError:
         if (
             context.config.known_zero_flatliners
             and pj.id in context.config.known_zero_flatliners
         ):
             context.logger.info(
-                "No model was trained for this known zero flatliner. No model needs to be trained either, since the fallback forecasts are sufficient."
+                "No model was trained for this known flatliner. No model needs to be trained either, since the fallback forecasts are sufficient."
             )
             return
         else:
-            raise InputDataOngoingZeroFlatlinerError(
-                'All recent load measurements are zero. Check the load profile of this pid as well as related/neighbouring prediction jobs. Afterwards, consider adding this pid to the "known_zero_flatliners" app_setting and possibly removing other pids from the same app_setting.'
+            raise InputDataOngoingFlatlinerError(
+                'All recent load measurements are constant. Check the load profile of this pid as well as related/neighbouring prediction jobs. Afterwards, consider adding this pid to the "known_zero_flatliners" app_setting and possibly removing other pids from the same app_setting.'
             )
 
 

{openstef-3.4.64 → openstef-3.4.65}/openstef/validation/validation.py

@@ -10,7 +10,8 @@ import numpy as np
 import pandas as pd
 import structlog
 
-from openstef.exceptions import InputDataOngoingZeroFlatlinerError
+from openstef.data_classes.prediction_job import PredictionJobDataClass
+from openstef.exceptions import InputDataOngoingFlatlinerError
 from openstef.model.regressors.regressor import OpenstfRegressor
 from openstef.preprocessing.preprocessing import replace_repeated_values_with_nan
 from openstef.settings import Settings
@@ -21,12 +22,15 @@ def validate(
     data: pd.DataFrame,
     flatliner_threshold_minutes: Union[int, None],
     resolution_minutes: int,
+    *,
+    detect_non_zero_flatliner: bool = False,
 ) -> pd.DataFrame:
     """Validate prediction job and timeseries data.
 
     Steps:
     1. Check if input dataframe has a datetime index.
-    1. Check if a zero flatliner pattern is ongoing (i.e. all recent measurements are zero).
+    1. Check if a flatliner pattern is ongoing (i.e. all recent measurements are constant,
+        0 in case detect_non_zero_flatliner = True).
     2. Replace repeated values for longer than flatliner_threshold_minutes with NaN.
 
     Args:
@@ -35,12 +39,14 @@ def validate(
         flatliner_threshold_minutes: int indicating the number of minutes after which constant load is considered a flatline.
             if None, the validation is effectively skipped
         resolution_minutes: The forecasting resolution in minutes.
+        detect_non_zero_flatliner: If True, a flatliner is detected for non-zero values. If False,
+            a flatliner is detected for zero values only.
 
     Returns:
         Dataframe where repeated values are set to None
 
     Raises:
-        InputDataOngoingZeroFlatlinerError: If all recent load measurements are zero.
+        InputDataOngoingFlatlinerError: If all recent load measurements are constant.
 
     """
     structlog.configure(
@@ -57,13 +63,15 @@ def validate(
         logger.info("Skipping validation of input data", pj_id=pj_id)
         return data
 
-    zero_flatliner_ongoing = detect_ongoing_zero_flatliner(
-        load=data.iloc[:, 0], duration_threshold_minutes=flatliner_threshold_minutes
+    flatliner_ongoing = detect_ongoing_flatliner(
+        load=data.iloc[:, 0],
+        duration_threshold_minutes=flatliner_threshold_minutes,
+        detect_non_zero_flatliner=detect_non_zero_flatliner,
     )
 
-    if zero_flatliner_ongoing:
-        raise InputDataOngoingZeroFlatlinerError(
-            "All recent load measurements are zero."
+    if flatliner_ongoing:
+        raise InputDataOngoingFlatlinerError(
+            "All recent load measurements are constant."
         )
 
     flatliner_threshold_repetitions = math.ceil(
@@ -228,18 +236,22 @@ def calc_completeness_features(
     return completeness
 
 
-def detect_ongoing_zero_flatliner(
+def detect_ongoing_flatliner(
     load: pd.Series,
     duration_threshold_minutes: int,
+    *,
+    detect_non_zero_flatliner: bool = False,
 ) -> bool:
-    """Detects if the latest measurements follow a zero flatliner pattern.
+    """Detects if the latest measurements follow a flatliner pattern.
 
     Args:
         load (pd.Series): A timeseries of measured load with a datetime index.
-        duration_threshold_minutes (int): A zero flatliner is only detected if it exceeds the threshold duration.
+        duration_threshold_minutes (int): A flatliner is only detected if it exceeds the threshold duration.
+        detect_non_zero_flatliner (bool): If True, a flatliner is detected for non-zero values. If False,
+            a flatliner is detected for zero values only.
 
     Returns:
-        bool: Indicating whether or not there is a zero flatliner ongoing for the given load.
+        bool: Indicating whether or not there is a flatliner ongoing for the given load.
 
     """
     # remove all timestamps in the future
@@ -249,7 +261,18 @@ def detect_ongoing_zero_flatliner(
         latest_measurement_time - timedelta(minutes=duration_threshold_minutes) :
     ].dropna()
 
-    return (latest_measurements == 0).all() & (not latest_measurements.empty)
+    flatliner_value = latest_measurements.median() if detect_non_zero_flatliner else 0
+
+    # check if all values are within a relative tolerance of each other
+    flatline_condition = np.isclose(
+        latest_measurements,
+        flatliner_value,
+        atol=0,
+        rtol=1e-5,
+    ).all()
+    non_empty_condition = not latest_measurements.empty
+
+    return flatline_condition & non_empty_condition
 
 
 def calc_completeness_dataframe(

{openstef-3.4.64 → openstef-3.4.65}/openstef.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: openstef
-Version: 3.4.64
+Version: 3.4.65
 Summary: Open short term energy forecaster
 Home-page: https://github.com/OpenSTEF/openstef
 Author: Alliander N.V

{openstef-3.4.64 → openstef-3.4.65}/setup.py

@@ -33,7 +33,7 @@ def read_long_description_from_readme():
 
 setup(
     name="openstef",
-    version="3.4.64",
+    version="3.4.65",
     packages=find_packages(include=["openstef", "openstef.*"]),
     description="Open short term energy forecaster",
     long_description=read_long_description_from_readme(),