openstef 3.4.10__py3-none-any.whl → 3.4.29__py3-none-any.whl

openstef/pipeline/train_model.py

@@ -23,6 +23,7 @@ from openstef.model.regressors.regressor import OpenstfRegressor
 from openstef.model.serializer import MLflowSerializer
 from openstef.model.standard_deviation_generator import StandardDeviationGenerator
 from openstef.model_selection.model_selection import split_data_train_validation_test
+from openstef.settings import Settings
 from openstef.validation import validation
 
 DEFAULT_TRAIN_HORIZONS_HOURS: list[float] = [0.25, 47.0]
@@ -31,6 +32,11 @@ MAXIMUM_MODEL_AGE: int = 7
 DEFAULT_EARLY_STOPPING_ROUNDS: int = 10
 PENALTY_FACTOR_OLD_MODEL: float = 1.2
 
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(
+        logging.getLevelName(Settings.log_level)
+    )
+)
 logger = structlog.get_logger(__name__)
 
 
@@ -60,6 +66,13 @@ def train_model_pipeline(
             - The validation dataset with forecasts
             - The test dataset with forecasts
 
+    Raises:
+        InputDataInsufficientError: when input data is insufficient.
+        InputDataWrongColumnOrderError: when input data has a invalid column order.
+            'load' column should be first and 'horizon' column last.
+        OldModelHigherScoreError: When old model is better than new model.
+        SkipSaveTrainingForecasts: If old model is better or younger than `MAXIMUM_MODEL_AGE`, the model is not saved.
+
     """
     # Initialize serializer
     serializer = MLflowSerializer(mlflow_tracking_uri=mlflow_tracking_uri)
@@ -164,6 +177,7 @@ def train_model_pipeline_core(
         InputDataInsufficientError: when input data is insufficient.
         InputDataWrongColumnOrderError: when input data has a invalid column order.
         OldModelHigherScoreError: When old model is better than new model.
+        InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
 
     Returns:
         - Fitted_model (OpenstfRegressor)
@@ -172,8 +186,6 @@ def train_model_pipeline_core(
         - Datasets (tuple[pd.DataFrmae, pd.DataFrame, pd.Dataframe): The train, validation and test sets
 
     """
-    logger = structlog.get_logger(__name__)
-
     # Call common pipeline
     (
         model,
@@ -257,6 +269,8 @@ def train_pipeline_common(
     Raises:
         InputDataInsufficientError: when input data is insufficient.
         InputDataWrongColumnOrderError: when input data has a invalid column order.
+            'load' column should be first and 'horizon' column last.
+        InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
 
     """
     data_with_features = train_pipeline_step_compute_features(
@@ -346,12 +360,9 @@ def train_pipeline_step_compute_features(
         InputDataInsufficientError: when input data is insufficient.
         InputDataWrongColumnOrderError: when input data has a invalid column order.
         ValueError: when the horizon is a string and the corresponding column in not in the input data
+        InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
 
     """
-    if pj["model"] == "proloaf":
-        # proloaf is only able to train with one horizon
-        horizons = [horizons[0]]
-
     if input_data.empty:
         raise InputDataInsufficientError("Input dataframe is empty")
     elif "load" not in input_data.columns:
@@ -423,6 +434,10 @@ def train_pipeline_step_train_model(
     Returns:
         The trained model
 
+    Raises:
+        NotImplementedError: When using invalid model type in the prediction job.
+        InputDataWrongColumnOrderError: When 'load' column is not first and 'horizon' column is not last.
+
     """
     # Test if first column is "load" and last column is "horizon"
     if train_data.columns[0] != "load" or train_data.columns[-1] != "horizon":
@@ -435,6 +450,7 @@ def train_pipeline_step_train_model(
     model = ModelCreator.create_model(
         pj["model"],
         quantiles=pj["quantiles"],
+        **(pj.model_kwargs or {}),
     )
 
     # split x and y data
@@ -523,7 +539,7 @@ def train_pipeline_step_split_data(
     if pj.train_split_func is None:
         split_func = split_data_train_validation_test
         split_args = {
-            "stratification_min_max": pj["model"] != "proloaf",
+            "stratification_min_max": True,
             "back_test": backtest,
         }
     else:

openstef/pipeline/utils.py

@@ -27,6 +27,9 @@ def generate_forecast_datetime_range(
     Returns:
         Start and end datetimes of the forecast range.
 
+    Raises:
+        ValueError: If the target column does not have null values.
+
     """
     # By labeling the True/False values (based on the isnull() statement) as clusters,
     # we find what True value belongs to what cluster and the amount of True clusters.

openstef/postprocessing/postprocessing.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from enum import Enum
 
 import numpy as np
@@ -10,6 +11,7 @@ import structlog
 from openstef.data_classes.prediction_job import PredictionJobDataClass
 from openstef.enums import ForecastType
 from openstef.feature_engineering import weather_features
+from openstef.settings import Settings
 
 # this is the default for "Lagerwey100"
 TURBINE_DATA = {
@@ -219,6 +221,11 @@ def add_prediction_job_properties_to_forecast(
         Dataframe with added metadata.
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     logger.info("Postproces in preparation of storing")
@@ -244,3 +251,25 @@ def add_prediction_job_properties_to_forecast(
     forecast["algtype"] = algorithm_type
 
     return forecast
+
+
+def sort_quantiles(
+    forecast: pd.DataFrame, quantile_col_start="quantile_P"
+) -> pd.DataFrame:
+    """Sort quantile values so quantiles do not cross.
+
+    This function assumes that all quantile columns start with 'quantile_P' For more academic details on why this is
+    mathematically sounds, please refer to Quantile and Probability Curves Without Crossing (Chernozhukov, 2010)
+
+    """
+    p_columns = [col for col in forecast.columns if col.startswith(quantile_col_start)]
+
+    if len(p_columns) == 0:
+        return forecast
+
+    # sort the columns
+    p_columns = np.sort(p_columns)
+
+    forecast.loc[:, p_columns] = forecast[p_columns].apply(sorted, axis=1).to_list()
+
+    return forecast

openstef/settings.py

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+from functools import lru_cache
+
+from openstef.app_settings import AppSettings
+
+
+@lru_cache
+def _get_app_settings() -> AppSettings:
+    return AppSettings()
+
+
+Settings = _get_app_settings()

openstef/tasks/calculate_kpi.py

@@ -18,6 +18,8 @@ Example:
         $ python calculate_kpi.py
 
 """
+import logging
+
 # Import builtins
 from datetime import datetime, timedelta
 from pathlib import Path
@@ -30,6 +32,7 @@ from openstef.data_classes.prediction_job import PredictionJobDataClass
 from openstef.enums import MLModelType
 from openstef.exceptions import NoPredictedLoadError, NoRealisedLoadError
 from openstef.metrics import metrics
+from openstef.settings import Settings
 from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
 from openstef.tasks.utils.taskcontext import TaskContext
 from openstef.validation import validation
@@ -69,6 +72,8 @@ def check_kpi_task(
     context: TaskContext,
     start_time: datetime,
     end_time: datetime,
+    threshold_optimizing=THRESHOLD_OPTIMIZING,
+    threshold_retraining=THRESHOLD_RETRAINING,
 ) -> None:
     # Apply default parameters if none are provided
     if start_time is None:
@@ -99,28 +104,21 @@ def check_kpi_task(
 
     # Add pid to the list of pids that should be retrained or optimized if
     # performance is insufficient
-    if kpis["47.0h"]["rMAE"] > THRESHOLD_RETRAINING:
+    if kpis["47.0h"]["rMAE"] > threshold_retraining:
         context.logger.warning(
             "Need to retrain model, retraining threshold rMAE 47h exceeded",
             t_ahead="47.0h",
             rMAE=kpis["47.0h"]["rMAE"],
-            retraining_threshold=THRESHOLD_RETRAINING,
+            retraining_threshold=threshold_retraining,
         )
-        function_name = "train_model"
-
-        context.logger.info("Adding tracy job", function=function_name)
-        context.database.ktp_api.add_tracy_job(pj["id"], function=function_name)
 
-    if kpis["47.0h"]["rMAE"] > THRESHOLD_OPTIMIZING:
+    if kpis["47.0h"]["rMAE"] > threshold_optimizing:
         context.logger.warning(
             "Need to optimize hyperparameters, optimizing threshold rMAE 47h exceeded",
             t_ahead="47.0h",
             rMAE=kpis["47.0h"]["rMAE"],
-            optimizing_threshold=THRESHOLD_OPTIMIZING,
+            optimizing_threshold=threshold_optimizing,
         )
-        function_name = "optimize_hyperparameters"
-        context.logger.info("Adding tracy job", function=function_name)
-        context.database.ktp_api.add_tracy_job(pj["id"], function=function_name)
 
 
 def calc_kpi_for_specific_pid(
@@ -160,7 +158,12 @@ def calc_kpi_for_specific_pid(
     COMPLETENESS_REALISED_THRESHOLDS = 0.7
     COMPLETENESS_PREDICTED_LOAD_THRESHOLD = 0.7
 
-    log = structlog.get_logger(__name__)
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
+    logger = structlog.get_logger(__name__)
 
     # If predicted is empty
     if len(predicted_load) == 0:
@@ -194,9 +197,9 @@ def calc_kpi_for_specific_pid(
 
     # Raise exception in case of constant load
     if combined.load.nunique() == 1:
-        structlog.get_logger(__name__).warning(
+        logger.warning(
             "The load is constant! KPIs will still be calculated, but relative metrics"
-            " will be nan"
+            " will be nan."
         )
 
     # Define output dictonary
@@ -213,7 +216,7 @@ def calc_kpi_for_specific_pid(
     date = pd.to_datetime(end_time)
 
     # Calculate model metrics and add them to the output dictionary
-    log.info("Start calculating kpis")
+    logger.info("Start calculating kpis")
     for hor_cols in hor_list:
         t_ahead_h = hor_cols[0].split("_")[1]
         fc = combined[hor_cols[0]]  # load predictions
@@ -272,7 +275,7 @@ def calc_kpi_for_specific_pid(
         )
 
         if completeness_realised < COMPLETENESS_REALISED_THRESHOLDS:
-            log.warning(
+            logger.warning(
                 "Completeness realised load too low",
                 prediction_id=pid,
                 start_time=start_time,
@@ -282,7 +285,7 @@ def calc_kpi_for_specific_pid(
             )
             set_incomplete_kpi_to_nan(kpis, t_ahead_h)
         if completeness_predicted_load.any() < COMPLETENESS_PREDICTED_LOAD_THRESHOLD:
-            log.warning(
+            logger.warning(
                 "Completeness predicted load of specific horizon too low",
                 prediction_id=pid,
                 horizon=t_ahead_h,

openstef/tasks/create_basecase_forecast.py

@@ -32,7 +32,10 @@ T_AHEAD_DAYS: int = 14
 
 
 def create_basecase_forecast_task(
-    pj: PredictionJobDataClass, context: TaskContext
+    pj: PredictionJobDataClass,
+    context: TaskContext,
+    t_behind_days=T_BEHIND_DAYS,
+    t_ahead_days=T_AHEAD_DAYS,
 ) -> None:
     """Top level task that creates a basecase forecast.
 
@@ -41,6 +44,8 @@ def create_basecase_forecast_task(
     Args:
         pj: Prediction job
         context: Contect object that holds a config manager and a database connection
+        t_behind_days: number of days included as history. This is used to generated lagged features for the to-be-forecasted period
+        t_ahead_days: number of days a basecase forecast is created for
 
     """
     # Check pipeline types
@@ -63,8 +68,8 @@ def create_basecase_forecast_task(
         return
 
     # Define datetime range for input data
-    datetime_start = datetime.utcnow() - timedelta(days=T_BEHIND_DAYS)
-    datetime_end = datetime.utcnow() + timedelta(days=T_AHEAD_DAYS)
+    datetime_start = datetime.utcnow() - timedelta(days=t_behind_days)
+    datetime_end = datetime.utcnow() + timedelta(days=t_ahead_days)
 
     # Retrieve input data
     input_data = context.database.get_model_input(
@@ -77,11 +82,14 @@ def create_basecase_forecast_task(
     # Make basecase forecast using the corresponding pipeline
     basecase_forecast = create_basecase_forecast_pipeline(pj, input_data)
 
-    # Do not store basecase forecasts for moments within next 48 hours.
+    # Do not store basecase forecasts for moments within the prediction job's horizon.
     # Those should be updated by regular forecast process.
     basecase_forecast = basecase_forecast.loc[
         basecase_forecast.index
-        > (pd.to_datetime(datetime.utcnow(), utc=True) + timedelta(hours=48)),
+        > (
+            pd.to_datetime(datetime.utcnow(), utc=True)
+            + timedelta(minutes=pj.horizon_minutes)
+        ),
         :,
     ]
 

openstef/tasks/create_components_forecast.py

@@ -21,11 +21,12 @@ Example:
         $ python create_components_forecast.py
 
 """
+import logging
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 
-import structlog
 import pandas as pd
+import structlog
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
 from openstef.enums import MLModelType
@@ -33,6 +34,7 @@ from openstef.exceptions import ComponentForecastTooShortHorizonError
 from openstef.pipeline.create_component_forecast import (
     create_components_forecast_pipeline,
 )
+from openstef.settings import Settings
 from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
 from openstef.tasks.utils.taskcontext import TaskContext
 
@@ -41,7 +43,10 @@ T_AHEAD_DAYS = 3
 
 
 def create_components_forecast_task(
-    pj: PredictionJobDataClass, context: TaskContext
+    pj: PredictionJobDataClass,
+    context: TaskContext,
+    t_behind_days: int = T_BEHIND_DAYS,
+    t_ahead_days: int = T_AHEAD_DAYS,
 ) -> None:
     """Top level task that creates a components forecast.
 
@@ -50,8 +55,19 @@ def create_components_forecast_task(
     Args:
         pj: Prediction job
         context: Contect object that holds a config manager and a database connection
+        t_behind_days: number of days in the past that the component forecast is created for
+        t_ahead_days: number of days in the future that the component forecast is created for
+
+    Raises:
+        ComponentForecastTooShortHorizonError: If the forecast horizon is too short
+         (less than 30 minutes in advance)
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
     if pj["train_components"] == 0:
         context.logger.info(
@@ -60,8 +76,8 @@ def create_components_forecast_task(
         return
 
     # Define datetime range for input data
-    datetime_start = datetime.utcnow() - timedelta(days=T_BEHIND_DAYS)
-    datetime_end = datetime.utcnow() + timedelta(days=T_AHEAD_DAYS)
+    datetime_start = datetime.utcnow() - timedelta(days=t_behind_days)
+    datetime_end = datetime.utcnow() + timedelta(days=t_ahead_days)
 
     logger.info(
         "Get predicted load", datetime_start=datetime_start, datetime_end=datetime_end

openstef/tasks/create_forecast.py

@@ -34,7 +34,9 @@ from openstef.validation.validation import detect_ongoing_zero_flatliner
 T_BEHIND_DAYS: int = 14
 
 
-def create_forecast_task(pj: PredictionJobDataClass, context: TaskContext) -> None:
+def create_forecast_task(
+    pj: PredictionJobDataClass, context: TaskContext, t_behind_days: int = T_BEHIND_DAYS
+) -> None:
     """Top level task that creates a forecast.
 
     On this task level all database and context manager dependencies are resolved.
@@ -45,6 +47,7 @@ def create_forecast_task(pj: PredictionJobDataClass, context: TaskContext) -> No
     Args:
         pj: Prediction job
         context: Contect object that holds a config manager and a database connection
+        t_behind_days: number of days included as history. This is used to generated lagged features for the to-be-forecasted period
 
     """
     # Check pipeline types
@@ -70,7 +73,7 @@ def create_forecast_task(pj: PredictionJobDataClass, context: TaskContext) -> No
     mlflow_tracking_uri = context.config.paths_mlflow_tracking_uri
 
     # Define datetime range for input data
-    datetime_start = datetime.utcnow() - timedelta(days=T_BEHIND_DAYS)
+    datetime_start = datetime.utcnow() - timedelta(days=t_behind_days)
     datetime_end = datetime.utcnow() + timedelta(seconds=pj.horizon_minutes * 60)
 
     # Retrieve input data

openstef/tasks/split_forecast.py

@@ -22,6 +22,7 @@ Example:
         $ python split_forecast.py
 
 """
+import logging
 from datetime import datetime
 from pathlib import Path
 
@@ -33,6 +34,7 @@ import structlog
 import openstef.monitoring.teams as monitoring
 from openstef.data_classes.prediction_job import PredictionJobDataClass
 from openstef.enums import MLModelType
+from openstef.settings import Settings
 from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
 from openstef.tasks.utils.taskcontext import TaskContext
 
@@ -70,6 +72,11 @@ def split_forecast_task(
         Energy splitting coefficients.
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     logger.info("Start splitting energy", pid=pj["id"])

openstef/tasks/train_model.py

@@ -23,22 +23,20 @@ from datetime import datetime, timedelta
 from pathlib import Path
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-
 from openstef.enums import MLModelType, PipelineType
 from openstef.exceptions import (
-    SkipSaveTrainingForecasts,
     InputDataOngoingZeroFlatlinerError,
+    SkipSaveTrainingForecasts,
 )
+from openstef.model.serializer import MLflowSerializer
 from openstef.pipeline.train_model import (
+    MAXIMUM_MODEL_AGE,
     train_model_pipeline,
     train_pipeline_step_load_model,
-    MAXIMUM_MODEL_AGE,
 )
 from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
 from openstef.tasks.utils.taskcontext import TaskContext
 
-from openstef.model.serializer import MLflowSerializer
-
 TRAINING_PERIOD_DAYS: int = 120
 DEFAULT_CHECK_MODEL_AGE: bool = True
 
@@ -65,6 +63,10 @@ def train_model_task(
         datetime_start: Start
         datetime_end: End
 
+    Raises:
+        SkipSaveTrainingForecasts: If old model is better or too young, you don't need to save the traing forcast.
+        InputDataOngoingZeroFlatlinerError: If all recent load measurements are zero.
+
     """
     # Check pipeline types
     if PipelineType.TRAIN not in pj.pipelines_to_run:

openstef/tasks/utils/taskcontext.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 import traceback
 from typing import Callable
 
@@ -9,6 +10,7 @@ import structlog
 from openstef.exceptions import PredictionJobException
 from openstef.monitoring.performance_meter import PerformanceMeter
 from openstef.monitoring.teams import post_teams
+from openstef.settings import Settings
 
 
 class TaskContext:
@@ -62,6 +64,11 @@ class TaskContext:
         self.database = database
 
     def __enter__(self):
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         self.logger = structlog.get_logger(__name__).bind(task=self.name)
 
         self.perf_meter = PerformanceMeter(self.logger)

openstef/validation/validation.py

@@ -1,17 +1,19 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
+import math
 from datetime import datetime, timedelta
 from typing import Union
 
-import math
 import numpy as np
 import pandas as pd
 import structlog
 
 from openstef.exceptions import InputDataOngoingZeroFlatlinerError
-from openstef.preprocessing.preprocessing import replace_repeated_values_with_nan
 from openstef.model.regressors.regressor import OpenstfRegressor
+from openstef.preprocessing.preprocessing import replace_repeated_values_with_nan
+from openstef.settings import Settings
 
 
 def validate(
@@ -37,7 +39,15 @@ def validate(
     Returns:
         Dataframe where repeated values are set to None
 
+    Raises:
+        InputDataOngoingZeroFlatlinerError: If all recent load measurements are zero.
+
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     if not isinstance(data.index, pd.DatetimeIndex):
@@ -81,6 +91,11 @@ def validate(
 
 
 def drop_target_na(data: pd.DataFrame) -> pd.DataFrame:
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
     len_original = len(data)
     # Remove where load is NA, NaN features are preserved
@@ -119,6 +134,11 @@ def is_data_sufficient(
     else:
         weights = model.feature_importance_dataframe
 
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
     # Set output variable
     is_sufficient = True
@@ -251,6 +271,11 @@ def calc_completeness_dataframe(
         Dataframe with fraction of completeness per column
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     if homogenise and isinstance(df.index, pd.DatetimeIndex) and len(df) > 0: