openstef 3.4.10__py3-none-any.whl → 3.4.44__py3-none-any.whl

openstef/tasks/calculate_kpi.py

@@ -18,6 +18,8 @@ Example:
         $ python calculate_kpi.py
 
 """
+import logging
+
 # Import builtins
 from datetime import datetime, timedelta
 from pathlib import Path
@@ -27,9 +29,10 @@ import pandas as pd
 import structlog
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import MLModelType
+from openstef.enums import ModelType
 from openstef.exceptions import NoPredictedLoadError, NoRealisedLoadError
 from openstef.metrics import metrics
+from openstef.settings import Settings
 from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
 from openstef.tasks.utils.taskcontext import TaskContext
 from openstef.validation import validation
@@ -39,7 +42,7 @@ THRESHOLD_RETRAINING = 0.25
 THRESHOLD_OPTIMIZING = 0.50
 
 
-def main(model_type: MLModelType = None, config=None, database=None) -> None:
+def main(model_type: ModelType = None, config=None, database=None) -> None:
     taskname = Path(__file__).name.replace(".py", "")
 
     if database is None or config is None:
@@ -49,7 +52,7 @@ def main(model_type: MLModelType = None, config=None, database=None) -> None:
         )
 
     if model_type is None:
-        model_type = [ml.value for ml in MLModelType]
+        model_type = [ml.value for ml in ModelType]
 
     with TaskContext(taskname, config, database) as context:
         # Set start and end time
@@ -69,6 +72,8 @@ def check_kpi_task(
     context: TaskContext,
     start_time: datetime,
     end_time: datetime,
+    threshold_optimizing=THRESHOLD_OPTIMIZING,
+    threshold_retraining=THRESHOLD_RETRAINING,
 ) -> None:
     # Apply default parameters if none are provided
     if start_time is None:
@@ -99,28 +104,21 @@ def check_kpi_task(
 
     # Add pid to the list of pids that should be retrained or optimized if
     # performance is insufficient
-    if kpis["47.0h"]["rMAE"] > THRESHOLD_RETRAINING:
+    if kpis["47.0h"]["rMAE"] > threshold_retraining:
         context.logger.warning(
             "Need to retrain model, retraining threshold rMAE 47h exceeded",
             t_ahead="47.0h",
             rMAE=kpis["47.0h"]["rMAE"],
-            retraining_threshold=THRESHOLD_RETRAINING,
+            retraining_threshold=threshold_retraining,
         )
-        function_name = "train_model"
-
-        context.logger.info("Adding tracy job", function=function_name)
-        context.database.ktp_api.add_tracy_job(pj["id"], function=function_name)
 
-    if kpis["47.0h"]["rMAE"] > THRESHOLD_OPTIMIZING:
+    if kpis["47.0h"]["rMAE"] > threshold_optimizing:
         context.logger.warning(
             "Need to optimize hyperparameters, optimizing threshold rMAE 47h exceeded",
             t_ahead="47.0h",
             rMAE=kpis["47.0h"]["rMAE"],
-            optimizing_threshold=THRESHOLD_OPTIMIZING,
+            optimizing_threshold=threshold_optimizing,
         )
-        function_name = "optimize_hyperparameters"
-        context.logger.info("Adding tracy job", function=function_name)
-        context.database.ktp_api.add_tracy_job(pj["id"], function=function_name)
 
 
 def calc_kpi_for_specific_pid(
@@ -160,7 +158,12 @@ def calc_kpi_for_specific_pid(
     COMPLETENESS_REALISED_THRESHOLDS = 0.7
     COMPLETENESS_PREDICTED_LOAD_THRESHOLD = 0.7
 
-    log = structlog.get_logger(__name__)
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
+    logger = structlog.get_logger(__name__)
 
     # If predicted is empty
     if len(predicted_load) == 0:
@@ -194,9 +197,9 @@ def calc_kpi_for_specific_pid(
 
     # Raise exception in case of constant load
     if combined.load.nunique() == 1:
-        structlog.get_logger(__name__).warning(
+        logger.warning(
             "The load is constant! KPIs will still be calculated, but relative metrics"
-            " will be nan"
+            " will be nan."
         )
 
     # Define output dictonary
@@ -213,7 +216,7 @@ def calc_kpi_for_specific_pid(
     date = pd.to_datetime(end_time)
 
     # Calculate model metrics and add them to the output dictionary
-    log.info("Start calculating kpis")
+    logger.info("Start calculating kpis")
     for hor_cols in hor_list:
         t_ahead_h = hor_cols[0].split("_")[1]
         fc = combined[hor_cols[0]]  # load predictions
@@ -272,7 +275,7 @@ def calc_kpi_for_specific_pid(
         )
 
         if completeness_realised < COMPLETENESS_REALISED_THRESHOLDS:
-            log.warning(
+            logger.warning(
                 "Completeness realised load too low",
                 prediction_id=pid,
                 start_time=start_time,
@@ -282,7 +285,7 @@ def calc_kpi_for_specific_pid(
             )
             set_incomplete_kpi_to_nan(kpis, t_ahead_h)
         if completeness_predicted_load.any() < COMPLETENESS_PREDICTED_LOAD_THRESHOLD:
-            log.warning(
+            logger.warning(
                 "Completeness predicted load of specific horizon too low",
                 prediction_id=pid,
                 horizon=t_ahead_h,

openstef/tasks/create_basecase_forecast.py

@@ -32,7 +32,10 @@ T_AHEAD_DAYS: int = 14
 
 
 def create_basecase_forecast_task(
-    pj: PredictionJobDataClass, context: TaskContext
+    pj: PredictionJobDataClass,
+    context: TaskContext,
+    t_behind_days=T_BEHIND_DAYS,
+    t_ahead_days=T_AHEAD_DAYS,
 ) -> None:
     """Top level task that creates a basecase forecast.
 
@@ -41,6 +44,8 @@ def create_basecase_forecast_task(
     Args:
         pj: Prediction job
         context: Contect object that holds a config manager and a database connection
+        t_behind_days: number of days included as history. This is used to generated lagged features for the to-be-forecasted period
+        t_ahead_days: number of days a basecase forecast is created for
 
     """
     # Check pipeline types
@@ -63,8 +68,8 @@ def create_basecase_forecast_task(
         return
 
     # Define datetime range for input data
-    datetime_start = datetime.utcnow() - timedelta(days=T_BEHIND_DAYS)
-    datetime_end = datetime.utcnow() + timedelta(days=T_AHEAD_DAYS)
+    datetime_start = datetime.utcnow() - timedelta(days=t_behind_days)
+    datetime_end = datetime.utcnow() + timedelta(days=t_ahead_days)
 
     # Retrieve input data
     input_data = context.database.get_model_input(
@@ -77,11 +82,14 @@ def create_basecase_forecast_task(
     # Make basecase forecast using the corresponding pipeline
     basecase_forecast = create_basecase_forecast_pipeline(pj, input_data)
 
-    # Do not store basecase forecasts for moments within next 48 hours.
+    # Do not store basecase forecasts for moments within the prediction job's horizon.
     # Those should be updated by regular forecast process.
     basecase_forecast = basecase_forecast.loc[
         basecase_forecast.index
-        > (pd.to_datetime(datetime.utcnow(), utc=True) + timedelta(hours=48)),
+        > (
+            pd.to_datetime(datetime.utcnow(), utc=True)
+            + timedelta(minutes=pj.horizon_minutes)
+        ),
         :,
     ]
 
@@ -89,7 +97,7 @@ def create_basecase_forecast_task(
     context.database.write_forecast(basecase_forecast, t_ahead_series=True)
 
 
-def main(config: object = None, database: object = None):
+def main(config: object = None, database: object = None, **kwargs):
     taskname = Path(__file__).name.replace(".py", "")
 
     if database is None or config is None:
@@ -102,7 +110,7 @@ def main(config: object = None, database: object = None):
         model_type = ["xgb", "xgb_quantile", "lgb"]
 
         PredictionJobLoop(context, model_type=model_type).map(
-            create_basecase_forecast_task, context
+            create_basecase_forecast_task, context, **kwargs
         )
 
 

openstef/tasks/create_components_forecast.py

@@ -21,18 +21,20 @@ Example:
         $ python create_components_forecast.py
 
 """
+import logging
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 
-import structlog
 import pandas as pd
+import structlog
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import MLModelType
+from openstef.enums import ModelType
 from openstef.exceptions import ComponentForecastTooShortHorizonError
 from openstef.pipeline.create_component_forecast import (
     create_components_forecast_pipeline,
 )
+from openstef.settings import Settings
 from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
 from openstef.tasks.utils.taskcontext import TaskContext
 
@@ -41,7 +43,10 @@ T_AHEAD_DAYS = 3
 
 
 def create_components_forecast_task(
-    pj: PredictionJobDataClass, context: TaskContext
+    pj: PredictionJobDataClass,
+    context: TaskContext,
+    t_behind_days: int = T_BEHIND_DAYS,
+    t_ahead_days: int = T_AHEAD_DAYS,
 ) -> None:
     """Top level task that creates a components forecast.
 
@@ -50,8 +55,19 @@ def create_components_forecast_task(
     Args:
         pj: Prediction job
         context: Contect object that holds a config manager and a database connection
+        t_behind_days: number of days in the past that the component forecast is created for
+        t_ahead_days: number of days in the future that the component forecast is created for
+
+    Raises:
+        ComponentForecastTooShortHorizonError: If the forecast horizon is too short
+         (less than 30 minutes in advance)
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
     if pj["train_components"] == 0:
         context.logger.info(
@@ -60,8 +76,8 @@ def create_components_forecast_task(
         return
 
     # Define datetime range for input data
-    datetime_start = datetime.utcnow() - timedelta(days=T_BEHIND_DAYS)
-    datetime_end = datetime.utcnow() + timedelta(days=T_AHEAD_DAYS)
+    datetime_start = datetime.utcnow() - timedelta(days=t_behind_days)
+    datetime_end = datetime.utcnow() + timedelta(days=t_ahead_days)
 
     logger.info(
         "Get predicted load", datetime_start=datetime_start, datetime_end=datetime_end
@@ -124,7 +140,7 @@ def create_components_forecast_task(
         )
 
 
-def main(config: object = None, database: object = None):
+def main(config: object = None, database: object = None, **kwargs):
     taskname = Path(__file__).name.replace(".py", "")
 
     if database is None or config is None:
@@ -134,12 +150,12 @@ def main(config: object = None, database: object = None):
         )
 
     with TaskContext(taskname, config, database) as context:
-        model_type = [ml.value for ml in MLModelType]
+        model_type = [ml.value for ml in ModelType]
 
         PredictionJobLoop(
             context,
             model_type=model_type,
-        ).map(create_components_forecast_task, context)
+        ).map(create_components_forecast_task, context, **kwargs)
 
 
 if __name__ == "__main__":

openstef/tasks/create_forecast.py

@@ -24,7 +24,7 @@ from datetime import datetime, timedelta
 from pathlib import Path
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import MLModelType, PipelineType
+from openstef.enums import ModelType, PipelineType
 from openstef.exceptions import InputDataOngoingZeroFlatlinerError
 from openstef.pipeline.create_forecast import create_forecast_pipeline
 from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
@@ -34,7 +34,9 @@ from openstef.validation.validation import detect_ongoing_zero_flatliner
 T_BEHIND_DAYS: int = 14
 
 
-def create_forecast_task(pj: PredictionJobDataClass, context: TaskContext) -> None:
+def create_forecast_task(
+    pj: PredictionJobDataClass, context: TaskContext, t_behind_days: int = T_BEHIND_DAYS
+) -> None:
     """Top level task that creates a forecast.
 
     On this task level all database and context manager dependencies are resolved.
@@ -45,6 +47,7 @@ def create_forecast_task(pj: PredictionJobDataClass, context: TaskContext) -> No
     Args:
         pj: Prediction job
         context: Contect object that holds a config manager and a database connection
+        t_behind_days: number of days included as history. This is used to generated lagged features for the to-be-forecasted period
 
     """
     # Check pipeline types
@@ -70,7 +73,7 @@ def create_forecast_task(pj: PredictionJobDataClass, context: TaskContext) -> No
     mlflow_tracking_uri = context.config.paths_mlflow_tracking_uri
 
     # Define datetime range for input data
-    datetime_start = datetime.utcnow() - timedelta(days=T_BEHIND_DAYS)
+    datetime_start = datetime.utcnow() - timedelta(days=t_behind_days)
     datetime_end = datetime.utcnow() + timedelta(seconds=pj.horizon_minutes * 60)
 
     # Retrieve input data
@@ -115,7 +118,7 @@ def create_forecast_task(pj: PredictionJobDataClass, context: TaskContext) -> No
     context.database.write_forecast(forecast, t_ahead_series=True)
 
 
-def main(model_type=None, config=None, database=None):
+def main(model_type=None, config=None, database=None, **kwargs):
     taskname = Path(__file__).name.replace(".py", "")
 
     if database is None or config is None:
@@ -126,10 +129,10 @@ def main(model_type=None, config=None, database=None):
 
     with TaskContext(taskname, config, database) as context:
         if model_type is None:
-            model_type = [ml.value for ml in MLModelType]
+            model_type = [ml.value for ml in ModelType]
 
         PredictionJobLoop(context, model_type=model_type).map(
-            create_forecast_task, context
+            create_forecast_task, context, **kwargs
         )
 
 

openstef/tasks/create_solar_forecast.py

@@ -186,7 +186,7 @@ def fides(data: pd.DataFrame, all_forecasts: bool = False):
     data = pd.DataFrame(index = index,
                         data = dict(load=np.sin(index.hour/24*np.pi)*np.random.uniform(0.7,1.7, 300)))
     data['insolation'] = data.load * np.random.uniform(0.8, 1.2, len(index)) + 0.1
-    data.loc[int(len(index)/3*2):,"load"] = np.NaN
+    data.loc[int(len(index)/3*2):,"load"] = np.nan
 
     """
     insolation_forecast = apply_fit_insol(data, add_to_df=False)
@@ -216,7 +216,7 @@ def fides(data: pd.DataFrame, all_forecasts: bool = False):
     return forecast
 
 
-def main(config=None, database=None):
+def main(config=None, database=None, **kwargs):
     taskname = Path(__file__).name.replace(".py", "")
 
     if database is None or config is None:
@@ -245,7 +245,7 @@ def main(config=None, database=None):
             )
 
         PredictionJobLoop(context, prediction_jobs=prediction_jobs).map(
-            make_solar_prediction_pj, context
+            make_solar_prediction_pj, context, kwargs=kwargs
         )
 
 
@@ -357,7 +357,7 @@ def apply_fit_insol(data, add_to_df=True, hours_delta=None, polynomial=False):
     data = pd.DataFrame(index = index,
                         data = dict(load=np.sin(index.hour/24*np.pi)*np.random.uniform(0.7,1.7, len(index))))
     data['insolation'] = data.load * np.random.uniform(0.8, 1.2, len(index)) + 0.1
-    data.loc[int(len(index)/3*2):,"load"] = np.NaN
+    data.loc[int(len(index)/3*2):,"load"] = np.nan
 
     """
     colname = list(data)[0]

openstef/tasks/optimize_hyperparameters.py

@@ -20,7 +20,7 @@ from datetime import datetime, timedelta
 from pathlib import Path
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import MLModelType, PipelineType
+from openstef.enums import ModelType, PipelineType
 from openstef.model.serializer import MLflowSerializer
 from openstef.monitoring import teams
 from openstef.pipeline.optimize_hyperparameters import optimize_hyperparameters_pipeline
@@ -124,7 +124,7 @@ def main(config=None, database=None):
         )
 
     with TaskContext(taskname, config, database) as context:
-        model_type = [ml.value for ml in MLModelType]
+        model_type = [ml.value for ml in ModelType]
 
         PredictionJobLoop(context, model_type=model_type).map(
             optimize_hyperparameters_task, context

openstef/tasks/split_forecast.py

@@ -22,6 +22,7 @@ Example:
         $ python split_forecast.py
 
 """
+import logging
 from datetime import datetime
 from pathlib import Path
 
@@ -32,7 +33,8 @@ import structlog
 
 import openstef.monitoring.teams as monitoring
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import MLModelType
+from openstef.enums import ModelType
+from openstef.settings import Settings
 from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
 from openstef.tasks.utils.taskcontext import TaskContext
 
@@ -49,7 +51,7 @@ def main(config=None, database=None):
         )
 
     with TaskContext(taskname, config, database) as context:
-        model_type = [ml.value for ml in MLModelType]
+        model_type = [ml.value for ml in ModelType]
 
         PredictionJobLoop(
             context,
@@ -70,6 +72,11 @@ def split_forecast_task(
         Energy splitting coefficients.
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     logger.info("Start splitting energy", pid=pj["id"])

openstef/tasks/train_model.py

@@ -23,22 +23,20 @@ from datetime import datetime, timedelta
 from pathlib import Path
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-
-from openstef.enums import MLModelType, PipelineType
+from openstef.enums import ModelType, PipelineType
 from openstef.exceptions import (
-    SkipSaveTrainingForecasts,
     InputDataOngoingZeroFlatlinerError,
+    SkipSaveTrainingForecasts,
 )
+from openstef.model.serializer import MLflowSerializer
 from openstef.pipeline.train_model import (
+    MAXIMUM_MODEL_AGE,
     train_model_pipeline,
     train_pipeline_step_load_model,
-    MAXIMUM_MODEL_AGE,
 )
 from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
 from openstef.tasks.utils.taskcontext import TaskContext
 
-from openstef.model.serializer import MLflowSerializer
-
 TRAINING_PERIOD_DAYS: int = 120
 DEFAULT_CHECK_MODEL_AGE: bool = True
 
@@ -65,6 +63,10 @@ def train_model_task(
         datetime_start: Start
         datetime_end: End
 
+    Raises:
+        SkipSaveTrainingForecasts: If old model is better or too young, you don't need to save the traing forcast.
+        InputDataOngoingZeroFlatlinerError: If all recent load measurements are zero.
+
     """
     # Check pipeline types
     if PipelineType.TRAIN not in pj.pipelines_to_run:
@@ -177,7 +179,7 @@ def main(model_type=None, config=None, database=None):
         )
 
     if model_type is None:
-        model_type = [ml.value for ml in MLModelType]
+        model_type = [ml.value for ml in ModelType]
 
     taskname = Path(__file__).name.replace(".py", "")
     datetime_now = datetime.utcnow()

openstef/tasks/utils/taskcontext.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 import traceback
 from typing import Callable
 
@@ -9,6 +10,7 @@ import structlog
 from openstef.exceptions import PredictionJobException
 from openstef.monitoring.performance_meter import PerformanceMeter
 from openstef.monitoring.teams import post_teams
+from openstef.settings import Settings
 
 
 class TaskContext:
@@ -62,6 +64,11 @@ class TaskContext:
         self.database = database
 
     def __enter__(self):
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         self.logger = structlog.get_logger(__name__).bind(task=self.name)
 
         self.perf_meter = PerformanceMeter(self.logger)

openstef/validation/validation.py

@@ -1,17 +1,19 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
+import math
 from datetime import datetime, timedelta
 from typing import Union
 
-import math
 import numpy as np
 import pandas as pd
 import structlog
 
 from openstef.exceptions import InputDataOngoingZeroFlatlinerError
-from openstef.preprocessing.preprocessing import replace_repeated_values_with_nan
 from openstef.model.regressors.regressor import OpenstfRegressor
+from openstef.preprocessing.preprocessing import replace_repeated_values_with_nan
+from openstef.settings import Settings
 
 
 def validate(
@@ -37,7 +39,15 @@ def validate(
     Returns:
         Dataframe where repeated values are set to None
 
+    Raises:
+        InputDataOngoingZeroFlatlinerError: If all recent load measurements are zero.
+
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     if not isinstance(data.index, pd.DatetimeIndex):
@@ -81,6 +91,11 @@ def validate(
 
 
 def drop_target_na(data: pd.DataFrame) -> pd.DataFrame:
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
     len_original = len(data)
     # Remove where load is NA, NaN features are preserved
@@ -119,6 +134,11 @@ def is_data_sufficient(
     else:
         weights = model.feature_importance_dataframe
 
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
     # Set output variable
     is_sufficient = True
@@ -224,7 +244,7 @@ def detect_ongoing_zero_flatliner(
     """
     # remove all timestamps in the future
     load = load[load.index.tz_localize(None) <= datetime.utcnow()]
-    latest_measurement_time = load.index.max()
+    latest_measurement_time = load.dropna().index.max()
     latest_measurements = load[
         latest_measurement_time - timedelta(minutes=duration_threshold_minutes) :
     ].dropna()
@@ -251,6 +271,11 @@ def calc_completeness_dataframe(
         Dataframe with fraction of completeness per column
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     if homogenise and isinstance(df.index, pd.DatetimeIndex) and len(df) > 0: