openstef 3.4.65__py3-none-any.whl → 3.4.66__py3-none-any.whl

openstef/pipeline/train_model.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: MPL-2.0
 import logging
 import os
-from typing import Optional, Union, Tuple
+from typing import Optional, Tuple, Union
 
 import pandas as pd
 import structlog
@@ -46,6 +46,7 @@ def train_model_pipeline(
     check_old_model_age: bool,
     mlflow_tracking_uri: str,
     artifact_folder: str,
+    ignore_existing_models: bool = False,
 ) -> Optional[tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]]:
     """Middle level pipeline that takes care of all persistent storage dependencies.
 
@@ -79,7 +80,7 @@ def train_model_pipeline(
 
     # Get old model and age
     old_model, model_specs, old_model_age = train_pipeline_step_load_model(
-        pj, serializer
+        pj, serializer, ignore_existing_models
     )
 
     # Check old model age and continue yes/no
@@ -106,6 +107,7 @@ def train_model_pipeline(
             input_data,
             old_model,
             horizons=horizons,
+            ignore_existing_models=ignore_existing_models,
         )
     except OldModelHigherScoreError as OMHSE:
         logger.error("Old model is better than new model", pid=pj["id"], exc_info=OMHSE)
@@ -155,6 +157,7 @@ def train_model_pipeline_core(
     input_data: pd.DataFrame,
     old_model: OpenstfRegressor = None,
     horizons: list[float] = DEFAULT_TRAIN_HORIZONS_HOURS,
+    ignore_existing_models: bool = False,
 ) -> Tuple[
     OpenstfRegressor,
     Report,
@@ -203,7 +206,7 @@ def train_model_pipeline_core(
     model_specs.feature_names = list(train_data.columns)
 
     # Check if new model is better than old model
-    if old_model:
+    if old_model and not ignore_existing_models:
         combined = pd.concat([train_data, validation_data])
         # skip the forecast column added at the end of dataframes
         if pj.save_train_forecasts:
@@ -220,6 +223,7 @@ def train_model_pipeline_core(
         # Try to compare new model to old model.
         # If this does not success, for example since the feature names of the
         # old model differ from the new model, the new model is considered better
+
         try:
             score_old_model = old_model.score(x_data, y_data)
 
@@ -315,25 +319,31 @@ def train_pipeline_common(
 
 
 def train_pipeline_step_load_model(
-    pj: PredictionJobDataClass, serializer: MLflowSerializer
+    pj: PredictionJobDataClass,
+    serializer: MLflowSerializer,
+    ignore_existing_models: bool = False,
 ) -> Tuple[OpenstfRegressor, ModelSpecificationDataClass, Union[int, float]]:
     old_model: Optional[OpenstfRegressor]
-    try:
-        old_model, model_specs = serializer.load_model(experiment_name=str(pj.id))
-        old_model_age = old_model.age  # Age attribute is openstef specific
-        return old_model, model_specs, old_model_age
-    except (AttributeError, FileNotFoundError, LookupError):
-        logger.warning("No old model found, training new model", pid=pj.id)
-    except Exception:
-        logger.exception("Old model could not be loaded, training new model", pid=pj.id)
+
+    if not ignore_existing_models:
+        try:
+            old_model, model_specs = serializer.load_model(experiment_name=str(pj.id))
+            old_model_age = old_model.age  # Age attribute is openstef specific
+            return old_model, model_specs, old_model_age
+        except (AttributeError, FileNotFoundError, LookupError):
+            logger.warning("No old model found, training new model", pid=pj.id)
+        except Exception:
+            logger.exception(
+                "Old model could not be loaded, training new model", pid=pj.id
+            )
+
     old_model = None
     old_model_age = float("inf")
     if pj["default_modelspecs"] is not None:
         model_specs = pj["default_modelspecs"]
         if model_specs.id != pj.id:
             raise RuntimeError(
-                "The id of the prediction job and its default model_specs do not"
-                " match."
+                "The id of the prediction job and its default model_specs do not match."
             )
     else:
         # create basic model_specs

openstef/tasks/train_model.py

@@ -19,8 +19,10 @@ Example:
         $ python model_train.py
 
 """
-from datetime import datetime, timedelta, UTC
+
+from datetime import UTC, datetime, timedelta
 from pathlib import Path
+from typing import Optional
 
 import pandas as pd
 
@@ -41,14 +43,16 @@ from openstef.tasks.utils.taskcontext import TaskContext
 
 TRAINING_PERIOD_DAYS: int = 120
 DEFAULT_CHECK_MODEL_AGE: bool = True
+DEFAULT_IGNORE_EXISTING_MODELS: bool = False
 
 
 def train_model_task(
     pj: PredictionJobDataClass,
     context: TaskContext,
     check_old_model_age: bool = DEFAULT_CHECK_MODEL_AGE,
-    datetime_start: datetime = None,
-    datetime_end: datetime = None,
+    datetime_start: Optional[datetime] = None,
+    datetime_end: Optional[datetime] = None,
+    ignore_existing_models: bool = DEFAULT_IGNORE_EXISTING_MODELS,
 ) -> None:
     """Train model task.
 
@@ -104,7 +108,9 @@ def train_model_task(
     serializer = MLflowSerializer(mlflow_tracking_uri=mlflow_tracking_uri)
 
     # Get old model and age
-    _, _, old_model_age = train_pipeline_step_load_model(pj, serializer)
+    _, _, old_model_age = train_pipeline_step_load_model(
+        pj, serializer, ignore_existing_models
+    )
 
     # Check old model age and continue yes/no
     if (old_model_age < MAXIMUM_MODEL_AGE) and check_old_model_age:
@@ -168,6 +174,7 @@ def train_model_task(
             check_old_model_age=check_old_model_age,
             mlflow_tracking_uri=mlflow_tracking_uri,
             artifact_folder=artifact_folder,
+            ignore_existing_models=ignore_existing_models,
         )
 
         if data_sets:

{openstef-3.4.65.dist-info → openstef-3.4.66.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: openstef
-Version: 3.4.65
+Version: 3.4.66
 Summary: Open short term energy forecaster
 Home-page: https://github.com/OpenSTEF/openstef
 Author: Alliander N.V

{openstef-3.4.65.dist-info → openstef-3.4.66.dist-info}/RECORD

@@ -73,7 +73,7 @@ openstef/pipeline/create_component_forecast.py,sha256=U2v_R-FSOXWVbWeknsJbkulN1Y
 openstef/pipeline/create_forecast.py,sha256=uvp5mQqGSOx-ANY-9o5reiBYNNby0npm-0lt4w9EQ18,5763
 openstef/pipeline/optimize_hyperparameters.py,sha256=uwXkzRA_fTSFt0yBuvvEoY5-4dMv42FPdS4hZocL-N8,11114
 openstef/pipeline/train_create_forecast_backtest.py,sha256=hBJPxfDkbrmFSSGZrRH1vTiIVqJP-SWe0ibVpHT_8Qg,6048
-openstef/pipeline/train_model.py,sha256=zFQS_XSqN4VQWEpvC1dvGN1rcI2tGOuFOTaN_dnKZSA,19808
+openstef/pipeline/train_model.py,sha256=8tqJcfqjT9gsXoOSBJxf3i-N_3BPmxbUqt_Ygd7Oao0,20134
 openstef/pipeline/utils.py,sha256=23mB31p19FoGWelLJzxNmqlzGwEr3fCDBEA37V2kpYY,2167
 openstef/plotting/__init__.py,sha256=KQjXzyafCt1bE7XDrSeV4TDUIO7MkwN_Br4ASOcNI2g,163
 openstef/plotting/load_forecast_plotter.py,sha256=n-dB2dQnqjWCvV3kBjnOZYQ03J-9jSIHVovJy3nGSnQ,8129
@@ -90,15 +90,15 @@ openstef/tasks/create_solar_forecast.py,sha256=HDrJrvTPCM8GS7EQwNr9uJNamf-nH2pu0
 openstef/tasks/create_wind_forecast.py,sha256=RhshkmNSyFWx4Y6yQn02GzHjWTREbN5A5GAeWv0JpcE,2907
 openstef/tasks/optimize_hyperparameters.py,sha256=3NT0KFgim8wAzWPJ0S-GULM3zoshyj63Ivp-g1_oPDw,4765
 openstef/tasks/split_forecast.py,sha256=X1D3MnnMdAb9wzDWubAJwfMkWpNGdRUPDvPAbJApNhg,9277
-openstef/tasks/train_model.py,sha256=ioEOMpKMveEdY63aRXSh4IsioVFTqj-EDggFmJ7nfiw,8514
+openstef/tasks/train_model.py,sha256=-d1VewDAaZV2B_JAnwl02Y3hONq7cPZrpH6X87_IOKA,8772
 openstef/tasks/utils/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/tasks/utils/dependencies.py,sha256=Jy9dtV_G7lTEa5Cdy--wvMxJuAb0adb3R0X4QDjVteM,3077
 openstef/tasks/utils/predictionjobloop.py,sha256=Ysy3zF5lzPMz_asYDKeF5m0qgVT3tCtwSPihqMjnI5Q,9580
 openstef/tasks/utils/taskcontext.py,sha256=L9K14ycwgVxbIVUjH2DIn_QWbnu-OfxcGtQ1K9T6sus,5630
 openstef/validation/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/validation/validation.py,sha256=24GEzLyjVqaE2a-MppbFS-YQT5n739BxD7fH3LK5LEE,12133
-openstef-3.4.65.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
-openstef-3.4.65.dist-info/METADATA,sha256=YRXl2VGYnkq3BNtZ90rovW5__cBKVf5XuV8KrV7mhLI,8816
-openstef-3.4.65.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-openstef-3.4.65.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
-openstef-3.4.65.dist-info/RECORD,,
+openstef-3.4.66.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
+openstef-3.4.66.dist-info/METADATA,sha256=L8J4MBiz55-LU8iettxkpAP4Nj5UF4kR7wi8WBlFvtY,8816
+openstef-3.4.66.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+openstef-3.4.66.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
+openstef-3.4.66.dist-info/RECORD,,