openstef 3.4.65__tar.gz → 3.4.66__tar.gz

{openstef-3.4.65 → openstef-3.4.66}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: openstef
-Version: 3.4.65
+Version: 3.4.66
 Summary: Open short term energy forecaster
 Home-page: https://github.com/OpenSTEF/openstef
 Author: Alliander N.V

{openstef-3.4.65 → openstef-3.4.66}/openstef/pipeline/train_model.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: MPL-2.0
 import logging
 import os
-from typing import Optional, Union, Tuple
+from typing import Optional, Tuple, Union
 
 import pandas as pd
 import structlog
@@ -46,6 +46,7 @@ def train_model_pipeline(
     check_old_model_age: bool,
     mlflow_tracking_uri: str,
     artifact_folder: str,
+    ignore_existing_models: bool = False,
 ) -> Optional[tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]]:
     """Middle level pipeline that takes care of all persistent storage dependencies.
 
@@ -79,7 +80,7 @@ def train_model_pipeline(
 
     # Get old model and age
     old_model, model_specs, old_model_age = train_pipeline_step_load_model(
-        pj, serializer
+        pj, serializer, ignore_existing_models
     )
 
     # Check old model age and continue yes/no
@@ -106,6 +107,7 @@ def train_model_pipeline(
             input_data,
             old_model,
             horizons=horizons,
+            ignore_existing_models=ignore_existing_models,
         )
     except OldModelHigherScoreError as OMHSE:
         logger.error("Old model is better than new model", pid=pj["id"], exc_info=OMHSE)
@@ -155,6 +157,7 @@ def train_model_pipeline_core(
     input_data: pd.DataFrame,
     old_model: OpenstfRegressor = None,
     horizons: list[float] = DEFAULT_TRAIN_HORIZONS_HOURS,
+    ignore_existing_models: bool = False,
 ) -> Tuple[
     OpenstfRegressor,
     Report,
@@ -203,7 +206,7 @@ def train_model_pipeline_core(
     model_specs.feature_names = list(train_data.columns)
 
     # Check if new model is better than old model
-    if old_model:
+    if old_model and not ignore_existing_models:
         combined = pd.concat([train_data, validation_data])
         # skip the forecast column added at the end of dataframes
         if pj.save_train_forecasts:
@@ -220,6 +223,7 @@ def train_model_pipeline_core(
         # Try to compare new model to old model.
         # If this does not success, for example since the feature names of the
         # old model differ from the new model, the new model is considered better
+
         try:
             score_old_model = old_model.score(x_data, y_data)
 
@@ -315,25 +319,31 @@ def train_pipeline_common(
 
 
 def train_pipeline_step_load_model(
-    pj: PredictionJobDataClass, serializer: MLflowSerializer
+    pj: PredictionJobDataClass,
+    serializer: MLflowSerializer,
+    ignore_existing_models: bool = False,
 ) -> Tuple[OpenstfRegressor, ModelSpecificationDataClass, Union[int, float]]:
     old_model: Optional[OpenstfRegressor]
-    try:
-        old_model, model_specs = serializer.load_model(experiment_name=str(pj.id))
-        old_model_age = old_model.age  # Age attribute is openstef specific
-        return old_model, model_specs, old_model_age
-    except (AttributeError, FileNotFoundError, LookupError):
-        logger.warning("No old model found, training new model", pid=pj.id)
-    except Exception:
-        logger.exception("Old model could not be loaded, training new model", pid=pj.id)
+
+    if not ignore_existing_models:
+        try:
+            old_model, model_specs = serializer.load_model(experiment_name=str(pj.id))
+            old_model_age = old_model.age  # Age attribute is openstef specific
+            return old_model, model_specs, old_model_age
+        except (AttributeError, FileNotFoundError, LookupError):
+            logger.warning("No old model found, training new model", pid=pj.id)
+        except Exception:
+            logger.exception(
+                "Old model could not be loaded, training new model", pid=pj.id
+            )
+
     old_model = None
     old_model_age = float("inf")
     if pj["default_modelspecs"] is not None:
         model_specs = pj["default_modelspecs"]
         if model_specs.id != pj.id:
             raise RuntimeError(
-                "The id of the prediction job and its default model_specs do not"
-                " match."
+                "The id of the prediction job and its default model_specs do not match."
             )
     else:
         # create basic model_specs

{openstef-3.4.65 → openstef-3.4.66}/openstef/tasks/train_model.py

@@ -19,8 +19,10 @@ Example:
         $ python model_train.py
 
 """
-from datetime import datetime, timedelta, UTC
+
+from datetime import UTC, datetime, timedelta
 from pathlib import Path
+from typing import Optional
 
 import pandas as pd
 
@@ -41,14 +43,16 @@ from openstef.tasks.utils.taskcontext import TaskContext
 
 TRAINING_PERIOD_DAYS: int = 120
 DEFAULT_CHECK_MODEL_AGE: bool = True
+DEFAULT_IGNORE_EXISTING_MODELS: bool = False
 
 
 def train_model_task(
     pj: PredictionJobDataClass,
     context: TaskContext,
     check_old_model_age: bool = DEFAULT_CHECK_MODEL_AGE,
-    datetime_start: datetime = None,
-    datetime_end: datetime = None,
+    datetime_start: Optional[datetime] = None,
+    datetime_end: Optional[datetime] = None,
+    ignore_existing_models: bool = DEFAULT_IGNORE_EXISTING_MODELS,
 ) -> None:
     """Train model task.
 
@@ -104,7 +108,9 @@ def train_model_task(
     serializer = MLflowSerializer(mlflow_tracking_uri=mlflow_tracking_uri)
 
     # Get old model and age
-    _, _, old_model_age = train_pipeline_step_load_model(pj, serializer)
+    _, _, old_model_age = train_pipeline_step_load_model(
+        pj, serializer, ignore_existing_models
+    )
 
     # Check old model age and continue yes/no
     if (old_model_age < MAXIMUM_MODEL_AGE) and check_old_model_age:
@@ -168,6 +174,7 @@ def train_model_task(
             check_old_model_age=check_old_model_age,
             mlflow_tracking_uri=mlflow_tracking_uri,
             artifact_folder=artifact_folder,
+            ignore_existing_models=ignore_existing_models,
         )
 
         if data_sets:

{openstef-3.4.65 → openstef-3.4.66}/openstef.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: openstef
-Version: 3.4.65
+Version: 3.4.66
 Summary: Open short term energy forecaster
 Home-page: https://github.com/OpenSTEF/openstef
 Author: Alliander N.V

{openstef-3.4.65 → openstef-3.4.66}/setup.py

@@ -33,7 +33,7 @@ def read_long_description_from_readme():
 
 setup(
     name="openstef",
-    version="3.4.65",
+    version="3.4.66",
     packages=find_packages(include=["openstef", "openstef.*"]),
     description="Open short term energy forecaster",
     long_description=read_long_description_from_readme(),