openstef 3.4.56__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

openstef/pipeline/optimize_hyperparameters.py

@@ -1,317 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-import logging
-import os
-from typing import Any
-
-import optuna
-import pandas as pd
-import structlog
-
-from openstef.data_classes.model_specifications import ModelSpecificationDataClass
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.exceptions import (
-    InputDataInsufficientError,
-    InputDataWrongColumnOrderError,
-)
-from openstef.feature_engineering.feature_applicator import TrainFeatureApplicator
-from openstef.metrics.reporter import Report, Reporter
-from openstef.model.model_creator import ModelCreator
-from openstef.model.objective import RegressorObjective
-from openstef.model.objective_creator import ObjectiveCreator
-from openstef.model.regressors.regressor import OpenstfRegressor
-from openstef.model.serializer import MLflowSerializer
-from openstef.model_selection.model_selection import split_data_train_validation_test
-from openstef.pipeline.train_model import (
-    DEFAULT_TRAIN_HORIZONS_HOURS,
-    train_model_pipeline_core,
-)
-from openstef.settings import Settings
-from openstef.validation import validation
-
-optuna.logging.enable_propagation()  # Propagate logs to the root logger.
-optuna.logging.disable_default_handler()  # Stop showing logs in sys.stderr.
-
-structlog.configure(
-    wrapper_class=structlog.make_filtering_bound_logger(
-        logging.getLevelName(Settings.log_level)
-    )
-)
-logger = structlog.get_logger(__name__)
-
-# See https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.Study.html#optuna.study.Study.optimize
-N_TRIALS: int = 100  # The number of trials.
-TIMEOUT: int = 600  # Stop study after the given number of second(s).
-
-
-def optimize_hyperparameters_pipeline(
-    pj: PredictionJobDataClass,
-    input_data: pd.DataFrame,
-    mlflow_tracking_uri: str,
-    artifact_folder: str,
-    n_trials: int = N_TRIALS,
-) -> dict:
-    """Optimize hyperparameters pipeline.
-
-    Expected prediction job key's: "name", "model"
-
-    Args:
-        pj: Prediction job
-        input_data: Raw training input data
-        mlflow_tracking_uri: Path/Uri to mlflow service
-        artifact_folder: Path where artifacts, such as trained models, are stored
-        horizons: horizons for feature engineering.
-        n_trials: The number of trials. Defaults to N_TRIALS.
-
-    Raises:
-        ValueError: If the input_date is insufficient.
-        InputDataInsufficientError: If the input dataframe is empty.
-        InputDataWrongColumnOrderError: If the load column is missing in the input dataframe.
-        OldModelHigherScoreError: When old model is better than new model.
-
-    Returns:
-        Optimized hyperparameters.
-
-    """
-    if pj.train_horizons_minutes is None:
-        horizons = DEFAULT_TRAIN_HORIZONS_HOURS
-    else:
-        horizons = [
-            horizon_minutes / 60 for horizon_minutes in pj.train_horizons_minutes
-        ]
-    (
-        best_model,
-        model_specs,
-        report,
-        trials,
-        best_trial_number,
-        best_params,
-    ) = optimize_hyperparameters_pipeline_core(pj, input_data, horizons, n_trials)
-
-    # Create serializer
-    serializer = MLflowSerializer(mlflow_tracking_uri=mlflow_tracking_uri)
-
-    # Save model, optimization results and report
-    serializer.save_model(
-        model=best_model,
-        experiment_name=str(pj["id"]),
-        model_type=pj["model"],
-        model_specs=model_specs,
-        report=report,
-        phase="Hyperparameter_opt",
-        trials=trials,
-        trial_number=best_trial_number,
-    )
-    if artifact_folder:
-        report_folder = os.path.join(artifact_folder, str(pj["id"]))
-        Reporter.write_report_to_disk(report=report, report_folder=report_folder)
-    return best_params
-
-
-def optimize_hyperparameters_pipeline_core(
-    pj: PredictionJobDataClass,
-    input_data: pd.DataFrame,
-    horizons: list[float] = DEFAULT_TRAIN_HORIZONS_HOURS,
-    n_trials: int = N_TRIALS,
-) -> tuple[
-    OpenstfRegressor, ModelSpecificationDataClass, Report, dict, int, dict[str, Any]
-]:
-    """Optimize hyperparameters pipeline core.
-
-    Expected prediction job key's: "name", "model"
-
-    Args:
-        pj: Prediction job
-        input_data: Raw training input data
-        horizons: horizons for feature engineering in hours.
-        n_trials: The number of trials. Defaults to N_TRIALS.
-
-    Raises:
-        ValueError: If the input_date is insufficient.
-        InputDataInsufficientError: If the input dataframe is empty.
-        InputDataWrongColumnOrderError: If the load column is missing in the input dataframe.
-        OldModelHigherScoreError: When old model is better than new model.
-        InputDataOngoingZeroFlatlinerError: When all recent load measurements are zero.
-
-    Returns:
-        - Best model,
-        - Model specifications of the best model,
-        - Report of the best training round,
-        - Trials,
-        - Best trial number,
-        - Optimized hyperparameters.
-
-    """
-    if input_data.empty:
-        raise InputDataInsufficientError("Input dataframe is empty")
-    elif "load" not in input_data.columns:
-        raise InputDataWrongColumnOrderError(
-            "Missing the load column in the input dataframe"
-        )
-
-    # Validate and clean data
-    validated_data = validation.drop_target_na(
-        validation.validate(
-            pj["id"],
-            input_data,
-            pj["flatliner_threshold_minutes"],
-            pj["resolution_minutes"],
-        )
-    )
-
-    # Check if sufficient data is left after cleaning
-    if not validation.is_data_sufficient(
-        validated_data, pj["completeness_threshold"], pj["minimal_table_length"]
-    ):
-        raise InputDataInsufficientError(
-            f"Input data is insufficient for {pj['name']} after validation and cleaning"
-        )
-
-    if pj.default_modelspecs:
-        feature_names = pj.default_modelspecs.feature_names
-        feature_modules = pj.default_modelspecs.feature_modules
-    else:
-        feature_names = None
-        feature_modules = []
-
-    if isinstance(horizons, str):
-        if horizons not in set(input_data.columns):
-            raise ValueError(
-                f"The horizon parameter specifies a column name ({horizons}) missing in"
-                " the input data."
-            )
-        else:
-            # sort data to avoid same date repeated multiple time
-            input_data = input_data.sort_values(horizons)
-
-    validated_data_with_features = TrainFeatureApplicator(
-        horizons=horizons, feature_names=feature_names, feature_modules=feature_modules
-    ).add_features(validated_data, pj=pj)
-
-    # Create objective (NOTE: this is a callable class)
-    objective = ObjectiveCreator.create_objective(model_type=pj["model"])
-
-    study, objective = optuna_optimization(
-        pj, objective, validated_data_with_features, n_trials
-    )
-
-    best_hyperparams = study.best_params
-    # The best_model could be accessed via study.user_attrs["best_model"]
-
-    logger.info(
-        f"Finished hyperparameter optimization, error objective {study.best_value} "
-        f"and params {best_hyperparams}"
-    )
-
-    # Add quantiles to hyperparams so they are stored with the model info
-    if pj["quantiles"]:
-        best_hyperparams.update(quantiles=pj["quantiles"])
-
-    # model specification
-    model_specs = ModelSpecificationDataClass(
-        id=pj["id"],
-        feature_names=list(validated_data_with_features.columns),
-        hyper_params=best_hyperparams,
-    )
-
-    # Train a model using the regular train pipeline.
-    # The train/validation/test split used in hyperparam optimisation
-    # is less suitable for an operational model.
-    model, report, model_specs, _ = train_model_pipeline_core(
-        pj=pj, input_data=input_data, model_specs=model_specs
-    )
-
-    trials = objective.get_trial_track()
-    best_trial_number = study.best_trial.number
-
-    return model, model_specs, report, trials, best_trial_number, study.best_params
-
-
-def optuna_optimization(
-    pj: PredictionJobDataClass,
-    objective: RegressorObjective,
-    validated_data_with_features: pd.DataFrame,
-    n_trials: int,
-) -> tuple[optuna.study.Study, RegressorObjective]:
-    """Perform hyperparameter optimization with optuna.
-
-    Args:
-        pj: Prediction job
-        objective: Objective function for optuna
-        validated_data_with_features: cleaned input dataframe
-        n_trials: number of optuna trials
-
-    Returns:
-        - Optimization study from optuna
-        - The objective object used by optuna
-
-    """
-    model = ModelCreator.create_model(pj["model"], **(pj.model_kwargs or {}))
-    # Apply set to default hyperparameters if they are specified in the pj
-    if pj.default_modelspecs:
-        valid_hyper_parameters = {
-            key: value
-            for key, value in pj.default_modelspecs.hyper_params.items()
-            if key in model.get_params().keys()
-        }
-        model.set_params(**valid_hyper_parameters)
-
-    study = optuna.create_study(
-        study_name=pj["model"],
-        pruner=optuna.pruners.MedianPruner(n_warmup_steps=5),
-        direction="minimize",
-    )
-
-    # Start with evaluating the default set of parameters,
-    # this way the optimization never get worse than the default values
-    study.enqueue_trial(objective.get_default_values())
-
-    if pj.train_split_func is None:
-        split_func = split_data_train_validation_test
-        split_args = {
-            "stratification_min_max": True,
-            "back_test": True,
-        }
-    else:
-        split_func, split_args = pj.train_split_func.load(
-            required_arguments=["data", "test_fraction", "validation_fraction"]
-        )
-
-    objective = objective(
-        model,
-        validated_data_with_features,
-        split_func=split_func,
-        split_args=split_args,
-    )
-
-    # Optuna updates the model by itself
-    # and the model is the optimized over this finishes
-    study.optimize(
-        objective,
-        n_trials=n_trials,
-        callbacks=[_log_study_progress_and_save_best_model],
-        show_progress_bar=False,
-        timeout=TIMEOUT,
-    )
-
-    return study, objective
-
-
-def _log_study_progress_and_save_best_model(
-    study: optuna.study.Study, trial: optuna.trial.FrozenTrial
-) -> None:
-    # Collect study and trial data
-    # trial_index = study.trials.index(trial)
-    # best_trial_index = study.trials.index(study.best_trial)
-    value = trial.value
-    params = trial.params
-    duration = (trial.datetime_complete - trial.datetime_start).total_seconds()
-    # Log information about this trial
-    logger.debug(
-        f"Trial {trial.number} finished with value: {value} and parameters: {params}."
-        f"Best trial is {study.best_trial.number}. Iteration took {duration} s"
-    )
-    # If this trial is the best save the model as a study attribute
-    if study.best_trial.number == trial.number:
-        study.set_user_attr(key="best_model", value=trial.user_attrs["model"])

openstef/pipeline/train_create_forecast_backtest.py

@@ -1,163 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-
-import pandas as pd
-
-from openstef.data_classes.model_specifications import ModelSpecificationDataClass
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.model.confidence_interval_applicator import ConfidenceIntervalApplicator
-from openstef.model.regressors.regressor import OpenstfRegressor
-from openstef.model_selection.model_selection import backtest_split_default
-from openstef.pipeline import train_model
-from openstef.postprocessing.postprocessing import (
-    add_prediction_job_properties_to_forecast,
-)
-
-DEFAULT_TRAIN_HORIZONS: list[float] = [0.25, 24.0]
-DEFAULT_EARLY_STOPPING_ROUNDS: int = 10
-
-
-def train_model_and_forecast_back_test(
-    pj: PredictionJobDataClass,
-    modelspecs: ModelSpecificationDataClass,
-    input_data: pd.DataFrame,
-    training_horizons: list[float] = None,
-    n_folds: int = 1,
-) -> tuple[
-    pd.DataFrame,
-    list[OpenstfRegressor],
-    list[pd.DataFrame],
-    list[pd.DataFrame],
-    list[pd.DataFrame],
-]:
-    """Pipeline for a back test.
-
-    When number of folds is larger than 1: apply pipeline for a back test when forecasting
-    the entire input range.
-
-        - Makes use of kfold cross validation in order to split data multiple times.
-        - Results of all the testsets are added together to obtain the forecast for the whole input range.
-        - Obtaining the days for each fold can be done either randomly or not
-        **DO NOT USE THIS PIPELINE FOR OPERATIONAL FORECASTS**
-
-    Args:
-        pj: Prediction job.
-        modelspecs: Dataclass containing model specifications
-        input_data: Input data
-        training_horizons: horizons to train on in hours.
-            These horizons are also used to make predictions (one for every horizon)
-        n_folds: number of folds to apply (if 1, no cross validation will be applied)
-
-    Returns:
-        - Forecast (pandas.DataFrame)
-        - Fitted models (list[OpenStfRegressor])
-        - Train data sets (list[pd.DataFrame])
-        - Validation data sets (list[pd.DataFrame])
-        - Test data sets (list[pd.DataFrame])
-
-    Raises:
-        InputDataInsufficientError: when input data is insufficient.
-        InputDataWrongColumnOrderError: when input data has a invalid column order.
-        ValueError: when the horizon is a string and the corresponding column in not in the input data
-        InputDataOngoingZeroFlatlinerError: when all recent load measurements are zero.
-
-    """
-    if pj.backtest_split_func is None:
-        backtest_split_func = backtest_split_default
-        backtest_split_args = {"stratification_min_max": True}
-    else:
-        backtest_split_func, backtest_split_args = pj.backtest_split_func.load(
-            required_arguments=["data", "n_folds"]
-        )
-
-    data_with_features = train_model.train_pipeline_step_compute_features(
-        input_data=input_data, pj=pj, model_specs=modelspecs, horizons=training_horizons
-    )
-
-    # The use of zip allows to take advantage of the lazy estimation mechanisms of Python, especially if the
-    # backtest_split_func returns a generator. This can avoid unwanted multiple data copies.
-    # 1. First we retrieve a generator (use of () comprehensive) on (model, forecast, train, val, test)
-    # 2. Then we unzip the result into generators separated by result type (models, forecasts, trains, vals, tests)
-    (
-        models_folds,
-        forecast_folds,
-        train_data_folds,
-        validation_data_folds,
-        test_data_folds,
-    ) = zip(
-        *(
-            train_model_and_forecast_test_core(
-                pj,
-                modelspecs,
-                train_data,
-                validation_data,
-                test_data,
-            )
-            + (train_data, validation_data, test_data)
-            for train_data, validation_data, test_data, _ in backtest_split_func(
-                data_with_features, n_folds, **backtest_split_args
-            )
-        )
-    )
-
-    return (
-        pd.concat(forecast_folds, axis=0).sort_index(),
-        list(models_folds),
-        list(train_data_folds),
-        list(validation_data_folds),
-        list(test_data_folds),
-    )
-
-
-def train_model_and_forecast_test_core(
-    pj: PredictionJobDataClass,
-    modelspecs: ModelSpecificationDataClass,
-    train_data: pd.DataFrame,
-    validation_data: pd.DataFrame,
-    test_data: pd.DataFrame,
-) -> tuple[OpenstfRegressor, pd.DataFrame]:
-    """Trains the model and forecast on the test set.
-
-    Args:
-        pj: Prediction job.
-        modelspecs: Dataclass containing model specifications
-        train_data: Train data with computed features
-        validation_data: Validation data with computed features
-        test_data: Test data with computed features
-
-    Returns:
-        - The trained model
-        - The forecast on the test set.
-
-    Raises:
-        NotImplementedError: When using invalid model type in the prediction job.
-        InputDataWrongColumnOrderError: When 'load' column is not first and 'horizon' column is not last.
-
-    """
-    model = train_model.train_pipeline_step_train_model(
-        pj, modelspecs, train_data, validation_data
-    )
-
-    # Predict
-    model_forecast = model.predict(test_data.iloc[:, 1:-1])
-    forecast = pd.DataFrame(index=test_data.index, data={"forecast": model_forecast})
-
-    # Define tAhead to something meaningfull in the context of a backtest
-    forecast["tAhead"] = test_data.iloc[:, -1]
-
-    # Add confidence
-    forecast = ConfidenceIntervalApplicator(
-        model, test_data.iloc[:, 1:-1]
-    ).add_confidence_interval(forecast, pj)
-
-    # Prepare for output
-    forecast = add_prediction_job_properties_to_forecast(
-        pj, forecast, algorithm_type="backtest"
-    )
-
-    # Add column with realised load and horizon information
-    forecast["realised"] = test_data.iloc[:, 0]
-    forecast["horizon"] = test_data.iloc[:, -1]
-
-    return model, forecast