lecrapaud 0.19.3__py3-none-any.whl → 0.20.0__py3-none-any.whl

lecrapaud/model_selection.py

@@ -15,7 +15,7 @@ from pydantic import BaseModel
 import ast
 
 # ML models
-from sklearn.model_selection import TimeSeriesSplit
+from sklearn.model_selection import TimeSeriesSplit, StratifiedKFold, KFold
 from sklearn.calibration import CalibratedClassifierCV
 from sklearn.metrics import (
     mean_absolute_percentage_error,
@@ -63,23 +63,38 @@ from ray.tune.logger import TBXLoggerCallback
 from ray.tune.schedulers import ASHAScheduler
 from ray.air import session
 
+# HyperOpt standalone
+from hyperopt import hp, fmin, tpe, Trials, STATUS_OK, space_eval
+
 # Internal library
 from lecrapaud.search_space import all_models
 from lecrapaud.directories import clean_directory
 from lecrapaud.utils import copy_any, contains_best, logger, serialize_for_json
-from lecrapaud.config import PYTHON_ENV
+from lecrapaud.config import PYTHON_ENV, LECRAPAUD_OPTIMIZATION_BACKEND
 from lecrapaud.feature_selection import load_train_data
 from lecrapaud.db import (
     Model,
     ModelSelection,
-    ModelTraining,
-    Score,
+    ModelSelectionScore,
     Target,
     Experiment,
 )
 
 os.environ["COVERAGE_FILE"] = str(Path(".coverage").resolve())
 
+# Suppress XGBoost and LightGBM logging
+import logging
+
+logging.getLogger("lightgbm").setLevel(logging.ERROR)
+logging.getLogger("xgboost").setLevel(logging.ERROR)
+
+# Set global verbosity for XGBoost
+xgb.set_config(verbosity=0)
+
+# Suppress warnings
+warnings.filterwarnings("ignore", category=UserWarning)
+warnings.filterwarnings("ignore", category=FutureWarning)
+
 # Reproducible result
 keras.utils.set_random_seed(42)
 np.random.seed(42)
@@ -410,6 +425,7 @@ class ModelEngine:
                     "metric": eval_metric,
                     "num_class": num_class,
                     "verbose": -1,
+                    "verbose_eval": False,
                 },
                 num_boost_round=params["num_boost_round"],
                 train_set=train_data,
@@ -421,6 +437,7 @@ class ModelEngine:
                     ),
                     lgb.record_evaluation(evals_result),
                     tensorboard_callback,
+                    lgb.log_evaluation(period=0),  # Disable evaluation logging
                 ],
             )
         else:
@@ -462,7 +479,7 @@ class ModelEngine:
                 if self.target_type == "regression"
                 else ("logloss" if num_class <= 2 else "mlogloss")
             )
-            xgb.set_config(verbosity=0)
+            # XGBoost verbosity already set globally
             model = xgb.train(
                 params={
                     **params["model_params"],
@@ -477,11 +494,11 @@ class ModelEngine:
                     xgb.callback.EarlyStopping(
                         rounds=params["early_stopping_rounds"], save_best=True
                     ),
-                    xgb.callback.EvaluationMonitor(),  # This shows evaluation results at each iteration
+                    # Removed EvaluationMonitor to suppress logs
                     tensorboard_callback,
                 ],
                 evals_result=evals_result,  # Record evaluation result
-                verbose_eval=10000,
+                verbose_eval=False,  # Disable evaluation logging
             )
 
         model.model_name = self.create_model
@@ -746,6 +763,171 @@ class ModelEngine:
         )
 
 
+def trainable_cv(
+    params,
+    x_train,
+    y_train,
+    x_val,
+    y_val,
+    model_name,
+    target_type,
+    experiment_name,
+    target_number,
+    create_model,
+    n_splits=3,
+    plot=False,
+    log_dir=None,
+    target_clf_thresholds: dict = None,
+    time_series=True,
+    recurrent=False,
+):
+    """Cross-validation version of trainable for hyperopt.
+
+    Uses TimeSeriesSplit for temporal data or StratifiedKFold/KFold for i.i.d. data.
+    Returns pooled metrics (single logloss/RMSE calculated on all concatenated predictions).
+    """
+    # Combine train and validation data for cross-validation
+    if recurrent:
+        x_train_val = np.concatenate([x_train, x_val], axis=0)
+        y_train_val = np.concatenate([y_train, y_val], axis=0)
+    else:
+        x_train_val = pd.concat([x_train, x_val], axis=0)
+        y_train_val = pd.concat([y_train, y_val], axis=0)
+        # Store original index for later use if needed
+        original_index = x_train_val.index.copy()
+        # Reset index for proper iloc indexing with CV splits
+        x_train_val = x_train_val.reset_index(drop=True)
+        y_train_val = y_train_val.reset_index(drop=True)
+
+    # Choose appropriate cross-validation splitter
+    if time_series:
+        # Time series split for temporal data
+        n_samples = len(x_train_val)
+        test_size = int(n_samples / (n_splits + 1))  # Ensure reasonable test size
+        cv_splitter = TimeSeriesSplit(n_splits=n_splits, test_size=test_size)
+    else:
+        # Stratified or regular K-fold for i.i.d. data
+        if target_type == "classification":
+            cv_splitter = StratifiedKFold(
+                n_splits=n_splits, shuffle=True, random_state=42
+            )
+        else:
+            cv_splitter = KFold(n_splits=n_splits, shuffle=True, random_state=42)
+
+    # Store all predictions and true values for pooled metrics
+    all_predictions = []
+    all_y_true = []
+    fold_times = []
+
+    # Get splits based on the CV strategy
+    if time_series or target_type == "regression":
+        splits = cv_splitter.split(x_train_val)
+    else:
+        # For stratified split, we need to pass y
+        if recurrent:
+            # Extract the target from the 2D array (first column is target)
+            y_for_split = y_train_val[:, 0]
+        else:
+            y_for_split = y_train_val
+        splits = cv_splitter.split(x_train_val, y_for_split)
+
+    for fold_idx, (train_idx, val_idx) in enumerate(splits):
+        # Extract fold data
+        if recurrent:
+            x_fold_train = x_train_val[train_idx]
+            y_fold_train = y_train_val[train_idx]
+            x_fold_val = x_train_val[val_idx]
+            y_fold_val = y_train_val[val_idx]
+        else:
+            x_fold_train = x_train_val.iloc[train_idx]
+            y_fold_train = y_train_val.iloc[train_idx]
+            x_fold_val = x_train_val.iloc[val_idx]
+            y_fold_val = y_train_val.iloc[val_idx]
+
+        # Train model for this fold
+        model = ModelEngine(
+            model_name=model_name,
+            target_type=target_type,
+            target_number=target_number,
+            create_model=create_model,
+            plot=False,  # Disable individual fold plots
+            log_dir=log_dir,
+        )
+
+        if recurrent:
+            timesteps = params["timesteps"]
+            x_fold_train = x_fold_train[:, -timesteps:, :]
+            x_fold_val = x_fold_val[:, -timesteps:, :]
+
+        # Fit model
+        model.fit(x_fold_train, y_fold_train, x_fold_val, y_fold_val, params)
+
+        # Get predictions
+        y_pred = model.predict(x_fold_val)
+
+        # Handle recurrent model indexing
+        if recurrent:
+            y_fold_val = pd.DataFrame(
+                y_fold_val, columns=["TARGET", "index"]
+            ).set_index("index")
+            y_pred.index = y_fold_val.index
+
+        # Store predictions and true values
+        all_predictions.append(y_pred)
+        all_y_true.append(y_fold_val)
+
+    # Concatenate all fold predictions
+    if target_type == "classification":
+        # For classification, we need to handle probability columns
+        all_pred_df = pd.concat(all_predictions, axis=0)
+        all_y_series = pd.concat(all_y_true, axis=0)
+        # Ensure we have a DataFrame with TARGET column
+        if isinstance(all_y_series, pd.Series):
+            all_y_df = pd.DataFrame({"TARGET": all_y_series})
+        else:
+            all_y_df = all_y_series
+    else:
+        # For regression, just concatenate the predictions
+        all_pred_series = pd.concat(all_predictions, axis=0)
+        all_y_series = pd.concat(all_y_true, axis=0)
+        all_pred_df = pd.DataFrame({"PRED": all_pred_series})
+        all_y_df = pd.DataFrame({"TARGET": all_y_series})
+
+    # Create combined prediction DataFrame
+    prediction = pd.concat([all_y_df[["TARGET"]], all_pred_df], axis=1)
+
+    # Calculate pooled metrics
+    score = {
+        "DATE": datetime.now(),
+        "MODEL_NAME": model_name,
+        "EVAL_DATA_STD": prediction["TARGET"].std(),
+    }
+
+    # Unscale if needed (for regression with scaling)
+    if (
+        model.need_scaling
+        and target_type == "regression"
+        and model.scaler_y is not None
+    ):
+        prediction.loc[:, "TARGET"] = model.scaler_y.inverse_transform(
+            prediction[["TARGET"]].values
+        )
+        prediction.loc[:, "PRED"] = model.scaler_y.inverse_transform(
+            prediction[["PRED"]].values
+        )
+
+    # Evaluate with pooled predictions
+    score.update(evaluate(prediction, target_type, target_clf_thresholds))
+
+    metric = "RMSE" if target_type == "regression" else "LOGLOSS"
+    logger.info(f"{model_name} CV pooled {metric}: {score[metric]:.4f}")
+
+    # Report to Ray if in Ray context
+    if session.get_session():
+        session.report(metrics=score)
+    return score
+
+
 def trainable(
     params,
     x_train,
@@ -757,7 +939,6 @@ def trainable(
     experiment_name,
     target_number,
     create_model,
-    type_name="hyperopts",
     plot=False,
     log_dir=None,
     target_clf_thresholds: dict = None,
@@ -783,9 +964,7 @@ def trainable(
         x_val = x_val[:, -timesteps:, :]
 
     # Compile and fit model on train set
-    start = time.time()
     model.fit(x_train, y_train, x_val, y_val, params)
-    stop = time.time()
 
     # Prediction on val set
     y_pred = model.predict(x_val)
@@ -815,8 +994,6 @@ def trainable(
     score = {
         "DATE": datetime.now(),
         "MODEL_NAME": model.model_name,
-        "TYPE": type_name,
-        "TRAINING_TIME": stop - start,
         "EVAL_DATA_STD": prediction["TARGET"].std(),
     }
 
@@ -825,7 +1002,8 @@ def trainable(
     metric = "RMSE" if target_type == "regression" else "LOGLOSS"
     logger.info(f"{model.model_name} scores on validation set: {score[metric]:.4f}")
 
-    if type_name == "hyperopts":
+    # Report to Ray if in Ray context
+    if session.get_session():
         session.report(metrics=score)
         return score
 
@@ -883,7 +1061,7 @@ class ModelSelectionEngine:
         experiment_name,
         perform_hyperopt=True,
         number_of_trials=20,
-        perform_crossval=False,
+        perform_crossval=False,  # This controls CV during hyperopt, not after
         plot=True,
         clean_dir=False,  # TODO: This has been unused because now feature_selection is in the target directory
         preserve_model=True,
@@ -896,6 +1074,7 @@ class ModelSelectionEngine:
         self.experiment_name = experiment_name
         self.plot = plot
         self.number_of_trials = number_of_trials
+        self.perform_crossval = perform_crossval
 
         if self.experiment_id is None:
             raise ValueError("Please provide a experiment.")
@@ -945,12 +1124,11 @@ class ModelSelectionEngine:
         # create model selection in db
         target = Target.find_by(name=f"TARGET_{self.target_number}")
         model_selection = ModelSelection.upsert(
-            match_fields=["target_id", "experiment_id"],
             target_id=target.id,
             experiment_id=self.experiment_id,
         )
 
-        # recurrent models starts at 9 # len(list_models)
+        # STEP 1 : TRAINING MODELS
         for i in self.models_idx:
             config = all_models[i]
             recurrent = config["recurrent"]
@@ -968,19 +1146,11 @@ class ModelSelectionEngine:
             elif perform_hyperopt:
                 clean_directory(self.results_dir)
 
-            logger.info(f"Training a {model_name}")
-            model = Model.upsert(
-                match_fields=["name", "type"],
-                name=model_name,
-                type=self.target_type,
-            )
-            model_training = ModelTraining.upsert(
-                match_fields=["model_id", "model_selection_id"],
-                model_id=model.id,
-                model_selection_id=model_selection.id,
+            logger.info(
+                f"{experiment_name} - Training a {model_name} at {datetime.now()} for TARGET_{self.target_number}"
             )
 
-            # getting data
+            # Getting data
             if recurrent:
                 # Clear cluster from previous Keras session graphs.
                 K.clear_session()
@@ -990,7 +1160,7 @@ class ModelSelectionEngine:
                     for i, e in enumerate(self.all_features)
                     if e in set(self.features)
                 ]
-                # TODO: Verify that features_idx are the right one, because scaling can re-arrange columns...
+                # TODO: Verify that features_idx are the right one, because scaling can re-arrange columns (should be good)...
                 x_train = x_train_reshaped[:, :, features_idx]
                 y_train = y_train_reshaped[:, [self.target_number, 0]]
                 x_val = x_val_reshaped[:, :, features_idx]
@@ -1020,7 +1190,8 @@ class ModelSelectionEngine:
                     y_test = test[f"TARGET_{self.target_number}"].rename("TARGET")
 
             log_dir = get_log_dir(self.target_dir, model_name)
-            # instantiate model
+
+            # Instantiate model
             model = ModelEngine(
                 target_number=self.target_number,
                 model_name=model_name,
@@ -1031,8 +1202,8 @@ class ModelSelectionEngine:
                 log_dir=log_dir,
             )
 
-            start = time.time()
             # Tuning hyperparameters
+            start = time.time()
             if perform_hyperopt:
                 model_best_params = self.hyperoptimize(
                     x_train, y_train, x_val, y_val, model
@@ -1049,7 +1220,7 @@ class ModelSelectionEngine:
                         f"Could not find {model_name} in current data. Try to run an hyperoptimization by setting `perform_hyperopt` to true, or pass `best_params`"
                     )
 
-            # save best params
+            # Save best params
             best_params_file = f"{self.target_dir}/best_params.json"
             try:
                 with open(best_params_file, "r") as f:
@@ -1061,114 +1232,25 @@ class ModelSelectionEngine:
             with open(best_params_file, "w") as f:
                 json.dump(json_dict, f, indent=4)
 
-            # Perform cross-validation of the best model on k-folds of train + val set
-            if perform_crossval:
-                x_train_val = pd.concat([x_train, x_val, x_test], axis=0)
-                y_train_val = pd.concat([y_train, y_val, y_test], axis=0)
-                n_splits = 4
-                n_samples = len(x_train_val)
-                test_size = int(n_samples / (n_splits + 4))
-                tscv = TimeSeriesSplit(n_splits=n_splits, test_size=test_size)
-
-                # Store the scores
-                cv_scores = []
-
-                for i, (train_index, val_index) in enumerate(tscv.split(x_train_val)):
-                    self.type_name = f"crossval_fold_{i}"
-
-                    if self.time_series:
-                        date_series = pd.concat(
-                            [
-                                train[self.date_column],
-                                val[self.date_column],
-                                test[self.date_column],
-                            ],
-                            axis=0,
-                        ).reset_index(drop=True)
-
-                        date_series = date_series.map(pd.Timestamp.fromordinal)
-
-                        # Now you can use the actual train/val indices to extract ranges
-                        train_start = date_series.iloc[train_index[0]]
-                        train_end = date_series.iloc[train_index[-1]]
-                        val_start = date_series.iloc[val_index[0]]
-                        val_end = date_series.iloc[val_index[-1]]
-
-                        logger.info(
-                            f"[Fold {i}] Train: {len(train_index)} samples from {train_start.date()} to {train_end.date()} | "
-                            f"Validation: {len(val_index)} samples from {val_start.date()} to {val_end.date()}"
-                        )
-                    else:
-                        logger.info(
-                            f"[Fold {i}] Train: {len(train_index)} samples | Validation: {len(val_index)} samples"
-                        )
-
-                    # Train the model and get the score
-                    if recurrent:
-                        cv_score, _, _ = self.train_model(
-                            params=model_best_params,
-                            x_train=x_train_val[train_index],
-                            y_train=y_train_val[train_index],
-                            x_val=x_train_val[val_index],
-                            y_val=y_train_val[val_index],
-                            model=model,
-                        )
-                    else:
-                        cv_score, _, _ = self.train_model(
-                            params=model_best_params,
-                            x_train=x_train_val.iloc[train_index],
-                            y_train=y_train_val.iloc[train_index],
-                            x_val=x_train_val.iloc[val_index],
-                            y_val=y_train_val.iloc[val_index],
-                            model=model,
-                        )
-
-                    # Append score to the list
-                    cv_scores.append(cv_score)
-
-                # Calculate mean of all numerical metrics across all cross-validation folds
-                cv_scores_df = pd.DataFrame(cv_scores)
-                # Get mean of all numeric columns
-                cv_means = cv_scores_df.mean(numeric_only=True).to_dict()
+            # Always evaluate on test set (no cross-validation here)
+            # The hyperopt already did CV if needed to find best params
+            best_score, best_model, best_pred = self.train_model(
+                params=model_best_params,
+                x_train=pd.concat([x_train, x_val], axis=0),
+                y_train=pd.concat([y_train, y_val], axis=0),
+                x_val=x_test,
+                y_val=y_test,
+                model=model,
+            )
+            stop = time.time()
+            training_time = stop - start
 
-                logger.info(f"👉 {model.model_name} mean cv scores on full dataset:")
-                for metric, value in cv_means.items():
+            logger.info(f"Model training finished in {training_time:.2f} seconds")
+            logger.info(f"👉 {model.model_name} scores on test set:")
+            for metric, value in best_score.items():
+                if isinstance(value, (int, float)):
                     logger.info(f"  {metric}: {value:.4f}")
 
-                # Retrain on entire training set, but keep score on cross-validation folds
-                # Get the test score using the best model
-                test_score, best_model, best_pred = self.train_model(
-                    params=model_best_params,
-                    x_train=pd.concat([x_train, x_val], axis=0),
-                    y_train=pd.concat([y_train, y_val], axis=0),
-                    x_val=x_test,
-                    y_val=y_test,
-                    model=model,
-                )
-
-                # Update all metrics with cross-validation means
-                for metric, value in cv_means.items():
-                    if metric in test_score:  # Only update existing metrics
-                        test_score[metric] = value
-                best_score = test_score
-                best_score["TYPE"] = "crossval"
-            else:
-                # Evaluate on test set
-                self.type_name = "testset"
-                best_score, best_model, best_pred = self.train_model(
-                    params=model_best_params,
-                    x_train=pd.concat([x_train, x_val], axis=0),
-                    y_train=pd.concat([y_train, y_val], axis=0),
-                    x_val=x_test,
-                    y_val=y_test,
-                    model=model,
-                )
-
-                logger.info(f"👉 {model.model_name} scores on test set:")
-                for metric, value in best_score.items():
-                    if isinstance(value, (int, float)):
-                        logger.info(f"  {metric}: {value:.4f}")
-
             # Save predictions
             best_pred.to_csv(
                 f"{self.results_dir}/prediction.csv",
@@ -1179,7 +1261,6 @@ class ModelSelectionEngine:
 
             # Save best model
             model_path = best_model.save(self.results_dir)
-
             model_path = Path(model_path).resolve()
             best_score["MODEL_PATH"] = model_path
 
@@ -1202,32 +1283,26 @@ class ModelSelectionEngine:
             scores_tracking.sort_values(self.metric, ascending=True, inplace=True)
             scores_tracking.to_csv(scores_tracking_path, index=False)
 
-            # Save model training metadata
-            stop = time.time()
-            training_time = stop - start
-            model_training.best_params = model_best_params
-            model_training.model_path = model_path
-            model_training.training_time = training_time
-            model_training.save()
-
-            # Store metrics in DB
+            # Save in db
             drop_cols = [
                 "DATE",
                 "MODEL_NAME",
-                "MODEL_PATH",
             ]
             best_score = {k: v for k, v in best_score.items() if k not in drop_cols}
             score_data = {k.lower(): v for k, v in best_score.items()}
-
-            Score.upsert(
-                match_fields=["model_training_id"],
-                model_training_id=model_training.id,
+            model = Model.upsert(
+                name=model_name,
+                type=self.target_type,
+            )
+            ModelSelectionScore.upsert(
+                model_id=model.id,
+                model_selection_id=model_selection.id,
+                best_params=serialize_for_json(model_best_params),
+                training_time=training_time,
                 **score_data,
             )
 
-            logger.info(f"Model training finished in {training_time:.2f} seconds")
-
-        # find best model type
+        # STEP 2 :FINDING BEST MODEL OVERALL
         scores_tracking_path = f"{self.target_dir}/scores_tracking.csv"
         scores_tracking = pd.read_csv(scores_tracking_path)
         best_score_overall = scores_tracking.iloc[0, :]
@@ -1238,12 +1313,11 @@ class ModelSelectionEngine:
         else:
             best_thresholds = None
 
-        # Remove any .best or .keras files
+        # Remove any .best or .keras files, and save best model in target_dir
         for file_path in glob.glob(os.path.join(self.target_dir, "*.best")) + glob.glob(
             os.path.join(self.target_dir, "*.keras")
         ):
             os.remove(file_path)
-        # Copy the best model in root training folder for this target
         best_model_path = Path(
             f"{self.target_dir}/{os.path.basename(best_score_overall['MODEL_PATH'])}"
         ).resolve()
@@ -1255,13 +1329,13 @@ class ModelSelectionEngine:
         with open(f"{self.target_dir}/best_params.json", "r") as f:
             best_model_params = json.load(f)[best_model_name]
 
-        # Save model_selection results to db
-
+        # Save to db
         model_selection = ModelSelection.get(model_selection.id)
-        model_selection.best_model_id = Model.find_by(
+        model = Model.find_by(
             name=best_score_overall["MODEL_NAME"], type=self.target_type
-        ).id
-        model_selection.best_model_params = best_model_params
+        )
+        model_selection.best_model_id = model.id
+        model_selection.best_model_params = serialize_for_json(best_model_params)
         model_selection.best_thresholds = best_thresholds
         model_selection.best_model_path = best_model_path
 
@@ -1286,7 +1360,169 @@ class ModelSelectionEngine:
         return best_model
 
     def hyperoptimize(self, x_train, y_train, x_val, y_val, model: ModelEngine):
-        self.type_name = "hyperopts"
+        """Choose between Ray Tune and HyperOpt standalone based on configuration."""
+        if LECRAPAUD_OPTIMIZATION_BACKEND == "hyperopt":
+            return self.hyperoptimize_hyperopt(x_train, y_train, x_val, y_val, model)
+        else:
+            return self.hyperoptimize_ray(x_train, y_train, x_val, y_val, model)
+
+    def hyperoptimize_hyperopt(
+        self, x_train, y_train, x_val, y_val, model: ModelEngine
+    ):
+        """Hyperparameter optimization using HyperOpt standalone (Celery-friendly)."""
+
+        logger.info("Start tuning hyperparameters with HyperOpt standalone...")
+
+        # Convert Ray search space to HyperOpt search space
+        def convert_search_space(ray_space):
+            """Convert Ray Tune search space to HyperOpt format."""
+            from ray.tune.search.sample import Categorical, Float, Integer
+
+            hp_space = {}
+            for key, value in ray_space.items():
+                if isinstance(value, Float):
+                    if (
+                        hasattr(value, "sampler")
+                        and value.sampler.__class__.__name__ == "LogUniform"
+                    ):
+                        # LogUniform distribution
+                        hp_space[key] = hp.loguniform(
+                            key, np.log(value.lower), np.log(value.upper)
+                        )
+                    else:
+                        # Uniform distribution
+                        hp_space[key] = hp.uniform(key, value.lower, value.upper)
+                elif isinstance(value, Integer):
+                    # Integer uniform distribution
+                    hp_space[key] = hp.randint(key, value.lower, value.upper)
+                elif isinstance(value, Categorical):
+                    # Categorical/choice distribution
+                    hp_space[key] = hp.choice(key, value.categories)
+                elif isinstance(value, dict):
+                    # Nested dict, recurse
+                    hp_space[key] = convert_search_space(value)
+                else:
+                    # Static value or unknown type
+                    hp_space[key] = value
+            return hp_space
+
+        # Create objective function for HyperOpt
+        def objective(params):
+            """Objective function to minimize."""
+            try:
+                # Convert numpy types to native Python types
+                params = serialize_for_json(params)
+
+                # Use existing trainable function based on perform_crossval
+                if self.perform_crossval:
+                    score = trainable_cv(
+                        params,
+                        x_train,
+                        y_train,
+                        x_val,
+                        y_val,
+                        model.model_name,
+                        self.target_type,
+                        self.experiment_name,
+                        self.target_number,
+                        model.create_model,
+                        n_splits=3,
+                        plot=model.plot,
+                        log_dir=model.log_dir,
+                        target_clf_thresholds=self.target_clf_thresholds,
+                        time_series=self.time_series,
+                        recurrent=model.recurrent,
+                    )
+                else:
+                    score, _, _ = trainable(
+                        params,
+                        x_train,
+                        y_train,
+                        x_val,
+                        y_val,
+                        model.model_name,
+                        self.target_type,
+                        self.experiment_name,
+                        self.target_number,
+                        model.create_model,
+                        plot=model.plot,
+                        log_dir=model.log_dir,
+                        target_clf_thresholds=self.target_clf_thresholds,
+                    )
+
+                # HyperOpt minimizes, so return the metric directly
+                loss = score[self.metric]
+
+                # Log trial info
+                logger.info(f"Trial completed - {self.metric}: {loss:.4f}")
+
+                return {
+                    "loss": loss,
+                    "status": STATUS_OK,
+                    "score": score,  # Keep full score dict for analysis
+                }
+
+            except Exception as e:
+                logger.error(f"Trial failed: {str(e)}")
+                return {"loss": float("inf"), "status": STATUS_OK, "error": str(e)}
+
+        # Convert search space
+        hp_search_space = convert_search_space(model.search_params)
+
+        # Run optimization
+        trials = Trials()
+        best_params = fmin(
+            fn=objective,
+            space=hp_search_space,
+            algo=tpe.suggest,
+            max_evals=self.number_of_trials,
+            trials=trials,
+            verbose=True,
+            show_progressbar=True,
+        )
+
+        # Get the actual parameter values (not just indices for hp.choice)
+        best_params = space_eval(hp_search_space, best_params)
+
+        # Convert numpy types to native Python types
+        best_params = serialize_for_json(best_params)
+
+        # Get best score from trials
+        best_trial_idx = np.argmin([t["result"]["loss"] for t in trials.trials])
+        best_score = trials.trials[best_trial_idx]["result"].get("score", {})
+
+        # Log results
+        logger.info(f"Best hyperparameters found were:\n{best_params}")
+        logger.info(f"Best Scores found were:\n{best_score}")
+
+        # Create summary DataFrame for consistency with Ray version
+        results_df = pd.DataFrame(
+            [
+                {
+                    "trial_id": i,
+                    self.metric: t["result"]["loss"],
+                    **{
+                        k: v
+                        for k, v in t["result"].get("score", {}).items()
+                        if isinstance(v, (int, float))
+                    },
+                }
+                for i, t in enumerate(trials.trials)
+                if t["result"]["status"] == STATUS_OK
+            ]
+        )
+
+        if not results_df.empty:
+            logger.info(f"Markdown table with all trials :\n{results_df.to_markdown()}")
+
+        # Save trial history for analysis
+        trials_path = f"{self.results_dir}/hyperopt_trials.pkl"
+        with open(trials_path, "wb") as f:
+            pickle.dump(trials, f)
+
+        return best_params
+
+    def hyperoptimize_ray(self, x_train, y_train, x_val, y_val, model: ModelEngine):
 
         def collect_error_logs(target_dir: int, storage_path: str):
             output_error_file = f"{target_dir}/errors.log"
@@ -1329,9 +1565,22 @@ class ModelSelectionEngine:
             }
         )
 
+        # Choose between regular trainable or CV version based on perform_crossval flag
+        # perform_crossval controls whether to use CV during hyperopt
+        if self.perform_crossval:
+            trainable_fn = trainable_cv
+            additional_params = {
+                "n_splits": 3,  # Can be made configurable
+                "time_series": self.time_series,  # Controls whether to use TimeSeriesSplit or StratifiedKFold
+                "recurrent": model.recurrent,
+            }
+        else:
+            trainable_fn = trainable
+            additional_params = {}
+
         tuner = Tuner(
             trainable=with_parameters(
-                trainable,
+                trainable_fn,
                 x_train=x_train,
                 y_train=y_train,
                 x_val=x_val,
@@ -1341,10 +1590,10 @@ class ModelSelectionEngine:
                 experiment_name=self.experiment_name,
                 target_number=self.target_number,
                 create_model=model.create_model,
-                type_name="hyperopts",
                 plot=model.plot,
                 log_dir=model.log_dir,
                 target_clf_thresholds=self.target_clf_thresholds,
+                **additional_params,
             ),
             param_space=model.search_params,
             tune_config=TuneConfig(
@@ -1398,7 +1647,6 @@ class ModelSelectionEngine:
             self.experiment_name,
             self.target_number,
             model.create_model,
-            self.type_name,
             model.plot,
             log_dir=model.log_dir,
             target_clf_thresholds=self.target_clf_thresholds,