spotforecast2 0.0.1__py3-none-any.whl

spotforecast2/model_selection/validation.py

@@ -0,0 +1,685 @@
+from __future__ import annotations
+from typing import Callable
+from copy import deepcopy
+import warnings
+import numpy as np
+import pandas as pd
+from joblib import Parallel, delayed, cpu_count
+from tqdm.auto import tqdm
+
+from spotforecast2.forecaster.metrics import add_y_train_argument, _get_metric
+from spotforecast2.exceptions import (
+    LongTrainingWarning,
+    IgnoredArgumentWarning,
+)
+from spotforecast2.model_selection.split_ts_cv import TimeSeriesFold
+from spotforecast2.model_selection.utils_common import (
+    check_backtesting_input,
+    select_n_jobs_backtesting,
+)
+from spotforecast2.forecaster.utils import set_skforecast_warnings
+
+
+def _backtesting_forecaster(
+    forecaster: object,
+    y: pd.Series,
+    cv: TimeSeriesFold,
+    metric: str | Callable | list[str | Callable],
+    exog: pd.Series | pd.DataFrame | None = None,
+    interval: float | list[float] | tuple[float] | str | object | None = None,
+    interval_method: str = "bootstrapping",
+    n_boot: int = 250,
+    use_in_sample_residuals: bool = True,
+    use_binned_residuals: bool = True,
+    random_state: int = 123,
+    return_predictors: bool = False,
+    n_jobs: int | str = "auto",
+    verbose: bool = False,
+    show_progress: bool = True,
+    suppress_warnings: bool = False,
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """
+    Backtesting of forecaster model following the folds generated by the TimeSeriesFold
+    class and using the metric(s) provided.
+
+    If `forecaster` is already trained and `initial_train_size` is set to `None` in the
+    TimeSeriesFold class, no initial train will be done and all data will be used
+    to evaluate the model. However, the first `len(forecaster.last_window)` observations
+    are needed to create the initial predictors, so no predictions are calculated for
+    them.
+
+    A copy of the original forecaster is created so that it is not modified during the
+    process.
+
+    Args:
+        forecaster (ForecasterRecursive, ForecasterDirect, ForecasterEquivalentDate):
+            Forecaster model.
+        y (pd.Series): Training time series.
+        cv (TimeSeriesFold): TimeSeriesFold object with the information needed to
+            split the data into folds.
+        metric (str | Callable | list): Metric used to quantify the goodness of fit
+            of the model.
+
+            - If `str`: {'mean_squared_error', 'mean_absolute_error',
+              'mean_absolute_percentage_error', 'mean_squared_log_error',
+              'mean_absolute_scaled_error', 'root_mean_squared_scaled_error'}
+            - If `Callable`: Function with arguments `y_true`, `y_pred` and `y_train`
+              (Optional) that returns a float.
+            - If `list`: List containing multiple strings and/or Callables.
+        exog (pd.Series | pd.DataFrame, optional): Exogenous variable/s included as
+            predictor/s. Must have the same number of observations as `y` and should
+            be aligned so that y[i] is regressed on exog[i]. Defaults to None.
+        interval (float | list | tuple | str | object, optional): Specifies whether
+            probabilistic predictions should be estimated and the method to use.
+            The following options are supported:
+
+            - If `float`, represents the nominal (expected) coverage (between 0 and 1).
+            For instance, `interval=0.95` corresponds to `[2.5, 97.5]` percentiles.
+            - If `list` or `tuple`: Sequence of percentiles to compute, each value must
+              be between 0 and 100 inclusive. For example, a 95% confidence interval can
+              be specified as `interval = [2.5, 97.5]` or multiple percentiles (e.g. 10,
+              50 and 90) as `interval = [10, 50, 90]`.
+            - If 'bootstrapping' (str): `n_boot` bootstrapping predictions will be
+              generated.
+            - If scipy.stats distribution object, the distribution parameters will
+              be estimated for each prediction.
+            - If None, no probabilistic predictions are estimated.
+            Defaults to None.
+        interval_method (str, optional): Technique used to estimate prediction
+            intervals. Available options:
+
+            - 'bootstrapping': Bootstrapping is used to generate prediction intervals.
+            - 'conformal': Employs the conformal prediction split method for
+              interval estimation.
+            Defaults to 'bootstrapping'.
+        n_boot (int, optional): Number of bootstrapping iterations to perform when
+            estimating prediction intervals. Defaults to 250.
+        use_in_sample_residuals (bool, optional): If `True`, residuals from the
+            training data are used as proxy of prediction error to create predictions.
+            If `False`, out of sample residuals (calibration) are used.
+            Out-of-sample residuals must be precomputed using Forecaster's
+            `set_out_sample_residuals()` method. Defaults to True.
+        use_binned_residuals (bool, optional): If `True`, residuals are selected
+            based on the predicted values (binned selection).
+            If `False`, residuals are selected randomly. Defaults to True.
+        random_state (int, optional): Seed for the random number generator to
+            ensure reproducibility. Defaults to 123.
+        return_predictors (bool, optional): If `True`, the predictors used to make
+            the predictions are also returned. Defaults to False.
+        n_jobs (int | str, optional): The number of jobs to run in parallel. If `-1`,
+            then the number of jobs is set to the number of cores. If 'auto', `n_jobs`
+            is set using the function `skforecast.utils.select_n_jobs_backtesting`.
+            Defaults to 'auto'.
+        verbose (bool, optional): Print number of folds and index of training and
+            validation sets used for backtesting. Defaults to False.
+        show_progress (bool, optional): Whether to show a progress bar.
+            Defaults to True.
+        suppress_warnings (bool, optional): If `True`, spotforecast warnings will be
+            suppressed during the backtesting process. See
+            `spotforecast.exceptions.warn_skforecast_categories` for more information.
+            Defaults to False.
+
+    Returns:
+        tuple (pd.DataFrame, pd.DataFrame):
+            - metric_values: Value(s) of the metric(s).
+            - backtest_predictions: Value of predictions. The DataFrame includes
+              the following columns:
+
+              - fold: Indicates the fold number where the prediction was made.
+              - pred: Predicted values for the corresponding series and time steps.
+
+              If `interval` is not `None`, additional columns are included depending
+              on the method:
+
+              - For `float`: Columns `lower_bound` and `upper_bound`.
+              - For `list` or `tuple` of 2 elements: Columns `lower_bound` and
+                `upper_bound`.
+              - For `list` or `tuple` with multiple percentiles: One column per
+                percentile (e.g., `p_10`, `p_50`, `p_90`).
+              - For `'bootstrapping'`: One column per bootstrapping iteration
+                (e.g., `pred_boot_0`, `pred_boot_1`, ..., `pred_boot_n`).
+              - For `scipy.stats` distribution objects: One column for each
+                estimated parameter of the distribution (e.g., `loc`, `scale`).
+
+              If `return_predictors` is `True`, one column per predictor is created.
+
+              Depending on the relation between `steps` and `fold_stride`, the output
+              may include repeated indexes (if `fold_stride < steps`) or gaps
+              (if `fold_stride > steps`). See Notes below for more details.
+
+    Notes:
+        Note on `fold_stride` vs. `steps`:
+
+        - If `fold_stride == steps`, test sets are placed back-to-back without overlap.
+          Each observation appears only once in the output DataFrame, so the
+          index is unique.
+        - If `fold_stride < steps`, test sets overlap. Multiple forecasts are
+          generated for the same observations and, therefore, the output
+          DataFrame contains repeated indexes.
+        - If `fold_stride > steps`, there are gaps between consecutive test sets.
+          Some observations in the series will not have associated predictions,
+          so the output DataFrame has non-contiguous indexes.
+
+    References:
+        .. [1] Forecasting: Principles and Practice (3rd ed) Rob J Hyndman and
+               George Athanasopoulos. https://otexts.com/fpp3/prediction-intervals.html
+        .. [2] MAPIE - Model Agnostic Prediction Interval Estimator.
+               https://mapie.readthedocs.io/en/stable/theoretical_description_regression.html#the-split-method
+    """
+
+    set_skforecast_warnings(suppress_warnings, action="ignore")
+
+    forecaster = deepcopy(forecaster)
+    is_regression = forecaster.__spotforecast_tags__["forecaster_task"] == "regression"
+    cv = deepcopy(cv)
+
+    cv.set_params(
+        {
+            "window_size": forecaster.window_size,
+            "differentiation": forecaster.differentiation_max,
+            "return_all_indexes": False,
+            "verbose": verbose,
+        }
+    )
+
+    refit = cv.refit
+    overlapping_folds = cv.overlapping_folds
+
+    if n_jobs == "auto":
+        n_jobs = select_n_jobs_backtesting(forecaster=forecaster, refit=refit)
+    elif not isinstance(refit, bool) and refit != 1 and n_jobs != 1:
+        warnings.warn(
+            "If `refit` is an integer other than 1 (intermittent refit). `n_jobs` "
+            "is set to 1 to avoid unexpected results during parallelization.",
+            IgnoredArgumentWarning,
+        )
+        n_jobs = 1
+    else:
+        n_jobs = n_jobs if n_jobs > 0 else cpu_count()
+
+    if not isinstance(metric, list):
+        metrics = [
+            (
+                _get_metric(metric=metric)
+                if isinstance(metric, str)
+                else add_y_train_argument(metric)
+            )
+        ]
+    else:
+        metrics = [
+            _get_metric(metric=m) if isinstance(m, str) else add_y_train_argument(m)
+            for m in metric
+        ]
+
+    store_in_sample_residuals = True if use_in_sample_residuals else False
+    if interval is None:
+        forecaster._probabilistic_mode = False
+    elif use_binned_residuals:
+        forecaster._probabilistic_mode = "binned"
+    else:
+        forecaster._probabilistic_mode = "no_binned"
+
+    folds = cv.split(X=y, as_pandas=False)
+    initial_train_size = cv.initial_train_size
+    window_size = cv.window_size
+    gap = cv.gap
+
+    if initial_train_size is not None:
+        # NOTE: This allows for parallelization when `refit` is `False`. The initial
+        # Forecaster fit occurs outside of the auxiliary function.
+        exog_train = exog.iloc[:initial_train_size,] if exog is not None else None
+        forecaster.fit(
+            y=y.iloc[:initial_train_size,],
+            exog=exog_train,
+            store_in_sample_residuals=store_in_sample_residuals,
+        )
+        folds[0][5] = False
+
+    if refit:
+        n_of_fits = int(len(folds) / refit)
+        if type(forecaster).__name__ != "ForecasterDirect" and n_of_fits > 50:
+            warnings.warn(
+                f"The forecaster will be fit {n_of_fits} times. This can take substantial"
+                f" amounts of time. If not feasible, try with `refit = False`.\n",
+                LongTrainingWarning,
+            )
+        elif (
+            type(forecaster).__name__ == "ForecasterDirect"
+            and n_of_fits * forecaster.max_step > 50
+        ):
+            warnings.warn(
+                f"The forecaster will be fit {n_of_fits * forecaster.max_step} times "
+                f"({n_of_fits} folds * {forecaster.max_step} estimators). This can take "
+                f"substantial amounts of time. If not feasible, try with `refit = False`.\n",
+                LongTrainingWarning,
+            )
+
+    if show_progress:
+        folds = tqdm(folds)
+
+    def _fit_predict_forecaster(
+        fold,
+        forecaster,
+        y,
+        exog,
+        store_in_sample_residuals,
+        gap,
+        interval,
+        interval_method,
+        n_boot,
+        use_in_sample_residuals,
+        use_binned_residuals,
+        random_state,
+        return_predictors,
+        is_regression,
+    ) -> pd.DataFrame:
+        """
+        Fit the forecaster and predict `steps` ahead. This is an auxiliary
+        function used to parallelize the backtesting_forecaster function.
+        """
+
+        train_iloc_start = fold[1][0]
+        train_iloc_end = fold[1][1]
+        last_window_iloc_start = fold[2][0]
+        last_window_iloc_end = fold[2][1]
+        test_iloc_start = fold[3][0]
+        test_iloc_end = fold[3][1]
+
+        if fold[5] is False:
+            # When the model is not fitted, last_window must be updated to include
+            # the data needed to make predictions.
+            last_window_y = y.iloc[last_window_iloc_start:last_window_iloc_end]
+        else:
+            # The model is fitted before making predictions. If `fixed_train_size`
+            # the train size doesn't increase but moves by `steps` in each iteration.
+            # If `False` the train size increases by `steps` in each iteration.
+            y_train = y.iloc[train_iloc_start:train_iloc_end,]
+            exog_train = (
+                exog.iloc[train_iloc_start:train_iloc_end,]
+                if exog is not None
+                else None
+            )
+            last_window_y = None
+            forecaster.fit(
+                y=y_train,
+                exog=exog_train,
+                store_in_sample_residuals=store_in_sample_residuals,
+            )
+
+        next_window_exog = (
+            exog.iloc[test_iloc_start:test_iloc_end,] if exog is not None else None
+        )
+
+        steps = len(range(test_iloc_start, test_iloc_end))
+        if type(forecaster).__name__ == "ForecasterDirect" and gap > 0:
+            # Select only the steps that need to be predicted if gap > 0
+            test_no_gap_iloc_start = fold[4][0]
+            test_no_gap_iloc_end = fold[4][1]
+            n_steps = test_no_gap_iloc_end - test_no_gap_iloc_start
+            steps = list(range(gap + 1, gap + 1 + n_steps))
+
+        preds = []
+        if is_regression:
+            if interval is not None:
+                kwargs_interval = {
+                    "steps": steps,
+                    "last_window": last_window_y,
+                    "exog": next_window_exog,
+                    "n_boot": n_boot,
+                    "use_in_sample_residuals": use_in_sample_residuals,
+                    "use_binned_residuals": use_binned_residuals,
+                    "random_state": random_state,
+                }
+                if interval_method == "bootstrapping":
+                    if interval == "bootstrapping":
+                        pred = forecaster.predict_bootstrapping(**kwargs_interval)
+                    elif isinstance(interval, (list, tuple)):
+                        quantiles = [q / 100 for q in interval]
+                        pred = forecaster.predict_quantiles(
+                            quantiles=quantiles, **kwargs_interval
+                        )
+                        if len(interval) == 2:
+                            pred.columns = ["lower_bound", "upper_bound"]
+                        else:
+                            pred.columns = [f"p_{p}" for p in interval]
+                    else:
+                        pred = forecaster.predict_dist(
+                            distribution=interval, **kwargs_interval
+                        )
+
+                    preds.append(pred)
+                else:
+                    pred = forecaster.predict_interval(
+                        method="conformal", interval=interval, **kwargs_interval
+                    )
+                    preds.append(pred)
+
+            # NOTE: This is done after probabilistic predictions to avoid repeating
+            # the same checks.
+            if interval is None or interval_method != "conformal":
+                pred = forecaster.predict(
+                    steps=steps,
+                    last_window=last_window_y,
+                    exog=next_window_exog,
+                    check_inputs=True if interval is None else False,
+                )
+                preds.insert(0, pred)
+        else:
+            pred = forecaster.predict_proba(
+                steps=steps, last_window=last_window_y, exog=next_window_exog
+            )
+            preds.append(pred)
+
+        if return_predictors:
+            pred = forecaster.create_predict_X(
+                steps=steps,
+                last_window=last_window_y,
+                exog=next_window_exog,
+                check_inputs=False,
+            )
+            preds.append(pred)
+
+        if len(preds) == 1:
+            pred = preds[0]
+        else:
+            pred = pd.concat(preds, axis=1)
+
+        if type(forecaster).__name__ != "ForecasterDirect" and gap > 0:
+            pred = pred.iloc[gap:,]
+
+        return pred
+
+    kwargs_fit_predict_forecaster = {
+        "forecaster": forecaster,
+        "y": y,
+        "exog": exog,
+        "store_in_sample_residuals": store_in_sample_residuals,
+        "gap": gap,
+        "interval": interval,
+        "interval_method": interval_method,
+        "n_boot": n_boot,
+        "use_in_sample_residuals": use_in_sample_residuals,
+        "use_binned_residuals": use_binned_residuals,
+        "random_state": random_state,
+        "return_predictors": return_predictors,
+        "is_regression": is_regression,
+    }
+    backtest_predictions = Parallel(n_jobs=n_jobs)(
+        delayed(_fit_predict_forecaster)(fold=fold, **kwargs_fit_predict_forecaster)
+        for fold in folds
+    )
+    fold_labels = [
+        np.repeat(fold[0], backtest_predictions[i].shape[0])
+        for i, fold in enumerate(folds)
+    ]
+
+    backtest_predictions = pd.concat(backtest_predictions)
+    if isinstance(backtest_predictions, pd.Series):
+        backtest_predictions = backtest_predictions.to_frame()
+
+    if not is_regression:
+        proba_cols = [f"{cls}_proba" for cls in forecaster.classes_]
+        idx_max = backtest_predictions[proba_cols].to_numpy().argmax(axis=1)
+        backtest_predictions.insert(0, "pred", np.array(forecaster.classes_)[idx_max])
+
+    backtest_predictions.insert(0, "fold", np.concatenate(fold_labels))
+
+    train_indexes = []
+    for i, fold in enumerate(folds):
+        fit_fold = fold[-1]
+        if i == 0 or fit_fold:
+            # NOTE: When using a scaled metric, `y_train` doesn't include the
+            # first window_size observations used to create the predictors and/or
+            # rolling features.
+            train_iloc_start = fold[1][0] + window_size
+            train_iloc_end = fold[1][1]
+            train_indexes.append(np.arange(train_iloc_start, train_iloc_end))
+
+    train_indexes = np.unique(np.concatenate(train_indexes))
+    y_train = y.iloc[train_indexes]
+
+    backtest_predictions_for_metrics = backtest_predictions
+    if overlapping_folds:
+        backtest_predictions_for_metrics = backtest_predictions_for_metrics.loc[
+            ~backtest_predictions_for_metrics.index.duplicated(keep="last")
+        ]
+
+    y_true = y.loc[backtest_predictions_for_metrics.index]
+    y_pred = backtest_predictions_for_metrics["pred"]
+    metric_values = [
+        [m(y_true=y_true, y_pred=y_pred, y_train=y_train) for m in metrics]
+    ]
+
+    metric_values = pd.DataFrame(
+        data=metric_values, columns=[m.__name__ for m in metrics]
+    )
+
+    set_skforecast_warnings(suppress_warnings, action="default")
+
+    return metric_values, backtest_predictions
+
+
+def backtesting_forecaster(
+    forecaster: object,
+    y: pd.Series,
+    cv: TimeSeriesFold,
+    metric: str | Callable | list[str | Callable],
+    exog: pd.Series | pd.DataFrame | None = None,
+    interval: float | list[float] | tuple[float] | str | object | None = None,
+    interval_method: str = "bootstrapping",
+    n_boot: int = 250,
+    use_in_sample_residuals: bool = True,
+    use_binned_residuals: bool = True,
+    random_state: int = 123,
+    return_predictors: bool = False,
+    n_jobs: int | str = "auto",
+    verbose: bool = False,
+    show_progress: bool = True,
+    suppress_warnings: bool = False,
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """
+    Backtesting of forecaster model following the folds generated by the TimeSeriesFold
+    class and using the metric(s) provided.
+
+    If `forecaster` is already trained and `initial_train_size` is set to `None` in the
+    TimeSeriesFold class, no initial train will be done and all data will be used
+    to evaluate the model. However, the first `len(forecaster.last_window)` observations
+    are needed to create the initial predictors, so no predictions are calculated for
+    them.
+
+    A copy of the original forecaster is created so that it is not modified during
+    the process.
+
+    Args:
+        forecaster (ForecasterRecursive, ForecasterDirect, ForecasterEquivalentDate):
+            Forecaster model.
+        y (pd.Series): Training time series.
+        cv (TimeSeriesFold): TimeSeriesFold object with the information needed to
+            split the data into folds.
+        metric (str | Callable | list): Metric used to quantify the goodness of fit
+            of the model.
+
+            - If `str`: {'mean_squared_error', 'mean_absolute_error',
+              'mean_absolute_percentage_error', 'mean_squared_log_error',
+              'mean_absolute_scaled_error', 'root_mean_squared_scaled_error'}
+            - If `Callable`: Function with arguments `y_true`, `y_pred` and `y_train`
+              (Optional) that returns a float.
+            - If `list`: List containing multiple strings and/or Callables.
+        exog (pd.Series | pd.DataFrame, optional): Exogenous variable/s included as
+            predictor/s. Must have the same number of observations as `y` and should
+            be aligned so that y[i] is regressed on exog[i]. Defaults to None.
+        interval (float | list | tuple | str | object, optional): Specifies whether
+            probabilistic predictions should be estimated and the method to use.
+            The following options are supported:
+
+            - If `float`, represents the nominal (expected) coverage (between 0 and 1).
+            For instance, `interval=0.95` corresponds to `[2.5, 97.5]` percentiles.
+            - If `list` or `tuple`: Sequence of percentiles to compute, each value must
+              be between 0 and 100 inclusive. For example, a 95% confidence interval can
+              be specified as `interval = [2.5, 97.5]` or multiple percentiles (e.g. 10,
+              50 and 90) as `interval = [10, 50, 90]`.
+            - If 'bootstrapping' (str): `n_boot` bootstrapping predictions will be
+              generated.
+            - If scipy.stats distribution object, the distribution parameters will
+              be estimated for each prediction.
+            - If None, no probabilistic predictions are estimated.
+            Defaults to None.
+        interval_method (str, optional): Technique used to estimate prediction
+            intervals. Available options:
+
+            - 'bootstrapping': Bootstrapping is used to generate prediction intervals.
+            - 'conformal': Employs the conformal prediction split method for
+              interval estimation.
+            Defaults to 'bootstrapping'.
+        n_boot (int, optional): Number of bootstrapping iterations to perform when
+            estimating prediction intervals. Defaults to 250.
+        use_in_sample_residuals (bool, optional): If `True`, residuals from the
+            training data are used as proxy of prediction error to create predictions.
+            If `False`, out of sample residuals (calibration) are used.
+            Out-of-sample residuals must be precomputed using Forecaster's
+            `set_out_sample_residuals()` method. Defaults to True.
+        use_binned_residuals (bool, optional): If `True`, residuals are selected
+            based on the predicted values (binned selection).
+            If `False`, residuals are selected randomly. Defaults to True.
+        random_state (int, optional): Seed for the random number generator to
+            ensure reproducibility. Defaults to 123.
+        return_predictors (bool, optional): If `True`, the predictors used to make
+            the predictions are also returned. Defaults to False.
+        n_jobs (int | str, optional): The number of jobs to run in parallel. If `-1`,
+            then the number of jobs is set to the number of cores. If 'auto', `n_jobs`
+            is set using the function `skforecast.utils.select_n_jobs_backtesting`.
+            Defaults to 'auto'.
+        verbose (bool, optional): Print number of folds and index of training and
+            validation sets used for backtesting. Defaults to False.
+        show_progress (bool, optional): Whether to show a progress bar.
+            Defaults to True.
+        suppress_warnings (bool, optional): If `True`, spotforecast warnings will be
+            suppressed during the backtesting process. See
+            `spotforecast.exceptions.warn_skforecast_categories` for more information.
+            Defaults to False.
+
+    Returns:
+        tuple (pd.DataFrame, pd.DataFrame):
+            - metric_values: Value(s) of the metric(s).
+            - backtest_predictions: Value of predictions. The DataFrame includes
+              the following columns:
+
+              - fold: Indicates the fold number where the prediction was made.
+              - pred: Predicted values for the corresponding series and time steps.
+
+              If `interval` is not `None`, additional columns are included depending
+              on the method:
+
+              - For `float`: Columns `lower_bound` and `upper_bound`.
+              - For `list` or `tuple` of 2 elements: Columns `lower_bound` and
+                `upper_bound`.
+              - For `list` or `tuple` with multiple percentiles: One column per
+                percentile (e.g., `p_10`, `p_50`, `p_90`).
+              - For `'bootstrapping'`: One column per bootstrapping iteration
+                (e.g., `pred_boot_0`, `pred_boot_1`, ..., `pred_boot_n`).
+              - For `scipy.stats` distribution objects: One column for each
+                estimated parameter of the distribution (e.g., `loc`, `scale`).
+
+              If `return_predictors` is `True`, one column per predictor is created.
+
+              Depending on the relation between `steps` and `fold_stride`, the output
+              may include repeated indexes (if `fold_stride < steps`) or gaps
+              (if `fold_stride > steps`). See Notes below for more details.
+
+    Notes:
+        Note on `fold_stride` vs. `steps`:
+
+        - If `fold_stride == steps`, test sets are placed back-to-back without overlap.
+          Each observation appears only once in the output DataFrame, so the
+          index is unique.
+        - If `fold_stride < steps`, test sets overlap. Multiple forecasts are
+          generated for the same observations and, therefore, the output
+          DataFrame contains repeated indexes.
+        - If `fold_stride > steps`, there are gaps between consecutive test sets.
+          Some observations in the series will not have associated predictions,
+          so the output DataFrame has non-contiguous indexes.
+
+    Examples:
+        >>> import pandas as pd
+        >>> from sklearn.ensemble import RandomForestRegressor
+        >>> from spotforecast2.forecaster.recursive import ForecasterRecursive
+        >>> from spotforecast2.model_selection import backtesting_forecaster, TimeSeriesFold
+        >>> y = pd.Series([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+        >>> forecaster = ForecasterRecursive(
+        ...     estimator=RandomForestRegressor(random_state=123),
+        ...     lags=2
+        ... )
+        >>> cv = TimeSeriesFold(
+        ...     steps=2,
+        ...     initial_train_size=5,
+        ...     refit=False
+        ... )
+        >>> metric_values, backtest_predictions = backtesting_forecaster(
+        ...     forecaster=forecaster,
+        ...     y=y,
+        ...     cv=cv,
+        ...     metric='mean_squared_error'
+        ... )
+        >>> metric_values
+           mean_squared_error
+        0            0.201334
+        >>> backtest_predictions
+           fold  pred
+        5     0  5.18
+        6     0  6.10
+        7     1  7.36
+        8     1  8.40
+        9     2  9.31
+    """
+
+    forecaters_allowed = [
+        "ForecasterRecursive",
+        "ForecasterDirect",
+        "ForecasterEquivalentDate",
+        "ForecasterRecursiveClassifier",
+    ]
+
+    if type(forecaster).__name__ not in forecaters_allowed:
+        raise TypeError(
+            f"`forecaster` must be of type {forecaters_allowed}. For all other "
+            f"types of forecasters use the other functions available in the "
+            f"`model_selection` module."
+        )
+
+    check_backtesting_input(
+        forecaster=forecaster,
+        cv=cv,
+        y=y,
+        metric=metric,
+        interval=interval,
+        interval_method=interval_method,
+        n_boot=n_boot,
+        use_in_sample_residuals=use_in_sample_residuals,
+        use_binned_residuals=use_binned_residuals,
+        random_state=random_state,
+        return_predictors=return_predictors,
+        n_jobs=n_jobs,
+        show_progress=show_progress,
+        suppress_warnings=suppress_warnings,
+    )
+
+    metric_values, backtest_predictions = _backtesting_forecaster(
+        forecaster=forecaster,
+        y=y,
+        cv=cv,
+        metric=metric,
+        exog=exog,
+        interval=interval,
+        interval_method=interval_method,
+        n_boot=n_boot,
+        use_in_sample_residuals=use_in_sample_residuals,
+        use_binned_residuals=use_binned_residuals,
+        random_state=random_state,
+        return_predictors=return_predictors,
+        n_jobs=n_jobs,
+        verbose=verbose,
+        show_progress=show_progress,
+        suppress_warnings=suppress_warnings,
+    )
+
+    return metric_values, backtest_predictions