ml4t-diagnostic 0.1.0a1__py3-none-any.whl

ml4t/diagnostic/evaluation/stats/backtest_overfitting.py

@@ -0,0 +1,219 @@
+"""Probability of Backtest Overfitting (PBO).
+
+PBO measures the probability that a strategy selected as best in-sample
+performs below median out-of-sample. A high PBO indicates overfitting.
+
+This module is intentionally separate from DSR/Sharpe inference because
+PBO is a model selection diagnostic, not a statistical inference tool.
+
+References
+----------
+Bailey, D. H., & López de Prado, M. (2014). "The Probability of Backtest
+Overfitting." Journal of Computational Finance, 20(4), 39-69.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class PBOResult:
+    """Result of Probability of Backtest Overfitting calculation.
+
+    Attributes
+    ----------
+    pbo : float
+        Probability of Backtest Overfitting (0 to 1).
+    pbo_pct : float
+        PBO as percentage (0 to 100).
+    n_combinations : int
+        Number of IS/OOS combinations evaluated.
+    n_strategies : int
+        Number of strategies compared.
+    is_best_rank_oos_median : float
+        Median OOS rank of IS-best strategy.
+    is_best_rank_oos_mean : float
+        Mean OOS rank of IS-best strategy.
+    degradation_mean : float
+        Average OOS performance degradation vs IS.
+    degradation_std : float
+        Std of degradation.
+    """
+
+    pbo: float
+    pbo_pct: float
+    n_combinations: int
+    n_strategies: int
+    is_best_rank_oos_median: float
+    is_best_rank_oos_mean: float
+    degradation_mean: float
+    degradation_std: float
+
+    def interpret(self) -> str:
+        """Generate human-readable interpretation."""
+        if self.pbo < 0.10:
+            risk_level = "LOW"
+            assessment = "Strategy selection appears robust"
+        elif self.pbo < 0.30:
+            risk_level = "MODERATE"
+            assessment = "Some overfitting risk - consider out-of-sample validation"
+        elif self.pbo < 0.50:
+            risk_level = "HIGH"
+            assessment = "Significant overfitting risk - results may not generalize"
+        else:
+            risk_level = "SEVERE"
+            assessment = "IS selection is counterproductive - consider alternative methods"
+
+        return (
+            f"Probability of Backtest Overfitting (PBO)\n"
+            f"  PBO: {self.pbo_pct:.1f}%\n"
+            f"  Risk Level: {risk_level}\n"
+            f"  Assessment: {assessment}\n"
+            f"\n"
+            f"  Combinations: {self.n_combinations}\n"
+            f"  Strategies: {self.n_strategies}\n"
+            f"  IS-Best OOS Rank: {self.is_best_rank_oos_median:.1f} (median), "
+            f"{self.is_best_rank_oos_mean:.1f} (mean)\n"
+            f"  Performance Degradation: {self.degradation_mean:.4f} +/- {self.degradation_std:.4f}"
+        )
+
+    def to_dict(self) -> dict[str, float]:
+        """Convert to dictionary."""
+        return {
+            "pbo": self.pbo,
+            "pbo_pct": self.pbo_pct,
+            "n_combinations": self.n_combinations,
+            "n_strategies": self.n_strategies,
+            "is_best_rank_oos_median": self.is_best_rank_oos_median,
+            "is_best_rank_oos_mean": self.is_best_rank_oos_mean,
+            "degradation_mean": self.degradation_mean,
+            "degradation_std": self.degradation_std,
+        }
+
+
+def compute_pbo(
+    is_performance: np.ndarray[Any, np.dtype[Any]],
+    oos_performance: np.ndarray[Any, np.dtype[Any]],
+) -> PBOResult:
+    """Compute Probability of Backtest Overfitting (PBO).
+
+    PBO measures the probability that a strategy selected as best in-sample
+    performs below median out-of-sample. A high PBO indicates overfitting.
+
+    Definition
+    ----------
+    From Bailey & López de Prado (2014):
+
+    .. math::
+
+        PBO = P(rank_{OOS}(\\arg\\max_{IS}) > N/2)
+
+    In plain English: what's the probability that the best in-sample strategy
+    ranks in the bottom half out-of-sample?
+
+    Interpretation
+    --------------
+    - PBO = 0%: No overfitting (best IS is also best OOS)
+    - PBO = 50%: Random selection (IS performance uncorrelated with OOS)
+    - PBO > 50%: Severe overfitting (IS selection is counterproductive)
+
+    Parameters
+    ----------
+    is_performance : np.ndarray, shape (n_folds, n_strategies) or (n_combinations,)
+        In-sample performance metrics (Sharpe, IC, returns) for each strategy.
+    oos_performance : np.ndarray, shape (n_folds, n_strategies) or (n_combinations,)
+        Out-of-sample performance metrics (same structure as is_performance).
+
+    Returns
+    -------
+    PBOResult
+        Result object with PBO and diagnostic metrics.
+        Call .interpret() for human-readable assessment.
+
+    Raises
+    ------
+    ValueError
+        If arrays have different shapes or fewer than 2 strategies.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> # 10 CV folds, 5 strategies
+    >>> is_perf = np.random.randn(10, 5)
+    >>> oos_perf = np.random.randn(10, 5)
+    >>> result = compute_pbo(is_perf, oos_perf)
+    >>> print(result.interpret())
+
+    References
+    ----------
+    Bailey, D. H., & López de Prado, M. (2014). "The Probability of Backtest
+    Overfitting." Journal of Computational Finance, 20(4), 39-69.
+    """
+    is_performance = np.asarray(is_performance)
+    oos_performance = np.asarray(oos_performance)
+
+    if is_performance.shape != oos_performance.shape:
+        raise ValueError(
+            f"is_performance and oos_performance must have same shape. "
+            f"Got {is_performance.shape} vs {oos_performance.shape}"
+        )
+
+    # Handle 1D input (single combination with multiple strategies)
+    if is_performance.ndim == 1:
+        is_performance = is_performance.reshape(1, -1)
+        oos_performance = oos_performance.reshape(1, -1)
+
+    n_combinations, n_strategies = is_performance.shape
+
+    if n_strategies < 2:
+        raise ValueError(f"Need at least 2 strategies, got {n_strategies}")
+
+    # For each combination, find the IS-best strategy and its OOS rank
+    is_best_oos_ranks = []
+    degradations = []
+
+    for i in range(n_combinations):
+        is_row = is_performance[i, :]
+        oos_row = oos_performance[i, :]
+
+        # Find strategy with best IS performance
+        is_best_idx = np.argmax(is_row)
+        is_best_is_perf = is_row[is_best_idx]
+        is_best_oos_perf = oos_row[is_best_idx]
+
+        # Compute OOS rank of IS-best strategy (1 = best, N = worst)
+        oos_ranks = n_strategies - np.argsort(np.argsort(oos_row))
+        is_best_oos_rank = oos_ranks[is_best_idx]
+        is_best_oos_ranks.append(is_best_oos_rank)
+
+        # Compute degradation (IS - OOS performance)
+        degradations.append(is_best_is_perf - is_best_oos_perf)
+
+    ranks_arr = np.array(is_best_oos_ranks)
+    degrad_arr = np.array(degradations)
+
+    # PBO = P(IS-best ranks in bottom half OOS)
+    median_rank = (n_strategies + 1) / 2
+    n_below_median = np.sum(ranks_arr > median_rank)
+    pbo = n_below_median / n_combinations
+
+    return PBOResult(
+        pbo=float(pbo),
+        pbo_pct=float(pbo * 100),
+        n_combinations=int(n_combinations),
+        n_strategies=int(n_strategies),
+        is_best_rank_oos_median=float(np.median(ranks_arr)),
+        is_best_rank_oos_mean=float(np.mean(ranks_arr)),
+        degradation_mean=float(np.mean(degrad_arr)),
+        degradation_std=float(np.std(degrad_arr)),
+    )
+
+
+__all__ = [
+    "PBOResult",
+    "compute_pbo",
+]

ml4t/diagnostic/evaluation/stats/bootstrap.py

@@ -0,0 +1,228 @@
+"""Bootstrap methods for statistical inference on time series data.
+
+This module implements bootstrap methods that preserve temporal dependence
+structure, which is critical for financial time series:
+- Stationary bootstrap (Politis & Romano, 1994)
+- Block bootstrap variants
+
+These methods are essential for valid statistical inference when data
+exhibits autocorrelation, which is common in financial returns.
+"""
+
+import warnings
+from typing import TYPE_CHECKING, Any, Union
+
+import numpy as np
+import pandas as pd
+import polars as pl
+from scipy.stats import spearmanr
+
+from ml4t.diagnostic.backends.adapter import DataFrameAdapter
+
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
+
+def stationary_bootstrap_ic(
+    predictions: Union[pl.Series, pd.Series, "NDArray[Any]"],
+    returns: Union[pl.Series, pd.Series, "NDArray[Any]"],
+    n_samples: int = 1000,
+    block_size: float | None = None,
+    confidence_level: float = 0.95,
+    return_details: bool = True,
+) -> float | dict[str, Any]:
+    """Calculate p-value and confidence intervals for IC using stationary bootstrap.
+
+    This method is more rigorous than the HAC approximation as it:
+    1. Preserves the temporal dependence structure of the data
+    2. Does not rely on asymptotic approximations for rank correlation
+    3. Provides accurate confidence intervals for finite samples
+
+    The stationary bootstrap (Politis & Romano, 1994) generates bootstrap samples
+    by resampling blocks of random length from the original data, preserving the
+    weak dependence structure of the time series.
+
+    Parameters
+    ----------
+    predictions : array-like
+        Model predictions or signals
+    returns : array-like
+        Actual returns or target values
+    n_samples : int, default=1000
+        Number of bootstrap samples to generate
+    block_size : float, optional
+        Expected block size for the stationary bootstrap.
+        If None, uses optimal block size based on data autocorrelation.
+    confidence_level : float, default=0.95
+        Confidence level for the confidence interval
+    return_details : bool, default=True
+        If True, returns detailed results including CI and p-value
+
+    Returns
+    -------
+    float or dict
+        If return_details=False: p-value for the null hypothesis (IC=0)
+        If return_details=True: Dictionary containing IC, p_value, CI, etc.
+
+    References
+    ----------
+    Politis, D. N., & Romano, J. P. (1994). The stationary bootstrap.
+    Journal of the American Statistical Association, 89(428), 1303-1313.
+    """
+    # Convert to numpy arrays
+    pred_array = DataFrameAdapter.to_numpy(predictions).flatten()
+    ret_array = DataFrameAdapter.to_numpy(returns).flatten()
+
+    # Validate inputs
+    if len(pred_array) != len(ret_array):
+        raise ValueError("Predictions and returns must have the same length")
+
+    # Remove NaN pairs
+    valid_mask = ~(np.isnan(pred_array) | np.isnan(ret_array))
+    pred_clean = pred_array[valid_mask]
+    ret_clean = ret_array[valid_mask]
+
+    n = len(pred_clean)
+    if n < 30:
+        warnings.warn(
+            f"Sample size ({n}) may be too small for reliable bootstrap inference",
+            stacklevel=2,
+        )
+
+    # Calculate observed IC
+    observed_ic, _ = spearmanr(pred_clean, ret_clean)
+
+    if np.isnan(observed_ic):
+        if return_details:
+            return {
+                "ic": np.nan,
+                "p_value": np.nan,
+                "ci_lower": np.nan,
+                "ci_upper": np.nan,
+                "bootstrap_mean": np.nan,
+                "bootstrap_std": np.nan,
+            }
+        return np.nan
+
+    # Determine optimal block size if not provided
+    if block_size is None:
+        block_size = _optimal_block_size(ret_clean)
+
+    # Generate bootstrap samples under null hypothesis
+    bootstrap_ics_null = np.zeros(n_samples)
+
+    for i in range(n_samples):
+        # Generate stationary bootstrap sample
+        boot_indices = _stationary_bootstrap_indices(n, block_size)
+        # Break relationship by independently bootstrapping predictions
+        boot_pred_null = pred_clean[_stationary_bootstrap_indices(n, block_size)]
+        boot_ret = ret_clean[boot_indices]
+
+        # Calculate IC on bootstrap sample
+        ic_boot, _ = spearmanr(boot_pred_null, boot_ret)
+        bootstrap_ics_null[i] = ic_boot if not np.isnan(ic_boot) else 0.0
+
+    # Calculate p-value (two-tailed test)
+    p_value = np.mean(np.abs(bootstrap_ics_null) >= np.abs(observed_ic))
+
+    # Calculate confidence interval using percentile method
+    bootstrap_ics_actual = np.zeros(n_samples)
+    for i in range(n_samples):
+        boot_indices = _stationary_bootstrap_indices(n, block_size)
+        boot_pred = pred_clean[boot_indices]
+        boot_ret = ret_clean[boot_indices]
+        ic_boot, _ = spearmanr(boot_pred, boot_ret)
+        bootstrap_ics_actual[i] = ic_boot if not np.isnan(ic_boot) else observed_ic
+
+    alpha = 1 - confidence_level
+    ci_lower = np.percentile(bootstrap_ics_actual, 100 * alpha / 2)
+    ci_upper = np.percentile(bootstrap_ics_actual, 100 * (1 - alpha / 2))
+
+    if not return_details:
+        return float(p_value)
+
+    return {
+        "ic": float(observed_ic),
+        "p_value": float(p_value),
+        "ci_lower": float(ci_lower),
+        "ci_upper": float(ci_upper),
+        "bootstrap_mean": float(np.mean(bootstrap_ics_actual)),
+        "bootstrap_std": float(np.std(bootstrap_ics_actual)),
+    }
+
+
+def _stationary_bootstrap_indices(n: int, block_size: float) -> "NDArray[np.int_]":
+    """Generate indices for one stationary bootstrap sample.
+
+    Parameters
+    ----------
+    n : int
+        Sample size
+    block_size : float
+        Expected block size (1/p where p is the probability of ending a block)
+
+    Returns
+    -------
+    np.ndarray
+        Bootstrap indices of length n
+    """
+    p = 1.0 / block_size  # Probability of ending a block
+    indices: list[int] = []
+
+    while len(indices) < n:
+        # Start a new block at a random position
+        start_idx = np.random.randint(0, n)
+        # Generate block length from geometric distribution
+        block_length = np.random.geometric(p)
+        # Add indices from this block (with wrapping)
+        for j in range(block_length):
+            if len(indices) >= n:
+                break
+            indices.append((start_idx + j) % n)
+
+    return np.array(indices[:n], dtype=np.int_)
+
+
+def _optimal_block_size(data: "NDArray[Any]") -> float:
+    """Estimate optimal block size for stationary bootstrap using autocorrelation.
+
+    Uses a simple rule based on lag-1 autocorrelation to determine block size.
+    Higher autocorrelation requires larger blocks to preserve dependence structure.
+
+    Parameters
+    ----------
+    data : np.ndarray
+        Time series data
+
+    Returns
+    -------
+    float
+        Optimal block size
+    """
+    n = len(data)
+
+    if n < 10:
+        return max(1, n // 3)
+
+    # Standardize the data
+    data_std = (data - np.mean(data)) / (np.std(data) + 1e-10)
+
+    # Calculate lag-1 autocorrelation
+    acf_1 = np.corrcoef(data_std[:-1], data_std[1:])[0, 1]
+
+    # Simple rule: block size increases with autocorrelation
+    if np.isnan(acf_1) or acf_1 < 0:
+        block_size = max(1, int(n ** (1 / 3)))
+    else:
+        # Positive autocorrelation: larger blocks needed
+        block_size = max(1, int(n ** (1 / 3) * (1 + 2 * acf_1)))
+
+    # Cap at n/3 to ensure reasonable variation
+    return min(block_size, n // 3)
+
+
+__all__ = [
+    "stationary_bootstrap_ic",
+    "_stationary_bootstrap_indices",
+    "_optimal_block_size",
+]