ml4t-diagnostic 0.1.0a1__py3-none-any.whl

ml4t/diagnostic/evaluation/metric_registry.py

@@ -0,0 +1,255 @@
+"""Metric registry for evaluation metrics with metadata.
+
+This module provides a centralized registry for evaluation metrics,
+including directionality (whether higher is better) and tier defaults.
+"""
+
+from collections.abc import Callable
+from typing import Any
+
+
+class MetricRegistry:
+    """Registry of evaluation metrics with metadata.
+
+    The MetricRegistry provides a centralized place to register and query
+    metrics, including their computation functions, directionality (whether
+    higher values are better), and tier defaults.
+
+    Attributes
+    ----------
+    _metrics : dict[str, Callable]
+        Mapping of metric names to computation functions
+    _directionality : dict[str, bool]
+        Mapping of metric names to directionality (True = higher is better)
+    _tier_defaults : dict[int, list[str]]
+        Default metrics for each evaluation tier
+
+    Examples
+    --------
+    >>> registry = MetricRegistry()
+    >>> registry.register("sharpe", sharpe_func, maximize=True, tiers=[1, 2, 3])
+    >>> func = registry.get("sharpe")
+    >>> registry.is_maximize("sharpe")
+    True
+    """
+
+    _instance: "MetricRegistry | None" = None
+
+    def __init__(self) -> None:
+        """Initialize empty registry."""
+        self._metrics: dict[str, Callable[..., Any]] = {}
+        self._directionality: dict[str, bool] = {}
+        self._tier_defaults: dict[int, list[str]] = {1: [], 2: [], 3: []}
+
+    @classmethod
+    def default(cls) -> "MetricRegistry":
+        """Get or create the default singleton registry instance.
+
+        Returns
+        -------
+        MetricRegistry
+            The default registry instance with standard metrics registered
+        """
+        if cls._instance is None:
+            cls._instance = cls()
+            cls._instance._register_defaults()
+        return cls._instance
+
+    @classmethod
+    def reset_default(cls) -> None:
+        """Reset the default singleton instance (primarily for testing)."""
+        cls._instance = None
+
+    def register(
+        self,
+        name: str,
+        func: Callable[..., Any],
+        maximize: bool = True,
+        tiers: list[int] | None = None,
+    ) -> None:
+        """Register a metric with the registry.
+
+        Parameters
+        ----------
+        name : str
+            Unique name for the metric
+        func : Callable
+            Function that computes the metric.
+            Signature: (predictions, actual, strategy_returns) -> float
+        maximize : bool, default True
+            Whether higher values are better (True) or lower (False)
+        tiers : list[int], optional
+            Evaluation tiers where this metric is a default
+        """
+        self._metrics[name] = func
+        self._directionality[name] = maximize
+        if tiers:
+            for tier in tiers:
+                if tier in self._tier_defaults and name not in self._tier_defaults[tier]:
+                    self._tier_defaults[tier].append(name)
+
+    def get(self, name: str) -> Callable[..., Any]:
+        """Get a metric function by name.
+
+        Parameters
+        ----------
+        name : str
+            Name of the metric
+
+        Returns
+        -------
+        Callable
+            The metric computation function
+
+        Raises
+        ------
+        KeyError
+            If metric name is not registered
+        """
+        if name not in self._metrics:
+            raise KeyError(f"Unknown metric: {name}. Available: {list(self._metrics.keys())}")
+        return self._metrics[name]
+
+    def is_maximize(self, name: str) -> bool:
+        """Get whether higher values are better for a metric.
+
+        Parameters
+        ----------
+        name : str
+            Name of the metric
+
+        Returns
+        -------
+        bool
+            True if higher values are better, False otherwise
+        """
+        if name in self._directionality:
+            return self._directionality[name]
+        return self._infer_directionality(name)
+
+    def _infer_directionality(self, name: str) -> bool:
+        """Infer directionality for unknown metrics based on naming conventions."""
+        normalized = name.lower().replace("-", "_").replace(" ", "_")
+
+        if any(term in normalized for term in ["drawdown", "risk", "error", "loss", "volatility"]):
+            return False
+
+        if any(
+            term in normalized
+            for term in ["return", "profit", "gain", "ratio", "score", "coefficient"]
+        ):
+            return True
+
+        return True  # Default to higher is better
+
+    def get_by_tier(self, tier: int) -> list[str]:
+        """Get default metrics for a specific tier.
+
+        Parameters
+        ----------
+        tier : int
+            Evaluation tier (1, 2, or 3)
+
+        Returns
+        -------
+        list[str]
+            List of default metric names for the tier
+        """
+        return self._tier_defaults.get(tier, []).copy()
+
+    def list_metrics(self) -> list[str]:
+        """List all registered metric names.
+
+        Returns
+        -------
+        list[str]
+            Sorted list of metric names
+        """
+        return sorted(self._metrics.keys())
+
+    def __contains__(self, name: str) -> bool:
+        """Check if a metric is registered."""
+        return name in self._metrics
+
+    def _register_defaults(self) -> None:
+        """Register default metrics."""
+        from . import metrics
+
+        # Core metrics with tier assignments
+        self.register(
+            "ic",
+            lambda pred, actual, _returns: metrics.information_coefficient(pred, actual),
+            maximize=True,
+            tiers=[1, 2, 3],
+        )
+        self.register(
+            "sharpe",
+            lambda _pred, _actual, returns: metrics.sharpe_ratio(returns),
+            maximize=True,
+            tiers=[1, 2],
+        )
+        self.register(
+            "sortino",
+            lambda _pred, _actual, returns: metrics.sortino_ratio(returns),
+            maximize=True,
+            tiers=[1],
+        )
+        self.register(
+            "max_drawdown",
+            lambda _pred, _actual, returns: metrics.maximum_drawdown(returns),
+            maximize=False,
+            tiers=[1],
+        )
+        self.register(
+            "hit_rate",
+            lambda pred, actual, _returns: metrics.hit_rate(pred, actual),
+            maximize=True,
+            tiers=[1, 2, 3],
+        )
+
+        # Additional directionality mappings for common metric names
+        self._register_directionality_defaults()
+
+    def _register_directionality_defaults(self) -> None:
+        """Register directionality for common metric names (for get_metric_directionality)."""
+        # Performance metrics (higher is better)
+        for name in [
+            "sharpe_ratio",
+            "sortino_ratio",
+            "calmar",
+            "calmar_ratio",
+            "information_ratio",
+            "omega_ratio",
+            "profit_factor",
+            "total_return",
+            "mean_return",
+            "cumulative_return",
+            "annualized_return",
+            "win_rate",
+            "accuracy",
+            "information_coefficient",
+            "ic_mean",
+            "spearman",
+            "pearson",
+            "r_squared",
+            "r2",
+            "t_statistic",
+            "z_score",
+        ]:
+            self._directionality[name] = True
+
+        # Risk metrics (lower is better)
+        for name in [
+            "maximum_drawdown",
+            "drawdown",
+            "volatility",
+            "downside_deviation",
+            "value_at_risk",
+            "var",
+            "cvar",
+            "conditional_value_at_risk",
+            "tracking_error",
+            "beta",
+            "p_value",
+        ]:
+            self._directionality[name] = False

ml4t/diagnostic/evaluation/metrics/AGENT.md

@@ -0,0 +1,23 @@
+# metrics/ - Feature Metrics
+
+IC, importance, and interaction analysis.
+
+## Modules
+
+| File | Lines | Purpose |
+|------|-------|---------|
+| ic.py | 530 | Core IC functions |
+| ic_statistics.py | 446 | HAC-adjusted IC, decay |
+| conditional_ic.py | 469 | Conditional IC |
+| importance_classical.py | 375 | PFI, MDI |
+| importance_mda.py | 371 | Mean Decrease Accuracy |
+| importance_shap.py | 715 | SHAP importance |
+| importance_analysis.py | 338 | Multi-method comparison |
+| interactions.py | 772 | H-statistic, SHAP interactions |
+| feature_outcome.py | 475 | Feature-outcome analysis |
+| monotonicity.py | 226 | Monotonicity tests |
+| risk_adjusted.py | 324 | Sharpe, Sortino, drawdown |
+
+## Key Functions
+
+`information_coefficient()`, `compute_ic_series()`, `analyze_ml_importance()`, `compute_h_statistic()`, `compute_shap_importance()`

ml4t/diagnostic/evaluation/metrics/__init__.py

@@ -0,0 +1,133 @@
+"""Core performance metrics for financial ML evaluation.
+
+This package implements the core metrics used across ml4t-diagnostic's
+Four-Tier Validation Framework:
+
+- **Tier 3**: Fast screening metrics (IC, hit rate)
+- **Tier 2**: Statistical significance metrics (HAC-adjusted IC, Sharpe with CI)
+- **Tier 1**: Comprehensive metrics (deflated Sharpe, maximum drawdown)
+
+All metrics are implemented with:
+- Mathematical correctness validated by property-based tests
+- Numerical stability for edge cases
+- Polars-native implementation for performance
+- Support for confidence intervals and statistical inference
+
+Submodules
+----------
+ic : Core Information Coefficient calculations
+ic_statistics : HAC-adjusted IC and decay analysis
+conditional_ic : IC conditional on feature quantiles
+monotonicity : Monotonic relationship tests
+risk_adjusted : Sharpe, Sortino, Maximum Drawdown
+basic : Hit rate, forward returns
+feature_outcome : Comprehensive feature-outcome analysis
+importance_classical : Permutation and MDI importance
+importance_shap : SHAP-based importance
+importance_mda : Mean Decrease in Accuracy importance
+importance_analysis : Multi-method importance comparison
+interactions : Feature interaction detection (H-statistic, SHAP)
+"""
+
+# IC metrics
+# Re-export cov_hac from statsmodels for backward compatibility
+from statsmodels.stats.sandwich_covariance import cov_hac
+
+# Basic metrics
+from ml4t.diagnostic.evaluation.metrics.basic import (
+    compute_forward_returns,
+    hit_rate,
+)
+
+# Conditional IC
+from ml4t.diagnostic.evaluation.metrics.conditional_ic import (
+    compute_conditional_ic,
+)
+
+# Feature outcome analysis
+from ml4t.diagnostic.evaluation.metrics.feature_outcome import (
+    analyze_feature_outcome,
+)
+
+# IC statistics
+from ml4t.diagnostic.evaluation.metrics.ic_statistics import (
+    compute_ic_decay,
+    compute_ic_hac_stats,
+)
+from ml4t.diagnostic.evaluation.metrics.importance_analysis import (
+    analyze_ml_importance,
+)
+
+# Importance methods
+from ml4t.diagnostic.evaluation.metrics.importance_classical import (
+    compute_mdi_importance,
+    compute_permutation_importance,
+)
+from ml4t.diagnostic.evaluation.metrics.importance_mda import (
+    compute_mda_importance,
+)
+from ml4t.diagnostic.evaluation.metrics.importance_shap import (
+    compute_shap_importance,
+)
+from ml4t.diagnostic.evaluation.metrics.information_coefficient import (
+    compute_ic_by_horizon,
+    compute_ic_ir,
+    compute_ic_series,
+    information_coefficient,
+)
+
+# Interaction detection
+from ml4t.diagnostic.evaluation.metrics.interactions import (
+    analyze_interactions,
+    compute_h_statistic,
+    compute_shap_interactions,
+)
+
+# Monotonicity
+from ml4t.diagnostic.evaluation.metrics.monotonicity import (
+    compute_monotonicity,
+)
+
+# Risk-adjusted metrics
+from ml4t.diagnostic.evaluation.metrics.risk_adjusted import (
+    maximum_drawdown,
+    sharpe_ratio,
+    sharpe_ratio_with_ci,
+    sortino_ratio,
+)
+
+__all__ = [
+    # IC metrics
+    "information_coefficient",
+    "compute_ic_series",
+    "compute_ic_by_horizon",
+    "compute_ic_ir",
+    # IC statistics
+    "compute_ic_hac_stats",
+    "compute_ic_decay",
+    "cov_hac",  # Re-exported from statsmodels for backward compatibility
+    # Conditional IC
+    "compute_conditional_ic",
+    # Monotonicity
+    "compute_monotonicity",
+    # Risk-adjusted
+    "sharpe_ratio",
+    "sharpe_ratio_with_ci",
+    "maximum_drawdown",
+    "sortino_ratio",
+    # Basic
+    "hit_rate",
+    "compute_forward_returns",
+    # Feature outcome
+    "analyze_feature_outcome",
+    # Importance
+    "compute_permutation_importance",
+    "compute_mdi_importance",
+    "compute_shap_importance",
+    "compute_mda_importance",
+    "analyze_ml_importance",
+    # Interactions
+    "compute_h_statistic",
+    "compute_shap_interactions",
+    "analyze_interactions",
+]

ml4t/diagnostic/evaluation/metrics/basic.py

@@ -0,0 +1,160 @@
+"""Basic metrics: hit rate and forward returns calculation.
+
+This module provides fundamental building blocks for feature evaluation.
+"""
+
+from typing import TYPE_CHECKING, Union, cast
+
+import numpy as np
+import pandas as pd
+import polars as pl
+
+from ml4t.diagnostic.backends.adapter import DataFrameAdapter
+
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
+
+def hit_rate(
+    predictions: Union[pl.Series, pd.Series, "NDArray"],
+    returns: Union[pl.Series, pd.Series, "NDArray"],
+) -> float:
+    """Calculate hit rate (percentage of correct directional predictions).
+
+    Hit rate measures what percentage of predictions correctly identify the
+    direction of subsequent returns (positive/negative).
+
+    Parameters
+    ----------
+    predictions : Union[pl.Series, pd.Series, np.ndarray]
+        Model predictions or scores
+    returns : Union[pl.Series, pd.Series, np.ndarray]
+        Forward returns corresponding to predictions
+
+    Returns
+    -------
+    float
+        Hit rate as a percentage (0-100)
+
+    Examples
+    --------
+    >>> predictions = np.array([0.1, -0.2, 0.3, -0.1])
+    >>> returns = np.array([0.02, -0.01, 0.05, 0.01])  # Note: last one wrong direction
+    >>> hr = hit_rate(predictions, returns)
+    >>> print(f"Hit Rate: {hr:.1f}%")
+    Hit Rate: 75.0%
+    """
+    # Convert inputs to numpy
+    pred_array = DataFrameAdapter.to_numpy(predictions).flatten()
+    ret_array = DataFrameAdapter.to_numpy(returns).flatten()
+
+    # Validate inputs
+    if len(pred_array) != len(ret_array):
+        raise ValueError("Predictions and returns must have the same length")
+
+    # Remove NaN pairs
+    valid_mask = ~(np.isnan(pred_array) | np.isnan(ret_array))
+    pred_clean = pred_array[valid_mask]
+    ret_clean = ret_array[valid_mask]
+
+    if len(pred_clean) == 0:
+        return np.nan
+
+    # Calculate directional accuracy
+    pred_direction = np.sign(pred_clean)
+    ret_direction = np.sign(ret_clean)
+
+    # Count correct predictions (same sign)
+    correct_predictions = pred_direction == ret_direction
+
+    # Handle zero returns/predictions by considering them neutral (correct)
+    zero_mask = (pred_clean == 0) | (ret_clean == 0)
+    correct_predictions[zero_mask] = True  # Conservative approach
+
+    hit_rate_value = np.mean(correct_predictions) * 100
+
+    return float(hit_rate_value)
+
+
+def compute_forward_returns(
+    prices: pl.DataFrame | pd.DataFrame,
+    periods: int | list[int] = 1,
+    price_col: str = "close",
+    group_col: str | None = None,
+) -> pl.DataFrame | pd.DataFrame:
+    """Compute forward returns for given periods.
+
+    This is a helper function for IC analysis, computing the forward-looking
+    returns that will be correlated with predictions/features.
+
+    Parameters
+    ----------
+    prices : Union[pl.DataFrame, pd.DataFrame]
+        Price data with at least price_col and optionally group_col
+    periods : Union[int, list[int]], default 1
+        Forward periods to compute (e.g., [1, 5, 21] for 1d, 1w, 1m)
+    price_col : str, default "close"
+        Column name containing prices
+    group_col : str | None, default None
+        Column for grouping (e.g., 'symbol' for multi-asset)
+
+    Returns
+    -------
+    Union[pl.DataFrame, pd.DataFrame]
+        DataFrame with forward return columns: fwd_ret_1, fwd_ret_5, etc.
+
+    Examples
+    --------
+    >>> prices = pl.DataFrame({
+    ...     "date": ["2024-01-01", "2024-01-02", "2024-01-03"],
+    ...     "close": [100.0, 102.0, 101.0]
+    ... })
+    >>> fwd_returns = compute_forward_returns(prices, periods=[1, 2])
+    >>> print(fwd_returns.columns)
+    ['date', 'close', 'fwd_ret_1', 'fwd_ret_2']
+    """
+    is_polars = isinstance(prices, pl.DataFrame)
+
+    # Ensure periods is a list
+    if isinstance(periods, int):
+        periods = [periods]
+
+    if is_polars:
+        df = cast(pl.DataFrame, prices).clone()
+
+        if group_col is not None:
+            # Group-wise forward returns (e.g., per symbol)
+            for period in periods:
+                col_name = f"fwd_ret_{period}"
+                df = df.with_columns(
+                    [
+                        (
+                            pl.col(price_col).shift(-period).over(group_col) / pl.col(price_col) - 1
+                        ).alias(col_name)
+                    ]
+                )
+        else:
+            # Simple forward returns
+            for period in periods:
+                col_name = f"fwd_ret_{period}"
+                df = df.with_columns(
+                    [(pl.col(price_col).shift(-period) / pl.col(price_col) - 1).alias(col_name)]
+                )
+
+        return df
+
+    # pandas - use different variable name to avoid type conflict
+    df_pd = cast(pd.DataFrame, prices).copy()
+
+    if group_col is not None:
+        # Group-wise forward returns
+        for period in periods:
+            col_name = f"fwd_ret_{period}"
+            df_pd[col_name] = df_pd.groupby(group_col)[price_col].pct_change(period).shift(-period)
+    else:
+        # Simple forward returns
+        for period in periods:
+            col_name = f"fwd_ret_{period}"
+            df_pd[col_name] = df_pd[price_col].pct_change(period).shift(-period)
+
+    return df_pd