ml4t-diagnostic 0.1.0a1__py3-none-any.whl

ml4t/diagnostic/evaluation/trade_shap/characterize.py

@@ -0,0 +1,413 @@
+"""Pattern characterization with proper statistical testing.
+
+This module provides PatternCharacterizer for characterizing error patterns
+identified through clustering, with:
+- Welch's t-test (doesn't assume equal variance)
+- Mann-Whitney U test (non-parametric)
+- Benjamini-Hochberg FDR correction for multiple testing
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+from scipy import stats
+
+from ml4t.diagnostic.evaluation.trade_shap.models import ErrorPattern
+
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
+
+@dataclass
+class CharacterizationConfig:
+    """Configuration for pattern characterization.
+
+    Attributes:
+        alpha: Significance level for statistical tests (default: 0.05)
+        top_n_features: Number of top features to include in characterization
+        use_fdr_correction: Whether to apply Benjamini-Hochberg FDR correction
+        min_samples_per_test: Minimum samples needed for each group in t-test
+    """
+
+    alpha: float = 0.05
+    top_n_features: int = 5
+    use_fdr_correction: bool = True
+    min_samples_per_test: int = 3
+
+
+@dataclass
+class FeatureStatistics:
+    """Statistical test results for a single feature.
+
+    Attributes:
+        feature_name: Name of the feature
+        mean_shap: Mean SHAP value in the cluster
+        mean_shap_other: Mean SHAP value in other clusters
+        p_value_t: P-value from Welch's t-test
+        p_value_mw: P-value from Mann-Whitney U test
+        q_value_t: FDR-corrected p-value (t-test), if correction applied
+        q_value_mw: FDR-corrected p-value (MW test), if correction applied
+        is_significant: Whether the feature is statistically significant
+    """
+
+    feature_name: str
+    mean_shap: float
+    mean_shap_other: float
+    p_value_t: float
+    p_value_mw: float
+    q_value_t: float | None = None
+    q_value_mw: float | None = None
+    is_significant: bool = False
+
+
+def benjamini_hochberg(
+    p_values: list[float], alpha: float = 0.05
+) -> tuple[list[float], list[bool]]:
+    """Apply Benjamini-Hochberg FDR correction to p-values.
+
+    Args:
+        p_values: List of raw p-values
+        alpha: Significance level (default: 0.05)
+
+    Returns:
+        Tuple of (q_values, is_significant) where:
+        - q_values: FDR-adjusted p-values (monotone)
+        - is_significant: Boolean mask for significant results
+
+    Note:
+        BH procedure controls False Discovery Rate (FDR) - the expected
+        proportion of false discoveries among rejected hypotheses.
+        This is less conservative than Bonferroni correction.
+    """
+    if not p_values:
+        return [], []
+
+    n = len(p_values)
+    p_array = np.asarray(p_values)
+
+    # Sort p-values and track original order
+    sorted_indices = np.argsort(p_array)
+    sorted_p = p_array[sorted_indices]
+
+    # BH adjustment: q_i = min(p_i * n / rank, 1.0)
+    # Then enforce monotonicity from largest to smallest
+    ranks = np.arange(1, n + 1)
+    q_sorted = np.minimum(sorted_p * n / ranks, 1.0)
+
+    # Enforce monotonicity: q[i] = min(q[i], q[i+1], ..., q[n])
+    # Process from end to start
+    for i in range(n - 2, -1, -1):
+        q_sorted[i] = min(q_sorted[i], q_sorted[i + 1])
+
+    # Restore original order
+    q_values = np.empty(n)
+    q_values[sorted_indices] = q_sorted
+
+    # Determine significance
+    is_significant = q_values < alpha
+
+    return q_values.tolist(), is_significant.tolist()
+
+
+class PatternCharacterizer:
+    """Characterizes error patterns with proper statistical testing.
+
+    Uses Welch's t-test (doesn't assume equal variance) and Mann-Whitney U test,
+    with optional Benjamini-Hochberg FDR correction for multiple testing.
+
+    Attributes:
+        config: Characterization configuration
+        feature_names: List of all feature names
+
+    Example:
+        >>> characterizer = PatternCharacterizer(feature_names)
+        >>> pattern = characterizer.characterize_cluster(
+        ...     cluster_shap=cluster_vectors,
+        ...     other_shap=other_vectors,
+        ...     cluster_id=0,
+        ... )
+        >>> print(pattern.top_features)
+    """
+
+    def __init__(
+        self,
+        feature_names: list[str],
+        config: CharacterizationConfig | None = None,
+    ) -> None:
+        """Initialize characterizer.
+
+        Args:
+            feature_names: List of all feature names
+            config: Characterization configuration (uses defaults if None)
+        """
+        self.feature_names = feature_names
+        self.config = config or CharacterizationConfig()
+
+    def characterize_cluster(
+        self,
+        cluster_shap: NDArray[np.floating[Any]],
+        other_shap: NDArray[np.floating[Any]],
+        cluster_id: int,
+        centroids: NDArray[np.floating[Any]] | None = None,
+    ) -> ErrorPattern:
+        """Characterize a single cluster as an error pattern.
+
+        Args:
+            cluster_shap: SHAP vectors for trades in this cluster (n_cluster x n_features)
+            other_shap: SHAP vectors for all other trades (n_other x n_features)
+            cluster_id: Cluster identifier (0-indexed)
+            centroids: Optional cluster centroids for separation score calculation
+
+        Returns:
+            ErrorPattern with statistical characterization
+        """
+        n_trades = cluster_shap.shape[0]
+        n_features = len(self.feature_names)
+
+        # Compute mean SHAP per feature for this cluster
+        mean_shap_cluster = np.mean(cluster_shap, axis=0)
+        mean_shap_other = (
+            np.mean(other_shap, axis=0) if len(other_shap) > 0 else np.zeros(n_features)
+        )
+
+        # Statistical tests for each feature
+        feature_stats = self._compute_feature_statistics(
+            cluster_shap, other_shap, mean_shap_cluster, mean_shap_other
+        )
+
+        # Apply FDR correction if configured
+        if self.config.use_fdr_correction:
+            feature_stats = self._apply_fdr_correction(feature_stats)
+
+        # Sort by absolute mean SHAP (descending)
+        feature_stats.sort(key=lambda x: abs(x.mean_shap), reverse=True)
+
+        # Take top N
+        top_stats = feature_stats[: self.config.top_n_features]
+
+        # Build top_features tuple list for ErrorPattern
+        top_features = [
+            (
+                fs.feature_name,
+                fs.mean_shap,
+                fs.p_value_t,
+                fs.p_value_mw,
+                fs.is_significant,
+            )
+            for fs in top_stats
+        ]
+
+        # Generate pattern description
+        description = self._generate_description(top_stats)
+
+        # Compute separation and distinctiveness scores
+        separation_score = self._compute_separation_score(mean_shap_cluster, centroids, cluster_id)
+        distinctiveness = self._compute_distinctiveness(mean_shap_cluster, mean_shap_other)
+
+        return ErrorPattern(
+            cluster_id=cluster_id,
+            n_trades=n_trades,
+            description=description,
+            top_features=top_features,
+            separation_score=separation_score,
+            distinctiveness=distinctiveness,
+        )
+
+    def _compute_feature_statistics(
+        self,
+        cluster_shap: NDArray[np.floating[Any]],
+        other_shap: NDArray[np.floating[Any]],
+        mean_shap_cluster: NDArray[np.floating[Any]],
+        mean_shap_other: NDArray[np.floating[Any]],
+    ) -> list[FeatureStatistics]:
+        """Compute statistical tests for each feature.
+
+        Uses Welch's t-test (equal_var=False) instead of standard t-test
+        to handle unequal variances between groups.
+        """
+        results = []
+
+        for idx, feature_name in enumerate(self.feature_names):
+            cluster_values = cluster_shap[:, idx]
+            other_values = other_shap[:, idx] if len(other_shap) > 0 else np.array([])
+
+            # Skip if insufficient samples
+            if (
+                len(cluster_values) < self.config.min_samples_per_test
+                or len(other_values) < self.config.min_samples_per_test
+            ):
+                results.append(
+                    FeatureStatistics(
+                        feature_name=feature_name,
+                        mean_shap=float(mean_shap_cluster[idx]),
+                        mean_shap_other=float(mean_shap_other[idx]),
+                        p_value_t=1.0,
+                        p_value_mw=1.0,
+                        is_significant=False,
+                    )
+                )
+                continue
+
+            # Welch's t-test (doesn't assume equal variance)
+            # This is the key fix: using equal_var=False
+            try:
+                t_stat, p_value_t = stats.ttest_ind(cluster_values, other_values, equal_var=False)
+                p_value_t = float(p_value_t) if not np.isnan(p_value_t) else 1.0
+            except Exception:
+                p_value_t = 1.0
+
+            # Mann-Whitney U test (non-parametric)
+            try:
+                _, p_value_mw = stats.mannwhitneyu(
+                    cluster_values, other_values, alternative="two-sided"
+                )
+                p_value_mw = float(p_value_mw) if not np.isnan(p_value_mw) else 1.0
+            except ValueError:
+                # Can fail if all values are identical
+                p_value_mw = 1.0
+
+            results.append(
+                FeatureStatistics(
+                    feature_name=feature_name,
+                    mean_shap=float(mean_shap_cluster[idx]),
+                    mean_shap_other=float(mean_shap_other[idx]),
+                    p_value_t=p_value_t,
+                    p_value_mw=p_value_mw,
+                    # Will be set after FDR correction
+                    is_significant=False,
+                )
+            )
+
+        return results
+
+    def _apply_fdr_correction(
+        self, feature_stats: list[FeatureStatistics]
+    ) -> list[FeatureStatistics]:
+        """Apply Benjamini-Hochberg FDR correction to all p-values.
+
+        This corrects for multiple testing across all features, reducing
+        false positive rate at the cost of some statistical power.
+        """
+        if not feature_stats:
+            return feature_stats
+
+        # Collect p-values
+        p_values_t = [fs.p_value_t for fs in feature_stats]
+        p_values_mw = [fs.p_value_mw for fs in feature_stats]
+
+        # Apply BH correction
+        q_values_t, sig_t = benjamini_hochberg(p_values_t, self.config.alpha)
+        q_values_mw, sig_mw = benjamini_hochberg(p_values_mw, self.config.alpha)
+
+        # Update statistics with corrected values
+        corrected = []
+        for i, fs in enumerate(feature_stats):
+            # Significant if either test rejects after FDR correction
+            is_sig = sig_t[i] or sig_mw[i]
+
+            corrected.append(
+                FeatureStatistics(
+                    feature_name=fs.feature_name,
+                    mean_shap=fs.mean_shap,
+                    mean_shap_other=fs.mean_shap_other,
+                    p_value_t=fs.p_value_t,
+                    p_value_mw=fs.p_value_mw,
+                    q_value_t=q_values_t[i],
+                    q_value_mw=q_values_mw[i],
+                    is_significant=is_sig,
+                )
+            )
+
+        return corrected
+
+    def _generate_description(self, top_stats: list[FeatureStatistics]) -> str:
+        """Generate human-readable pattern description."""
+        if not top_stats:
+            return "Unknown pattern"
+
+        # Filter to significant features only
+        sig_features = [fs for fs in top_stats if fs.is_significant]
+
+        # Fall back to top features if none significant
+        features_to_use = sig_features[:3] if sig_features else top_stats[:2]
+
+        components = []
+        for fs in features_to_use:
+            direction = "High" if fs.mean_shap > 0 else "Low"
+            arrow = "↑" if fs.mean_shap > 0 else "↓"
+            components.append(f"{direction} {fs.feature_name} ({arrow}{fs.mean_shap:.2f})")
+
+        if len(components) == 1:
+            return f"{components[0]} → Losses"
+        return " + ".join(components) + " → Losses"
+
+    def _compute_separation_score(
+        self,
+        centroid: NDArray[np.floating[Any]],
+        all_centroids: NDArray[np.floating[Any]] | None,
+        cluster_id: int,
+    ) -> float:
+        """Compute separation score (distance to nearest other cluster)."""
+        if all_centroids is None or len(all_centroids) <= 1:
+            return 0.0
+
+        min_distance = float("inf")
+        for i, other_centroid in enumerate(all_centroids):
+            if i != cluster_id:
+                distance = float(np.linalg.norm(centroid - other_centroid))
+                min_distance = min(min_distance, distance)
+
+        return min_distance if min_distance != float("inf") else 0.0
+
+    def _compute_distinctiveness(
+        self,
+        cluster_centroid: NDArray[np.floating[Any]],
+        other_mean: NDArray[np.floating[Any]],
+    ) -> float:
+        """Compute distinctiveness (ratio of max SHAP vs other clusters)."""
+        max_cluster = np.max(np.abs(cluster_centroid))
+        max_other = np.max(np.abs(other_mean))
+
+        if max_other == 0:
+            return float(max_cluster) if max_cluster > 0 else 1.0
+
+        return float(max_cluster / max_other)
+
+    def characterize_all_clusters(
+        self,
+        shap_vectors: NDArray[np.floating[Any]],
+        cluster_labels: list[int],
+        n_clusters: int,
+        centroids: NDArray[np.floating[Any]] | None = None,
+    ) -> list[ErrorPattern]:
+        """Characterize all clusters.
+
+        Args:
+            shap_vectors: All SHAP vectors (n_samples x n_features)
+            cluster_labels: Cluster assignment for each sample
+            n_clusters: Total number of clusters
+            centroids: Optional cluster centroids
+
+        Returns:
+            List of ErrorPattern for each cluster
+        """
+        labels_array = np.asarray(cluster_labels)
+        patterns = []
+
+        for cluster_id in range(n_clusters):
+            mask = labels_array == cluster_id
+            cluster_shap = shap_vectors[mask]
+            other_shap = shap_vectors[~mask]
+
+            pattern = self.characterize_cluster(
+                cluster_shap=cluster_shap,
+                other_shap=other_shap,
+                cluster_id=cluster_id,
+                centroids=centroids,
+            )
+            patterns.append(pattern)
+
+        return patterns

ml4t/diagnostic/evaluation/trade_shap/cluster.py

@@ -0,0 +1,302 @@
+"""Hierarchical clustering for trade error patterns.
+
+Provides clustering of SHAP vectors to identify distinct error patterns,
+with proper handling of small sample sizes.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Literal
+
+import numpy as np
+
+from ml4t.diagnostic.evaluation.trade_shap.models import ClusteringResult
+
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
+
+DistanceMetric = Literal["euclidean", "cosine", "correlation", "cityblock"]
+LinkageMethod = Literal["ward", "average", "complete", "single"]
+
+
+@dataclass
+class ClusteringConfig:
+    """Configuration for hierarchical clustering.
+
+    Attributes:
+        distance_metric: Distance metric for pdist ('euclidean', 'cosine', etc.)
+        linkage_method: Linkage method for hierarchical clustering
+        min_cluster_size: Minimum trades per cluster
+        min_trades_for_clustering: Minimum trades required to attempt clustering
+    """
+
+    distance_metric: DistanceMetric = "euclidean"
+    linkage_method: LinkageMethod = "ward"
+    min_cluster_size: int = 5
+    min_trades_for_clustering: int = 10
+
+
+def find_optimal_clusters(
+    linkage_matrix: NDArray[np.floating[Any]],
+    n_samples: int,
+    min_cluster_size: int = 5,
+) -> int:
+    """Find optimal number of clusters using elbow method.
+
+    Uses the acceleration of merge distances (second derivative) to find
+    the "elbow" point in the dendrogram.
+
+    Args:
+        linkage_matrix: Linkage matrix from hierarchical clustering
+        n_samples: Total number of samples
+        min_cluster_size: Minimum samples per cluster
+
+    Returns:
+        Optimal number of clusters respecting min_cluster_size constraint
+
+    Note:
+        The key fix here is respecting min_cluster_size even when that means
+        returning 1 cluster. Previously, the code would force 2 clusters even
+        when there weren't enough samples to support min_cluster_size per cluster.
+    """
+    # Get merge distances (last column of linkage matrix)
+    distances = linkage_matrix[:, 2]
+
+    # Compute first derivative (rate of change)
+    first_deriv = np.diff(distances)
+
+    # Compute second derivative (acceleration)
+    second_deriv = np.diff(first_deriv)
+
+    # Find elbow: Maximum acceleration point
+    if len(second_deriv) > 0:
+        elbow_idx = int(np.argmax(second_deriv))
+        # Convert index to number of clusters
+        # linkage_matrix has (n_samples - 1) rows
+        n_clusters = max(1, n_samples - elbow_idx - 2)
+    else:
+        # Fallback: sqrt(n) heuristic
+        n_clusters = max(1, int(np.sqrt(n_samples)))
+
+    # CRITICAL FIX: Respect min_cluster_size constraint
+    # max_clusters is at least 1 to avoid edge case where we'd return 0
+    max_clusters = max(1, n_samples // min_cluster_size)
+    n_clusters = min(n_clusters, max_clusters)
+
+    # Only force at least 2 clusters if we have room for them
+    # This is the bug fix: don't force 2 if max_clusters < 2
+    if max_clusters >= 2:
+        n_clusters = max(2, n_clusters)
+
+    return int(n_clusters)
+
+
+def compute_cluster_sizes(
+    labels: NDArray[np.intp] | list[int],
+    n_clusters: int,
+) -> list[int]:
+    """Compute number of samples in each cluster using vectorized bincount.
+
+    Args:
+        labels: Cluster assignment for each sample (0-indexed)
+        n_clusters: Total number of clusters
+
+    Returns:
+        List of cluster sizes
+    """
+    labels_array = np.asarray(labels, dtype=np.intp)
+    counts = np.bincount(labels_array, minlength=n_clusters)
+    return counts.tolist()
+
+
+def compute_centroids(
+    vectors: NDArray[np.floating[Any]],
+    labels: NDArray[np.intp] | list[int],
+    n_clusters: int,
+) -> NDArray[np.floating[Any]]:
+    """Compute cluster centroids (mean vector per cluster) using vectorized operations.
+
+    Args:
+        vectors: SHAP vectors of shape (n_samples, n_features)
+        labels: Cluster assignment for each sample (0-indexed)
+        n_clusters: Total number of clusters
+
+    Returns:
+        Centroids of shape (n_clusters, n_features)
+    """
+    labels_array = np.asarray(labels, dtype=np.intp)
+    n_features = vectors.shape[1]
+
+    centroids = np.zeros((n_clusters, n_features), dtype=np.float64)
+
+    for k in range(n_clusters):
+        mask = labels_array == k
+        if np.any(mask):
+            centroids[k] = vectors[mask].mean(axis=0)
+
+    return centroids
+
+
+class HierarchicalClusterer:
+    """Hierarchical clustering for SHAP vectors.
+
+    Provides clustering of trade SHAP vectors to identify distinct error patterns,
+    with quality metrics and dendrogram support.
+
+    Attributes:
+        config: Clustering configuration
+
+    Example:
+        >>> clusterer = HierarchicalClusterer()
+        >>> result = clusterer.cluster(shap_vectors, n_clusters=3)
+        >>> print(f"Silhouette: {result.silhouette_score:.3f}")
+    """
+
+    def __init__(self, config: ClusteringConfig | None = None) -> None:
+        """Initialize clusterer.
+
+        Args:
+            config: Clustering configuration (uses defaults if None)
+        """
+        self.config = config or ClusteringConfig()
+
+    def cluster(
+        self,
+        vectors: NDArray[np.floating[Any]],
+        n_clusters: int | None = None,
+    ) -> ClusteringResult:
+        """Cluster SHAP vectors using hierarchical clustering.
+
+        Args:
+            vectors: SHAP vectors of shape (n_samples, n_features)
+            n_clusters: Number of clusters (auto-determined if None)
+
+        Returns:
+            ClusteringResult with assignments, linkage matrix, and quality metrics
+
+        Raises:
+            ValueError: If insufficient samples or invalid input shape
+            ImportError: If scipy is not installed
+        """
+        # Validate inputs
+        if vectors.size == 0:
+            raise ValueError("Cannot cluster empty vectors")
+
+        if vectors.ndim != 2:
+            raise ValueError(
+                f"vectors must be 2D array (n_samples, n_features), got shape {vectors.shape}"
+            )
+
+        n_samples, n_features = vectors.shape
+
+        if n_samples < self.config.min_trades_for_clustering:
+            raise ValueError(
+                f"Insufficient samples for clustering: {n_samples} < "
+                f"{self.config.min_trades_for_clustering}"
+            )
+
+        # Import scipy
+        try:
+            import scipy.cluster.hierarchy as sch
+            from scipy.spatial.distance import pdist
+        except ImportError as e:
+            raise ImportError(
+                "scipy required for clustering. Install with: pip install scipy"
+            ) from e
+
+        # Compute pairwise distances
+        distances = pdist(vectors, metric=self.config.distance_metric)
+
+        # Perform hierarchical clustering
+        linkage_matrix = sch.linkage(distances, method=self.config.linkage_method)
+
+        # Determine number of clusters
+        if n_clusters is None:
+            n_clusters = find_optimal_clusters(
+                linkage_matrix, n_samples, self.config.min_cluster_size
+            )
+
+        # Cut dendrogram to get cluster assignments
+        labels = sch.fcluster(linkage_matrix, t=n_clusters, criterion="maxclust")
+        # fcluster returns 1-indexed labels, convert to 0-indexed
+        labels = labels - 1
+
+        # Compute cluster metrics
+        cluster_sizes = compute_cluster_sizes(labels, n_clusters)
+        centroids = compute_centroids(vectors, labels, n_clusters)
+
+        # Compute quality metrics
+        silhouette = self._compute_silhouette(vectors, labels)
+        davies_bouldin = self._compute_davies_bouldin(vectors, labels)
+        calinski_harabasz = self._compute_calinski_harabasz(vectors, labels)
+
+        return ClusteringResult(
+            n_clusters=n_clusters,
+            cluster_assignments=labels.tolist(),
+            linkage_matrix=linkage_matrix,
+            centroids=centroids,
+            silhouette_score=silhouette,
+            davies_bouldin_score=davies_bouldin,
+            calinski_harabasz_score=calinski_harabasz,
+            cluster_sizes=cluster_sizes,
+            distance_metric=self.config.distance_metric,
+            linkage_method=self.config.linkage_method,
+        )
+
+    def _compute_silhouette(
+        self,
+        vectors: NDArray[np.floating[Any]],
+        labels: NDArray[np.intp],
+    ) -> float:
+        """Compute silhouette score for clustering quality.
+
+        Returns:
+            Silhouette score (-1 to 1, higher is better)
+        """
+        try:
+            from sklearn.metrics import silhouette_score
+
+            # Need at least 2 clusters for silhouette
+            unique_labels = np.unique(labels)
+            if len(unique_labels) < 2:
+                return 0.0
+
+            return float(silhouette_score(vectors, labels))
+        except ImportError:
+            return 0.0
+
+    def _compute_davies_bouldin(
+        self,
+        vectors: NDArray[np.floating[Any]],
+        labels: NDArray[np.intp],
+    ) -> float | None:
+        """Compute Davies-Bouldin index (lower is better)."""
+        try:
+            from sklearn.metrics import davies_bouldin_score
+
+            unique_labels = np.unique(labels)
+            if len(unique_labels) < 2:
+                return None
+
+            return float(davies_bouldin_score(vectors, labels))
+        except ImportError:
+            return None
+
+    def _compute_calinski_harabasz(
+        self,
+        vectors: NDArray[np.floating[Any]],
+        labels: NDArray[np.intp],
+    ) -> float | None:
+        """Compute Calinski-Harabasz score (higher is better)."""
+        try:
+            from sklearn.metrics import calinski_harabasz_score
+
+            unique_labels = np.unique(labels)
+            if len(unique_labels) < 2:
+                return None
+
+            return float(calinski_harabasz_score(vectors, labels))
+        except ImportError:
+            return None