ml4t-diagnostic 0.1.0a1__py3-none-any.whl

ml4t/diagnostic/evaluation/trade_dashboard/types.py

@@ -0,0 +1,129 @@
+"""Dashboard data types and configuration.
+
+Provides unified data structures for the dashboard to eliminate dict/object branching.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+
+@dataclass
+class DashboardConfig:
+    """Configuration for the dashboard.
+
+    Attributes
+    ----------
+    allow_pickle_upload : bool
+        Whether to allow uploading pickle files. Disabled by default for security.
+    styled : bool
+        Whether to apply professional CSS styling.
+    title : str
+        Dashboard title.
+    """
+
+    allow_pickle_upload: bool = False  # Security: disabled by default
+    styled: bool = False
+    title: str = "Trade SHAP Diagnostics"
+
+
+@dataclass
+class DashboardBundle:
+    """Unified data container for all dashboard tabs.
+
+    This normalizes the varied input formats (dict vs object, different field names)
+    into a single consistent representation that all tabs can consume.
+
+    Attributes
+    ----------
+    trades_df : pd.DataFrame
+        One row per trade with stable columns:
+        - trade_id: str
+        - entry_time: datetime
+        - exit_time: datetime (optional)
+        - pnl: float
+        - return_pct: float (optional)
+        - symbol: str (optional)
+        Sorted chronologically by entry_time for time-series tests.
+    returns : np.ndarray | None
+        Trade returns array. Prefers return_pct if available, falls back to pnl.
+    returns_label : str
+        What the returns array represents: "return_pct", "pnl", or "none".
+    explanations : list[dict]
+        Normalized explanation dictionaries with stable keys:
+        - trade_id: str
+        - shap_vector: list[float]
+        - top_features: list[tuple[str, float]]
+        - trade_metrics: dict (optional)
+    patterns_df : pd.DataFrame
+        One row per error pattern with stable columns:
+        - cluster_id: int
+        - n_trades: int
+        - description: str
+        - top_features: list[tuple]
+        - hypothesis: str (optional)
+        - actions: list[str] (optional)
+        - confidence: float (optional)
+    n_trades_analyzed : int
+        Total number of trades analyzed.
+    n_trades_explained : int
+        Number of trades successfully explained.
+    n_trades_failed : int
+        Number of trades that failed explanation.
+    failed_trades : list[tuple[str, str]]
+        List of (trade_id, reason) for failed explanations.
+    config : DashboardConfig
+        Dashboard configuration.
+    """
+
+    trades_df: pd.DataFrame
+    returns: np.ndarray | None
+    returns_label: str  # "return_pct" | "pnl" | "none"
+    explanations: list[dict[str, Any]]
+    patterns_df: pd.DataFrame
+    n_trades_analyzed: int
+    n_trades_explained: int
+    n_trades_failed: int
+    failed_trades: list[tuple[str, str]]
+    config: DashboardConfig = field(default_factory=DashboardConfig)
+
+
+@dataclass
+class ReturnSummary:
+    """Summary statistics for a returns series.
+
+    Attributes
+    ----------
+    n_samples : int
+        Number of samples.
+    mean : float
+        Mean return.
+    std : float
+        Standard deviation.
+    sharpe : float
+        Sharpe ratio (mean / std).
+    skewness : float
+        Skewness of distribution.
+    kurtosis : float
+        Kurtosis of distribution (not excess, 3.0 for normal).
+    min_val : float
+        Minimum value.
+    max_val : float
+        Maximum value.
+    win_rate : float
+        Fraction of positive returns.
+    """
+
+    n_samples: int
+    mean: float
+    std: float
+    sharpe: float
+    skewness: float
+    kurtosis: float
+    min_val: float
+    max_val: float
+    win_rate: float

ml4t/diagnostic/evaluation/trade_shap/__init__.py

@@ -0,0 +1,102 @@
+"""Trade-level SHAP diagnostics models.
+
+This package contains the data models for Trade SHAP analysis.
+The main TradeShapAnalyzer and HypothesisGenerator classes are imported
+from the parent module for backward compatibility.
+
+For analysis, import from the evaluation module:
+    >>> from ml4t.diagnostic.evaluation import TradeShapAnalyzer
+
+For models only:
+    >>> from ml4t.diagnostic.evaluation.trade_shap import (
+    ...     TradeShapResult,
+    ...     ErrorPattern,
+    ...     ClusteringResult,
+    ...     TradeShapExplanation,
+    ... )
+"""
+
+# Import models from the dedicated models module
+from ml4t.diagnostic.evaluation.trade_shap.alignment import (
+    AlignmentResult,
+    TimestampAligner,
+)
+from ml4t.diagnostic.evaluation.trade_shap.characterize import (
+    CharacterizationConfig,
+    FeatureStatistics,
+    PatternCharacterizer,
+    benjamini_hochberg,
+)
+from ml4t.diagnostic.evaluation.trade_shap.cluster import (
+    ClusteringConfig,
+    HierarchicalClusterer,
+    compute_centroids,
+    compute_cluster_sizes,
+    find_optimal_clusters,
+)
+from ml4t.diagnostic.evaluation.trade_shap.explain import TradeShapExplainer
+from ml4t.diagnostic.evaluation.trade_shap.hypotheses import (
+    HypothesisConfig,
+    HypothesisGenerator,
+    Template,
+    TemplateMatcher,
+    load_templates,
+)
+from ml4t.diagnostic.evaluation.trade_shap.models import (
+    ClusteringResult,
+    ErrorPattern,
+    TradeExplainFailure,
+    TradeShapExplanation,
+    TradeShapResult,
+)
+from ml4t.diagnostic.evaluation.trade_shap.normalize import (
+    NormalizationType,
+    normalize,
+    normalize_l1,
+    normalize_l2,
+    standardize,
+)
+from ml4t.diagnostic.evaluation.trade_shap.pipeline import (
+    TradeShapPipeline,
+    TradeShapPipelineConfig,
+)
+
+__all__ = [
+    # Alignment
+    "TimestampAligner",
+    "AlignmentResult",
+    # Explainer
+    "TradeShapExplainer",
+    # Normalization
+    "normalize",
+    "normalize_l1",
+    "normalize_l2",
+    "standardize",
+    "NormalizationType",
+    # Clustering
+    "HierarchicalClusterer",
+    "ClusteringConfig",
+    "find_optimal_clusters",
+    "compute_cluster_sizes",
+    "compute_centroids",
+    # Characterization
+    "PatternCharacterizer",
+    "CharacterizationConfig",
+    "FeatureStatistics",
+    "benjamini_hochberg",
+    # Hypothesis generation
+    "HypothesisGenerator",
+    "HypothesisConfig",
+    "TemplateMatcher",
+    "Template",
+    "load_templates",
+    # Pipeline
+    "TradeShapPipeline",
+    "TradeShapPipelineConfig",
+    # Result models
+    "TradeShapResult",
+    "TradeShapExplanation",
+    "TradeExplainFailure",
+    "ClusteringResult",
+    "ErrorPattern",
+]

ml4t/diagnostic/evaluation/trade_shap/alignment.py

@@ -0,0 +1,188 @@
+"""Fast timestamp alignment for trade SHAP analysis.
+
+This module provides O(log n) timestamp lookup instead of O(n) linear scan,
+using precomputed indices and binary search for nearest-match scenarios.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
+
+@dataclass(frozen=True)
+class AlignmentResult:
+    """Result of timestamp alignment.
+
+    Attributes:
+        index: Index into the feature DataFrame, or None if not found
+        exact: Whether this was an exact match
+        distance_seconds: Distance in seconds from target (0 if exact)
+    """
+
+    index: int | None
+    exact: bool
+    distance_seconds: float
+
+
+@dataclass
+class TimestampAligner:
+    """Fast timestamp alignment using precomputed indices.
+
+    Provides O(1) exact match lookup via dict and O(log n) nearest-match
+    via binary search on sorted numpy datetime64 array.
+
+    Attributes:
+        timestamps_ns: Sorted numpy array of timestamps as int64 nanoseconds
+        index_by_ts: Dict mapping datetime to index for O(1) exact lookup
+        tolerance_seconds: Maximum allowed distance for nearest match
+        _sorted_indices: Original indices corresponding to sorted timestamps
+
+    Example:
+        >>> import pandas as pd
+        >>> timestamps = pd.DatetimeIndex(['2024-01-01', '2024-01-02', '2024-01-03'])
+        >>> aligner = TimestampAligner.from_datetime_index(timestamps, tolerance_seconds=3600)
+        >>> result = aligner.align(datetime(2024, 1, 2))
+        >>> result.index
+        1
+        >>> result.exact
+        True
+    """
+
+    timestamps_ns: NDArray[np.int64]
+    index_by_ts: dict[datetime, int] = field(default_factory=dict)
+    tolerance_seconds: float = 0.0
+    _sorted_indices: NDArray[np.intp] = field(default_factory=lambda: np.array([], dtype=np.intp))
+
+    @classmethod
+    def from_datetime_index(
+        cls,
+        timestamps: NDArray | list[datetime],
+        tolerance_seconds: float = 0.0,
+    ) -> TimestampAligner:
+        """Create aligner from datetime index or array.
+
+        Args:
+            timestamps: DatetimeIndex, numpy datetime64 array, or list of datetimes
+            tolerance_seconds: Maximum allowed distance for nearest match (default: 0 = exact only)
+
+        Returns:
+            TimestampAligner ready for fast lookups
+
+        Raises:
+            ValueError: If timestamps array is empty
+        """
+        # Convert to numpy datetime64[ns] if needed
+        ts_array = np.asarray(timestamps, dtype="datetime64[ns]")
+
+        if len(ts_array) == 0:
+            raise ValueError("Cannot create aligner from empty timestamp array")
+
+        # Convert to int64 nanoseconds for fast comparison
+        ts_ns = ts_array.astype(np.int64)
+
+        # Get sort order (we need original indices)
+        sorted_indices = np.argsort(ts_ns)
+        sorted_ts_ns = ts_ns[sorted_indices]
+
+        # Build exact-match dict using original timestamps
+        # For duplicates, keep FIRST occurrence (standard behavior)
+        index_by_ts: dict[datetime, int] = {}
+        for i, ts in enumerate(timestamps):
+            if hasattr(ts, "to_pydatetime"):
+                # pandas Timestamp
+                dt = ts.to_pydatetime()
+            elif isinstance(ts, np.datetime64):
+                # numpy datetime64
+                dt = ts.astype("datetime64[us]").astype(datetime)
+            else:
+                dt = ts
+            # Only store first occurrence of each timestamp
+            if dt not in index_by_ts:
+                index_by_ts[dt] = i
+
+        return cls(
+            timestamps_ns=sorted_ts_ns,
+            index_by_ts=index_by_ts,
+            tolerance_seconds=tolerance_seconds,
+            _sorted_indices=sorted_indices,
+        )
+
+    def align(self, target: datetime) -> AlignmentResult:
+        """Find index for target timestamp.
+
+        First attempts exact match via dict lookup (O(1)).
+        If no exact match and tolerance > 0, uses binary search for nearest (O(log n)).
+
+        Args:
+            target: Target timestamp to align
+
+        Returns:
+            AlignmentResult with index (or None), exact flag, and distance
+        """
+        # Try exact match first (O(1))
+        if target in self.index_by_ts:
+            return AlignmentResult(index=self.index_by_ts[target], exact=True, distance_seconds=0.0)
+
+        # No exact match - if no tolerance, return None
+        if self.tolerance_seconds <= 0:
+            return AlignmentResult(index=None, exact=False, distance_seconds=float("inf"))
+
+        # Binary search for nearest (O(log n))
+        target_ns = np.datetime64(target, "ns").astype(np.int64)
+        insert_pos = np.searchsorted(self.timestamps_ns, target_ns)
+
+        # Check neighbors
+        candidates = []
+        if insert_pos > 0:
+            candidates.append(insert_pos - 1)
+        if insert_pos < len(self.timestamps_ns):
+            candidates.append(insert_pos)
+
+        if not candidates:
+            return AlignmentResult(index=None, exact=False, distance_seconds=float("inf"))
+
+        # Find closest
+        best_idx = None
+        best_distance_ns = float("inf")
+
+        for sorted_idx in candidates:
+            distance_ns = abs(self.timestamps_ns[sorted_idx] - target_ns)
+            if distance_ns < best_distance_ns:
+                best_distance_ns = distance_ns
+                best_idx = sorted_idx
+
+        # Convert to seconds and check tolerance
+        distance_seconds = best_distance_ns / 1e9
+
+        if distance_seconds <= self.tolerance_seconds:
+            # Map back to original index
+            original_idx = int(self._sorted_indices[best_idx])
+            return AlignmentResult(
+                index=original_idx,
+                exact=False,
+                distance_seconds=distance_seconds,
+            )
+
+        return AlignmentResult(index=None, exact=False, distance_seconds=distance_seconds)
+
+    def align_many(self, targets: list[datetime]) -> list[AlignmentResult]:
+        """Align multiple timestamps.
+
+        Args:
+            targets: List of target timestamps
+
+        Returns:
+            List of AlignmentResult for each target
+        """
+        return [self.align(t) for t in targets]
+
+    def __len__(self) -> int:
+        """Number of timestamps in the aligner."""
+        return len(self.timestamps_ns)