quantumflow-sdk 0.2.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

quantumflow/pipeline/anomaly_detector.py

@@ -0,0 +1,521 @@
+"""
+Anomaly Detection for Data Pipelines.
+
+Detects:
+- Gradient explosion (>threshold) and vanishing (<threshold)
+- NaN/Inf values
+- Statistical deviations (moving average, z-score)
+- Domain-specific anomalies (energy spikes, RMSD divergence, VaR breach)
+
+Example:
+    detector = AnomalyDetector()
+
+    # Add custom detector
+    detector.register_detector("my_detector", my_detector_fn)
+
+    # Detect anomalies
+    result = detector.detect(state, step=100)
+    if result.is_anomaly:
+        print(f"Anomaly: {result.anomaly_type}")
+"""
+
+import math
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional, Tuple
+from enum import Enum
+
+logger = logging.getLogger(__name__)
+
+
+class AnomalyType(str, Enum):
+    """Types of anomalies that can be detected."""
+
+    GRADIENT_EXPLOSION = "gradient_explosion"
+    GRADIENT_VANISHING = "gradient_vanishing"
+    NAN_DETECTED = "nan_detected"
+    INF_DETECTED = "inf_detected"
+    ENERGY_SPIKE = "energy_spike"
+    RMSD_DIVERGENCE = "rmsd_divergence"
+    VAR_BREACH = "var_breach"
+    DRAWDOWN_BREACH = "drawdown_breach"
+    STATISTICAL_DEVIATION = "statistical_deviation"
+    CUSTOM = "custom"
+
+
+class AnomalySeverity(str, Enum):
+    """Severity levels for anomalies."""
+
+    INFO = "info"
+    WARNING = "warning"
+    CRITICAL = "critical"
+
+
+@dataclass
+class AnomalyResult:
+    """Result of anomaly detection."""
+
+    is_anomaly: bool
+    anomaly_type: Optional[str] = None
+    severity: str = "warning"
+    message: str = ""
+    step: int = 0
+    detector_name: Optional[str] = None
+    threshold: Optional[float] = None
+    actual_value: Optional[float] = None
+    data: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "is_anomaly": self.is_anomaly,
+            "anomaly_type": self.anomaly_type,
+            "severity": self.severity,
+            "message": self.message,
+            "step": self.step,
+            "detector_name": self.detector_name,
+            "threshold": self.threshold,
+            "actual_value": self.actual_value,
+            "data": self.data,
+        }
+
+
+# Type for custom detector functions
+DetectorFn = Callable[["PipelineState", int, Dict[str, Any]], Optional[AnomalyResult]]
+
+
+class AnomalyDetector:
+    """
+    Detects anomalies in pipeline execution.
+
+    Built-in detectors:
+    - Gradient explosion/vanishing
+    - NaN/Inf detection
+    - Statistical deviation (moving average, z-score)
+
+    Supports custom detector registration for domain-specific checks.
+    """
+
+    def __init__(
+        self,
+        window_size: int = 50,
+        z_score_threshold: float = 3.0,
+        moving_avg_deviation: float = 2.0,
+    ):
+        """
+        Initialize anomaly detector.
+
+        Args:
+            window_size: Window for moving statistics
+            z_score_threshold: Z-score threshold for statistical anomalies
+            moving_avg_deviation: Multiplier for moving average deviation
+        """
+        self.window_size = window_size
+        self.z_score_threshold = z_score_threshold
+        self.moving_avg_deviation = moving_avg_deviation
+
+        # Custom detectors
+        self._custom_detectors: Dict[str, DetectorFn] = {}
+
+        # History for statistical analysis
+        self._metric_history: Dict[str, List[float]] = {}
+
+    def register_detector(self, name: str, detector_fn: DetectorFn):
+        """
+        Register a custom anomaly detector.
+
+        Args:
+            name: Detector name
+            detector_fn: Function(state, step, config) -> Optional[AnomalyResult]
+        """
+        self._custom_detectors[name] = detector_fn
+        logger.info(f"Registered custom detector: {name}")
+
+    def unregister_detector(self, name: str):
+        """Unregister a custom detector."""
+        if name in self._custom_detectors:
+            del self._custom_detectors[name]
+
+    def detect(
+        self,
+        state: "PipelineState",
+        step: int,
+        gradient_threshold: float = 100.0,
+        vanishing_threshold: float = 1e-7,
+        config: Optional[Dict[str, Any]] = None,
+    ) -> Optional[AnomalyResult]:
+        """
+        Run all anomaly detectors on current state.
+
+        Args:
+            state: Current pipeline state
+            step: Current step number
+            gradient_threshold: Threshold for gradient explosion
+            vanishing_threshold: Threshold for vanishing gradients
+            config: Additional configuration for detectors
+
+        Returns:
+            AnomalyResult if anomaly detected, None otherwise
+        """
+        config = config or {}
+
+        # Check NaN/Inf first (most critical)
+        result = self._check_nan_inf(state, step)
+        if result:
+            return result
+
+        # Check gradients
+        result = self._check_gradients(
+            state, step, gradient_threshold, vanishing_threshold
+        )
+        if result:
+            return result
+
+        # Check statistical deviation
+        result = self._check_statistical(state, step)
+        if result:
+            return result
+
+        # Run custom detectors
+        for name, detector_fn in self._custom_detectors.items():
+            try:
+                result = detector_fn(state, step, config)
+                if result and result.is_anomaly:
+                    result.detector_name = name
+                    return result
+            except Exception as e:
+                logger.warning(f"Custom detector {name} failed: {e}")
+
+        return None
+
+    def _check_nan_inf(self, state: "PipelineState", step: int) -> Optional[AnomalyResult]:
+        """Check for NaN or Inf values in state."""
+        values_to_check = []
+
+        # Check metrics
+        for name, value in state.metrics.items():
+            if isinstance(value, (int, float)):
+                values_to_check.append((f"metrics.{name}", value))
+
+        # Check gradients
+        for i, grad in enumerate(state.gradient_history[-10:]):
+            values_to_check.append((f"gradient[{i}]", grad))
+
+        # Check weights
+        if state.weights:
+            for i, w in enumerate(state.weights[:100]):  # Check first 100
+                values_to_check.append((f"weight[{i}]", w))
+
+        # Detect NaN
+        for name, value in values_to_check:
+            if isinstance(value, float) and math.isnan(value):
+                return AnomalyResult(
+                    is_anomaly=True,
+                    anomaly_type=AnomalyType.NAN_DETECTED.value,
+                    severity=AnomalySeverity.CRITICAL.value,
+                    message=f"NaN detected in {name}",
+                    step=step,
+                    detector_name="nan_inf_detector",
+                    actual_value=value,
+                    data={"location": name},
+                )
+
+        # Detect Inf
+        for name, value in values_to_check:
+            if isinstance(value, float) and math.isinf(value):
+                return AnomalyResult(
+                    is_anomaly=True,
+                    anomaly_type=AnomalyType.INF_DETECTED.value,
+                    severity=AnomalySeverity.CRITICAL.value,
+                    message=f"Inf detected in {name}",
+                    step=step,
+                    detector_name="nan_inf_detector",
+                    actual_value=value,
+                    data={"location": name},
+                )
+
+        return None
+
+    def _check_gradients(
+        self,
+        state: "PipelineState",
+        step: int,
+        explosion_threshold: float,
+        vanishing_threshold: float,
+    ) -> Optional[AnomalyResult]:
+        """Check for gradient explosion or vanishing."""
+        if not state.gradient_history:
+            return None
+
+        # Get recent gradient magnitude
+        recent_grads = state.gradient_history[-10:]
+        if not recent_grads:
+            return None
+
+        # Compute gradient norm (average magnitude)
+        grad_norm = sum(abs(g) for g in recent_grads) / len(recent_grads)
+
+        # Check explosion
+        if grad_norm > explosion_threshold:
+            return AnomalyResult(
+                is_anomaly=True,
+                anomaly_type=AnomalyType.GRADIENT_EXPLOSION.value,
+                severity=AnomalySeverity.CRITICAL.value,
+                message=f"Gradient explosion detected: {grad_norm:.2e} > {explosion_threshold}",
+                step=step,
+                detector_name="gradient_detector",
+                threshold=explosion_threshold,
+                actual_value=grad_norm,
+                data={"recent_gradients": recent_grads},
+            )
+
+        # Check vanishing
+        if grad_norm < vanishing_threshold and grad_norm > 0:
+            return AnomalyResult(
+                is_anomaly=True,
+                anomaly_type=AnomalyType.GRADIENT_VANISHING.value,
+                severity=AnomalySeverity.WARNING.value,
+                message=f"Vanishing gradient detected: {grad_norm:.2e} < {vanishing_threshold}",
+                step=step,
+                detector_name="gradient_detector",
+                threshold=vanishing_threshold,
+                actual_value=grad_norm,
+                data={"recent_gradients": recent_grads},
+            )
+
+        return None
+
+    def _check_statistical(
+        self, state: "PipelineState", step: int
+    ) -> Optional[AnomalyResult]:
+        """Check for statistical deviations in metrics."""
+        for metric_name, value in state.metrics.items():
+            if not isinstance(value, (int, float)):
+                continue
+
+            # Update history
+            if metric_name not in self._metric_history:
+                self._metric_history[metric_name] = []
+
+            history = self._metric_history[metric_name]
+            history.append(value)
+
+            # Keep window size
+            if len(history) > self.window_size:
+                history.pop(0)
+
+            # Need enough history for statistics
+            if len(history) < 10:
+                continue
+
+            # Compute statistics
+            mean = sum(history) / len(history)
+            variance = sum((x - mean) ** 2 for x in history) / len(history)
+            std = math.sqrt(variance) if variance > 0 else 0
+
+            if std == 0:
+                continue
+
+            # Z-score check
+            z_score = abs(value - mean) / std
+
+            if z_score > self.z_score_threshold:
+                return AnomalyResult(
+                    is_anomaly=True,
+                    anomaly_type=AnomalyType.STATISTICAL_DEVIATION.value,
+                    severity=AnomalySeverity.WARNING.value,
+                    message=f"Statistical anomaly in {metric_name}: z-score={z_score:.2f}",
+                    step=step,
+                    detector_name="statistical_detector",
+                    threshold=self.z_score_threshold,
+                    actual_value=z_score,
+                    data={
+                        "metric_name": metric_name,
+                        "metric_value": value,
+                        "mean": mean,
+                        "std": std,
+                        "z_score": z_score,
+                    },
+                )
+
+        return None
+
+    def clear_history(self):
+        """Clear metric history."""
+        self._metric_history.clear()
+
+
+# ============================================================
+# Domain-Specific Detector Factories
+# ============================================================
+
+
+def create_energy_spike_detector(
+    threshold_multiplier: float = 5.0,
+) -> DetectorFn:
+    """
+    Create detector for energy spikes (healthcare/chemistry).
+
+    Args:
+        threshold_multiplier: Multiplier for energy spike detection
+
+    Returns:
+        Detector function
+    """
+    energy_history: List[float] = []
+
+    def detector(state: "PipelineState", step: int, config: Dict[str, Any]) -> Optional[AnomalyResult]:
+        energy = state.metrics.get("energy") or state.metrics.get("ground_energy")
+
+        if energy is None:
+            return None
+
+        energy_history.append(energy)
+        if len(energy_history) > 100:
+            energy_history.pop(0)
+
+        if len(energy_history) < 5:
+            return None
+
+        # Check for sudden spike
+        recent_mean = sum(energy_history[-10:-1]) / (len(energy_history[-10:-1]) or 1)
+        if recent_mean == 0:
+            return None
+
+        spike_ratio = abs(energy - recent_mean) / abs(recent_mean)
+
+        if spike_ratio > threshold_multiplier:
+            return AnomalyResult(
+                is_anomaly=True,
+                anomaly_type=AnomalyType.ENERGY_SPIKE.value,
+                severity=AnomalySeverity.CRITICAL.value,
+                message=f"Energy spike detected: {spike_ratio:.2f}x above recent average",
+                step=step,
+                threshold=threshold_multiplier,
+                actual_value=spike_ratio,
+                data={"energy": energy, "recent_mean": recent_mean},
+            )
+
+        return None
+
+    return detector
+
+
+def create_rmsd_detector(max_rmsd: float = 10.0) -> DetectorFn:
+    """
+    Create detector for RMSD divergence (protein folding).
+
+    Args:
+        max_rmsd: Maximum allowed RMSD in Angstroms
+
+    Returns:
+        Detector function
+    """
+
+    def detector(state: "PipelineState", step: int, config: Dict[str, Any]) -> Optional[AnomalyResult]:
+        rmsd = state.metrics.get("rmsd")
+
+        if rmsd is None:
+            return None
+
+        if rmsd > max_rmsd:
+            return AnomalyResult(
+                is_anomaly=True,
+                anomaly_type=AnomalyType.RMSD_DIVERGENCE.value,
+                severity=AnomalySeverity.CRITICAL.value,
+                message=f"RMSD divergence: {rmsd:.2f}Å > {max_rmsd}Å threshold",
+                step=step,
+                threshold=max_rmsd,
+                actual_value=rmsd,
+                data={"rmsd": rmsd},
+            )
+
+        return None
+
+    return detector
+
+
+def create_var_detector(max_var_breach: float = 0.05) -> DetectorFn:
+    """
+    Create detector for Value-at-Risk breach (finance).
+
+    Args:
+        max_var_breach: Maximum VaR breach percentage (e.g., 0.05 = 5%)
+
+    Returns:
+        Detector function
+    """
+
+    def detector(state: "PipelineState", step: int, config: Dict[str, Any]) -> Optional[AnomalyResult]:
+        var = state.metrics.get("var") or state.metrics.get("value_at_risk")
+        portfolio_value = state.metrics.get("portfolio_value", 1.0)
+
+        if var is None:
+            return None
+
+        var_ratio = abs(var) / portfolio_value if portfolio_value > 0 else 0
+
+        if var_ratio > max_var_breach:
+            return AnomalyResult(
+                is_anomaly=True,
+                anomaly_type=AnomalyType.VAR_BREACH.value,
+                severity=AnomalySeverity.CRITICAL.value,
+                message=f"VaR breach: {var_ratio:.2%} > {max_var_breach:.2%} threshold",
+                step=step,
+                threshold=max_var_breach,
+                actual_value=var_ratio,
+                data={"var": var, "portfolio_value": portfolio_value},
+            )
+
+        return None
+
+    return detector
+
+
+def create_drawdown_detector(max_drawdown: float = 0.20) -> DetectorFn:
+    """
+    Create detector for maximum drawdown breach (finance).
+
+    Args:
+        max_drawdown: Maximum allowed drawdown (e.g., 0.20 = 20%)
+
+    Returns:
+        Detector function
+    """
+    peak_value: List[float] = [0.0]
+
+    def detector(state: "PipelineState", step: int, config: Dict[str, Any]) -> Optional[AnomalyResult]:
+        portfolio_value = state.metrics.get("portfolio_value")
+
+        if portfolio_value is None:
+            return None
+
+        # Update peak
+        if portfolio_value > peak_value[0]:
+            peak_value[0] = portfolio_value
+
+        if peak_value[0] == 0:
+            return None
+
+        # Calculate drawdown
+        drawdown = (peak_value[0] - portfolio_value) / peak_value[0]
+
+        if drawdown > max_drawdown:
+            return AnomalyResult(
+                is_anomaly=True,
+                anomaly_type=AnomalyType.DRAWDOWN_BREACH.value,
+                severity=AnomalySeverity.CRITICAL.value,
+                message=f"Drawdown breach: {drawdown:.2%} > {max_drawdown:.2%} threshold",
+                step=step,
+                threshold=max_drawdown,
+                actual_value=drawdown,
+                data={
+                    "current_value": portfolio_value,
+                    "peak_value": peak_value[0],
+                    "drawdown": drawdown,
+                },
+            )
+
+        return None
+
+    return detector