tsagentkit 1.0.2__py3-none-any.whl

tsagentkit/monitoring/alerts.py

@@ -0,0 +1,302 @@
+"""Alert conditions and triggering for monitoring.
+
+Provides configurable alert conditions for coverage, drift,
+and model performance monitoring.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import UTC
+from typing import Any
+
+
+@dataclass(frozen=True)
+class AlertCondition:
+    """Alert condition configuration.
+
+    Defines when an alert should be triggered based on a metric
+    and threshold.
+
+    Attributes:
+        name: Alert name/identifier
+        metric: Metric to monitor (e.g., "coverage", "drift_score")
+        operator: Comparison operator ("lt", "gt", "eq", "ne")
+        threshold: Threshold value for triggering
+        severity: Alert severity ("info", "warning", "critical")
+        message: Optional custom alert message
+    """
+
+    name: str
+    metric: str
+    operator: str  # "lt", "gt", "eq", "ne"
+    threshold: float
+    severity: str = "warning"
+    message: str | None = None
+
+    def evaluate(self, value: float) -> bool:
+        """Evaluate if condition is met.
+
+        Args:
+            value: Current metric value
+
+        Returns:
+            True if alert should trigger
+        """
+        if self.operator == "lt":
+            return value < self.threshold
+        if self.operator == "gt":
+            return value > self.threshold
+        if self.operator == "eq":
+            return value == self.threshold
+        if self.operator == "ne":
+            return value != self.threshold
+        if self.operator == "lte":
+            return value <= self.threshold
+        if self.operator == "gte":
+            return value >= self.threshold
+        return False
+
+    def format_message(self, value: float, context: dict[str, Any] | None = None) -> str:
+        """Format alert message with current value.
+
+        Args:
+            value: Current metric value
+            context: Additional context for message formatting
+
+        Returns:
+            Formatted alert message
+        """
+        if self.message:
+            ctx = context or {}
+            return self.message.format(
+                name=self.name,
+                metric=self.metric,
+                value=value,
+                threshold=self.threshold,
+                **ctx,
+            )
+
+        op_str = {
+            "lt": "below",
+            "gt": "above",
+            "eq": "equal to",
+            "ne": "not equal to",
+            "lte": "at or below",
+            "gte": "at or above",
+        }.get(self.operator, self.operator)
+
+        return f"Alert '{self.name}': {self.metric} ({value:.4f}) is {op_str} threshold ({self.threshold:.4f})"
+
+
+@dataclass
+class Alert:
+    """Triggered alert instance.
+
+    Represents an alert that has been triggered with full context.
+
+    Attributes:
+        condition: The alert condition that triggered
+        value: The metric value that triggered the alert
+        timestamp: ISO 8601 timestamp of when alert triggered
+        context: Additional context about the alert
+    """
+
+    condition: AlertCondition
+    value: float
+    timestamp: str
+    context: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            "name": self.condition.name,
+            "metric": self.condition.metric,
+            "severity": self.condition.severity,
+            "value": self.value,
+            "threshold": self.condition.threshold,
+            "operator": self.condition.operator,
+            "timestamp": self.timestamp,
+            "message": self.condition.format_message(self.value, self.context),
+            "context": self.context,
+        }
+
+
+class AlertManager:
+    """Manager for alert conditions and triggering.
+
+    Provides a centralized way to define alert conditions and
+check them against current metrics.
+
+    Example:
+        >>> manager = AlertManager()
+        >>> manager.add_condition(AlertCondition(
+        ...     name="low_coverage",
+        ...     metric="coverage_80",
+        ...     operator="lt",
+        ...     threshold=0.75,
+        ...     severity="critical",
+        ... ))
+        >>> alerts = manager.check_metrics({"coverage_80": 0.70})
+        >>> print(len(alerts))
+        1
+    """
+
+    def __init__(self) -> None:
+        """Initialize alert manager."""
+        self.conditions: list[AlertCondition] = []
+        self._alert_history: list[Alert] = []
+
+    def add_condition(self, condition: AlertCondition) -> None:
+        """Add an alert condition.
+
+        Args:
+            condition: Alert condition to add
+        """
+        self.conditions.append(condition)
+
+    def remove_condition(self, name: str) -> bool:
+        """Remove an alert condition by name.
+
+        Args:
+            name: Name of condition to remove
+
+        Returns:
+            True if condition was found and removed
+        """
+        for i, cond in enumerate(self.conditions):
+            if cond.name == name:
+                self.conditions.pop(i)
+                return True
+        return False
+
+    def check_metrics(
+        self,
+        metrics: dict[str, float],
+        context: dict[str, Any] | None = None,
+    ) -> list[Alert]:
+        """Check all conditions against current metrics.
+
+        Args:
+            metrics: Dictionary of metric names to values
+            context: Additional context for alert messages
+
+        Returns:
+            List of triggered alerts
+        """
+        from datetime import datetime
+
+        triggered: list[Alert] = []
+
+        for condition in self.conditions:
+            if condition.metric not in metrics:
+                continue
+
+            value = metrics[condition.metric]
+            if condition.evaluate(value):
+                alert = Alert(
+                    condition=condition,
+                    value=value,
+                    timestamp=datetime.now(UTC).isoformat(),
+                    context=context or {},
+                )
+                triggered.append(alert)
+                self._alert_history.append(alert)
+
+        return triggered
+
+    def check_coverage(
+        self,
+        coverage_checks: list[Any],
+        context: dict[str, Any] | None = None,
+    ) -> list[Alert]:
+        """Check coverage results against conditions.
+
+        Args:
+            coverage_checks: List of CoverageCheck objects
+            context: Additional context
+
+        Returns:
+            List of triggered alerts
+        """
+        metrics: dict[str, float] = {}
+        for check in coverage_checks:
+            if hasattr(check, "is_acceptable") and hasattr(check, "actual_coverage"):
+                metric_name = f"coverage_{check.expected_coverage:.0%}"
+                metrics[metric_name] = check.actual_coverage
+
+        return self.check_metrics(metrics, context)
+
+    def get_alert_history(
+        self,
+        severity: str | None = None,
+    ) -> list[Alert]:
+        """Get history of triggered alerts.
+
+        Args:
+            severity: Filter by severity (optional)
+
+        Returns:
+            List of historical alerts
+        """
+        if severity is None:
+            return self._alert_history.copy()
+        return [a for a in self._alert_history if a.condition.severity == severity]
+
+    def clear_history(self) -> None:
+        """Clear alert history."""
+        self._alert_history.clear()
+
+
+def create_default_coverage_alerts(
+    coverage_levels: list[float] | None = None,
+    tolerance: float = 0.05,
+) -> list[AlertCondition]:
+    """Create default alert conditions for coverage monitoring.
+
+    Args:
+        coverage_levels: Coverage levels to monitor (default: [0.5, 0.8, 0.95])
+        tolerance: Tolerance for coverage deviation
+
+    Returns:
+        List of alert conditions
+    """
+    levels = coverage_levels or [0.5, 0.8, 0.95]
+    conditions: list[AlertCondition] = []
+
+    for level in levels:
+        conditions.append(
+            AlertCondition(
+                name=f"low_coverage_{level:.0%}",
+                metric=f"coverage_{level:.0%}",
+                operator="lt",
+                threshold=level - tolerance,
+                severity="warning" if level < 0.9 else "critical",
+                message=f"Coverage for {level:.0%} interval is below acceptable threshold",
+            )
+        )
+
+    return conditions
+
+
+def create_default_drift_alerts(
+    drift_threshold: float = 0.05,
+) -> list[AlertCondition]:
+    """Create default alert conditions for drift monitoring.
+
+    Args:
+        drift_threshold: Drift score threshold
+
+    Returns:
+        List of alert conditions
+    """
+    return [
+        AlertCondition(
+            name="drift_detected",
+            metric="drift_score",
+            operator="gt",
+            threshold=drift_threshold,
+            severity="warning",
+            message="Data drift detected above threshold",
+        ),
+    ]

tsagentkit/monitoring/coverage.py

@@ -0,0 +1,203 @@
+"""Coverage monitoring for quantile forecasts.
+
+Provides interval coverage checks to verify that prediction intervals
+are well-calibrated over time.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import pandas as pd
+
+
+@dataclass(frozen=True)
+class CoverageCheck:
+    """Interval coverage check for a specific quantile.
+
+    Tracks the actual coverage rate compared to expected coverage
+    for a given quantile level, with per-horizon breakdown.
+
+    Attributes:
+        quantile: The quantile level (e.g., 0.1 for lower bound)
+        expected_coverage: Expected coverage rate
+        actual_coverage: Actual observed coverage rate
+        hit_rate_by_horizon: Coverage rate for each forecast horizon
+        tolerance: Acceptable deviation from expected coverage
+    """
+
+    quantile: float
+    expected_coverage: float
+    actual_coverage: float
+    hit_rate_by_horizon: dict[int, float]
+    tolerance: float = 0.05
+
+    def is_acceptable(self) -> bool:
+        """Check if coverage is within tolerance of expected.
+
+        For interval coverage, we want the actual coverage to be at least
+        the expected coverage minus tolerance. Being slightly over is fine,
+        but being under indicates the intervals are too narrow.
+        """
+        return self.actual_coverage >= (self.expected_coverage - self.tolerance)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            "quantile": self.quantile,
+            "expected_coverage": self.expected_coverage,
+            "actual_coverage": self.actual_coverage,
+            "hit_rate_by_horizon": self.hit_rate_by_horizon,
+            "tolerance": self.tolerance,
+            "is_acceptable": self.is_acceptable(),
+        }
+
+
+class CoverageMonitor:
+    """Monitor quantile coverage over time.
+
+    Provides functionality to check if prediction intervals are
+    well-calibrated by comparing actual vs expected coverage rates.
+
+    Example:
+        >>> monitor = CoverageMonitor()
+        >>> checks = monitor.check(
+        ...     forecasts=forecast_df,
+        ...     actuals=actual_df,
+        ...     quantiles=[0.1, 0.5, 0.9],
+        ... )
+        >>> for check in checks:
+        ...     print(f"Q{check.quantile}: {check.actual_coverage:.2%}")
+    """
+
+    def check(
+        self,
+        forecasts: pd.DataFrame,
+        actuals: pd.DataFrame,
+        quantiles: list[float],
+        tolerance: float = 0.05,
+    ) -> list[CoverageCheck]:
+        """Check coverage for given quantiles.
+
+        Args:
+            forecasts: Forecast dataframe with quantile columns (q_0.1, q_0.9, etc.)
+            actuals: Actual values dataframe
+            quantiles: List of quantile levels to check
+            tolerance: Acceptable deviation from expected coverage
+
+        Returns:
+            List of CoverageCheck objects, one per quantile pair
+        """
+        results: list[CoverageCheck] = []
+
+        # Ensure required columns exist
+        if "unique_id" not in forecasts.columns or "ds" not in forecasts.columns:
+            return results
+
+        # Merge forecasts with actuals
+        merged = forecasts.merge(
+            actuals[["unique_id", "ds", "y"]],
+            on=["unique_id", "ds"],
+            how="inner",
+        )
+
+        if merged.empty:
+            return results
+
+        # Calculate coverage for each quantile pair
+        for i, lower_q in enumerate(quantiles):
+            for upper_q in quantiles[i + 1 :]:
+                lower_col = f"q_{lower_q}"
+                upper_col = f"q_{upper_q}"
+
+                if lower_col not in merged.columns or upper_col not in merged.columns:
+                    continue
+
+                # Calculate overall coverage
+                in_interval = (merged["y"] >= merged[lower_col]) & (
+                    merged["y"] <= merged[upper_col]
+                )
+                actual_coverage = in_interval.mean()
+
+                # Calculate coverage by horizon if available
+                hit_rate_by_horizon: dict[int, float] = {}
+                if "h" in merged.columns or "horizon" in merged.columns:
+                    h_col = "h" if "h" in merged.columns else "horizon"
+                    for h in sorted(merged[h_col].unique()):
+                        h_data = merged[merged[h_col] == h]
+                        if len(h_data) > 0:
+                            h_in_interval = (h_data["y"] >= h_data[lower_col]) & (
+                                h_data["y"] <= h_data[upper_col]
+                            )
+                            hit_rate_by_horizon[int(h)] = float(h_in_interval.mean())
+
+                # Expected coverage is the difference between quantiles
+                expected_coverage = upper_q - lower_q
+
+                results.append(
+                    CoverageCheck(
+                        quantile=lower_q,
+                        expected_coverage=expected_coverage,
+                        actual_coverage=float(actual_coverage),
+                        hit_rate_by_horizon=hit_rate_by_horizon,
+                        tolerance=tolerance,
+                    )
+                )
+
+        return results
+
+    def check_single_quantile(
+        self,
+        forecasts: pd.DataFrame,
+        actuals: pd.DataFrame,
+        quantile: float,
+        tolerance: float = 0.05,
+    ) -> CoverageCheck | None:
+        """Check coverage for a single quantile (e.g., median).
+
+        For a single quantile, checks if actuals fall below the quantile
+        at the expected rate.
+
+        Args:
+            forecasts: Forecast dataframe
+            actuals: Actual values dataframe
+            quantile: Quantile level to check
+            tolerance: Acceptable deviation
+
+        Returns:
+            CoverageCheck or None if data not available
+        """
+        q_col = f"q_{quantile}"
+        if q_col not in forecasts.columns:
+            return None
+
+        merged = forecasts.merge(
+            actuals[["unique_id", "ds", "y"]],
+            on=["unique_id", "ds"],
+            how="inner",
+        )
+
+        if merged.empty:
+            return None
+
+        # For single quantile: check if actual <= quantile at quantile rate
+        below_quantile = merged["y"] <= merged[q_col]
+        actual_coverage = float(below_quantile.mean())
+
+        hit_rate_by_horizon: dict[int, float] = {}
+        if "h" in merged.columns or "horizon" in merged.columns:
+            h_col = "h" if "h" in merged.columns else "horizon"
+            for h in sorted(merged[h_col].unique()):
+                h_data = merged[merged[h_col] == h]
+                if len(h_data) > 0:
+                    h_below = h_data["y"] <= h_data[q_col]
+                    hit_rate_by_horizon[int(h)] = float(h_below.mean())
+
+        return CoverageCheck(
+            quantile=quantile,
+            expected_coverage=quantile,
+            actual_coverage=actual_coverage,
+            hit_rate_by_horizon=hit_rate_by_horizon,
+            tolerance=tolerance,
+        )