detectkit 0.3.17__tar.gz → 0.4.1__tar.gz

{detectkit-0.3.17/detectkit.egg-info → detectkit-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detectkit
-Version: 0.3.17
+Version: 0.4.1
 Summary: Metric monitoring with automatic anomaly detection
 Author: detectkit team
 License: MIT
@@ -55,9 +55,13 @@ Requires-Dist: timesfm>=0.1.0; extra == "all"
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: pytest-requests-mock>=0.1; extra == "dev"
+Requires-Dist: requests-mock>=1.12; extra == "dev"
 Requires-Dist: black>=23.0; extra == "dev"
 Requires-Dist: mypy>=1.0; extra == "dev"
 Requires-Dist: ruff>=0.1.0; extra == "dev"
+Provides-Extra: integration
+Requires-Dist: testcontainers[clickhouse]>=4.0; extra == "integration"
 Dynamic: license-file
 
 # detectkit

{detectkit-0.3.17 → detectkit-0.4.1}/detectkit/__init__.py

@@ -4,7 +4,7 @@ detectk - Anomaly Detection for Time-Series Metrics
 A Python library for data analysts and engineers to monitor metrics with automatic anomaly detection.
 """
 
-__version__ = "0.3.14"
+__version__ = "0.4.1"
 
 from detectkit.core.interval import Interval
 from detectkit.core.models import ColumnDefinition, TableModel

{detectkit-0.3.17 → detectkit-0.4.1}/detectkit/alerting/channels/factory.py

@@ -2,7 +2,6 @@
 Alert channel factory for creating channel instances from configuration.
 """
 
-import os
 from typing import Dict, List
 
 from detectkit.alerting.channels.base import BaseAlertChannel
@@ -11,6 +10,7 @@ from detectkit.alerting.channels.slack import SlackChannel
 from detectkit.alerting.channels.webhook import WebhookChannel
 from detectkit.alerting.channels.telegram import TelegramChannel
 from detectkit.alerting.channels.email import EmailChannel
+from detectkit.utils.env_interpolation import interpolate_env_vars
 
 
 class AlertChannelFactory:
@@ -82,42 +82,12 @@ class AlertChannelFactory:
 
     @classmethod
     def _interpolate_env_vars(cls, params: Dict) -> Dict:
-        """
-        Interpolate environment variables in parameter values.
-
-        Supports formats:
-        - ${VAR_NAME}
-        - {{ env_var('VAR_NAME') }}
+        """Interpolate ``${VAR}`` and ``{{ env_var('VAR') }}`` placeholders.
 
-        Args:
-            params: Parameters dictionary
-
-        Returns:
-            Parameters with interpolated values
+        Delegates to :func:`detectkit.utils.env_interpolation.interpolate_env_vars`,
+        which walks nested dicts/lists recursively.
         """
-        import re
-
-        interpolated = {}
-
-        for key, value in params.items():
-            if isinstance(value, str):
-                # Handle ${VAR} format
-                value = re.sub(
-                    r'\$\{([^}]+)\}',
-                    lambda m: os.environ.get(m.group(1), m.group(0)),
-                    value,
-                )
-
-                # Handle {{ env_var('VAR') }} format
-                value = re.sub(
-                    r"\{\{\s*env_var\(['\"]([^'\"]+)['\"]\)\s*\}\}",
-                    lambda m: os.environ.get(m.group(1), m.group(0)),
-                    value,
-                )
-
-            interpolated[key] = value
-
-        return interpolated
+        return interpolate_env_vars(params)
 
     @classmethod
     def create_from_config(cls, channel_config: Dict) -> BaseAlertChannel:

detectkit-0.4.1/detectkit/alerting/orchestrator/__init__.py

@@ -0,0 +1,19 @@
+"""Public surface of the alert-orchestrator package."""
+
+from detectkit.alerting.orchestrator._types import (
+    AlertConditions,
+    DetectionRecord,
+    _direction_from_metadata,
+    _parse_detection_metadata,
+)
+from detectkit.alerting.orchestrator.orchestrator import AlertOrchestrator
+
+__all__ = [
+    "AlertOrchestrator",
+    "AlertConditions",
+    "DetectionRecord",
+    # Re-exported for callers (notably TaskManager) that build
+    # DetectionRecord rows manually before handing them to the orchestrator.
+    "_direction_from_metadata",
+    "_parse_detection_metadata",
+]

detectkit-0.4.1/detectkit/alerting/orchestrator/_base.py

@@ -0,0 +1,46 @@
+"""Shared state for orchestrator mixins."""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+import numpy as np
+
+from detectkit.alerting.orchestrator._types import (
+    AlertConditions,
+    DetectionRecord,
+)
+from detectkit.core.interval import Interval
+
+
+class _OrchestratorBase:
+    def __init__(
+        self,
+        metric_name: str,
+        interval: Interval,
+        alert_config_id: str,
+        conditions: Optional[AlertConditions] = None,
+        timezone_display: str = "UTC",
+        internal=None,  # InternalTablesManager
+        alert_config=None,  # AlertConfig
+        description: Optional[str] = None,
+        mentions: Optional[List[str]] = None,
+    ):
+        self.metric_name = metric_name
+        self.interval = interval
+        self.alert_config_id = alert_config_id
+        self.conditions = conditions or AlertConditions()
+        self.timezone_display = timezone_display
+        self.internal = internal
+        self.alert_config = alert_config
+        self.description = description
+        self.mentions = mentions or []
+
+    @staticmethod
+    def _group_by_timestamp(
+        detections: List[DetectionRecord],
+    ) -> Dict[np.datetime64, List[DetectionRecord]]:
+        grouped: Dict[np.datetime64, List[DetectionRecord]] = {}
+        for d in detections:
+            grouped.setdefault(d.timestamp, []).append(d)
+        return grouped

detectkit-0.4.1/detectkit/alerting/orchestrator/_cooldown.py

@@ -0,0 +1,40 @@
+"""Cooldown logic — suppresses repeat alerts within a configured window."""
+
+from __future__ import annotations
+
+from detectkit.alerting.orchestrator._base import _OrchestratorBase
+from detectkit.core.interval import Interval
+from detectkit.utils.datetime_utils import now_utc_naive
+
+
+class _CooldownMixin(_OrchestratorBase):
+    def _is_in_cooldown(self) -> bool:
+        """Return ``True`` while a previously sent alert is still cooling down.
+
+        Logic:
+            1. No ``alert_cooldown`` configured → never in cooldown.
+            2. No internal manager wired in → can't read state, allow alert.
+            3. Never alerted before → no cooldown.
+            4. ``cooldown_reset_on_recovery`` and a recovery has happened
+               since the last alert → cooldown is reset, allow alert.
+            5. Otherwise: ``elapsed < cooldown_seconds`` → suppress.
+        """
+        if not self.alert_config or not self.alert_config.alert_cooldown:
+            return False
+        if not self.internal:
+            return False
+
+        last_sent = self.internal.get_last_alert_timestamp(
+            self.metric_name, self.alert_config_id
+        )
+        if not last_sent:
+            return False
+
+        cooldown_seconds = Interval(self.alert_config.alert_cooldown).seconds
+        elapsed = (now_utc_naive() - last_sent).total_seconds()
+
+        if self.alert_config.cooldown_reset_on_recovery:
+            if self._check_recovery_since_last_alert(last_sent):
+                return False
+
+        return elapsed < cooldown_seconds

detectkit-0.4.1/detectkit/alerting/orchestrator/_decision.py

@@ -0,0 +1,148 @@
+"""Decision logic: ``should_alert`` and the consecutive-anomaly helpers."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+
+from detectkit.alerting.channels.base import AlertData
+from detectkit.alerting.orchestrator._base import _OrchestratorBase
+from detectkit.alerting.orchestrator._types import DetectionRecord
+from detectkit.utils.datetime_utils import now_utc, to_aware_utc
+
+
+class _DecisionMixin(_OrchestratorBase):
+    def should_alert(
+        self,
+        recent_detections: List[DetectionRecord],
+    ) -> Tuple[bool, Optional[AlertData]]:
+        """Decide whether to fire an alert from recent detections.
+
+        Steps (cheap → expensive):
+            1. Bail out on empty input.
+            2. Honour the alert cooldown so we don't spam channels.
+            3. Require ``min_detectors`` triggering on the latest point.
+            4. Require ``consecutive_anomalies`` matching the direction.
+        """
+        if not recent_detections:
+            return False, None
+
+        # Cooldown is checked first so a noisy run doesn't waste effort.
+        if self._is_in_cooldown():
+            return False, None
+
+        detections_by_time = self._group_by_timestamp(recent_detections)
+        timestamps_sorted = sorted(detections_by_time.keys(), reverse=True)
+
+        latest_anomalies = [
+            d for d in detections_by_time[timestamps_sorted[0]] if d.is_anomaly
+        ]
+        if len(latest_anomalies) < self.conditions.min_detectors:
+            return False, None
+
+        consecutive = self._count_consecutive_anomalies(
+            detections_by_time, timestamps_sorted
+        )
+        if consecutive < self.conditions.consecutive_anomalies:
+            return False, None
+
+        return True, self._build_alert_data(latest_anomalies, consecutive)
+
+    def _count_consecutive_anomalies(
+        self,
+        detections_by_time: Dict[np.datetime64, List[DetectionRecord]],
+        timestamps_sorted: List[np.datetime64],
+    ) -> int:
+        """Walk timestamps newest→oldest counting matching anomalies."""
+        direction_condition = self.conditions.direction
+        consecutive = 0
+        prev_direction: Optional[str] = None
+
+        for ts in timestamps_sorted:
+            anomalies = [d for d in detections_by_time[ts] if d.is_anomaly]
+            if len(anomalies) < self.conditions.min_detectors:
+                break
+
+            current_direction = anomalies[0].direction
+
+            if direction_condition == "any":
+                consecutive += 1
+            elif direction_condition == "same":
+                if prev_direction is None:
+                    consecutive = 1
+                    prev_direction = current_direction
+                elif current_direction == prev_direction:
+                    consecutive += 1
+                else:
+                    break  # direction flipped → stop counting
+            elif direction_condition == "up":
+                if current_direction == "up":
+                    consecutive += 1
+                else:
+                    break
+            elif direction_condition == "down":
+                if current_direction == "down":
+                    consecutive += 1
+                else:
+                    break
+            else:
+                # Unknown direction policy — treat as "any" to stay safe.
+                consecutive += 1
+
+        return consecutive
+
+    def _build_alert_data(
+        self,
+        anomalies: List[DetectionRecord],
+        consecutive_count: int,
+    ) -> AlertData:
+        primary = anomalies[0]
+
+        if len(anomalies) > 1:
+            max_severity = max(d.severity for d in anomalies)
+            detector_names = [d.detector_name for d in anomalies]
+            detector_name = f"{len(anomalies)} detectors"
+            detector_params = "; ".join(
+                f"{d.detector_name}: {d.detector_params}" for d in anomalies
+            )
+            combined_metadata = {
+                "detectors": detector_names,
+                "count": len(anomalies),
+            }
+            for i, d in enumerate(anomalies):
+                combined_metadata[f"detector_{i}_metadata"] = d.detection_metadata
+        else:
+            max_severity = primary.severity
+            detector_name = primary.detector_name
+            detector_params = primary.detector_params
+            combined_metadata = primary.detection_metadata
+
+        return AlertData(
+            metric_name=self.metric_name,
+            timestamp=primary.timestamp,
+            timezone=self.timezone_display,
+            value=primary.value,
+            confidence_lower=primary.confidence_lower,
+            confidence_upper=primary.confidence_upper,
+            detector_name=detector_name,
+            detector_params=detector_params,
+            direction=primary.direction,
+            severity=max_severity,
+            detection_metadata=combined_metadata,
+            consecutive_count=consecutive_count,
+            description=self.description,
+            mentions=self.mentions,
+        )
+
+    def get_last_complete_point(self, now: Optional[datetime] = None) -> datetime:
+        """Floor ``now`` to the previous fully completed interval boundary."""
+        if now is None:
+            now = now_utc()
+        now = to_aware_utc(now)
+
+        interval_seconds = self.interval.seconds
+        floored = (int(now.timestamp()) // interval_seconds) * interval_seconds
+        last_complete = floored - interval_seconds
+        return datetime.fromtimestamp(last_complete, tz=timezone.utc)

detectkit-0.4.1/detectkit/alerting/orchestrator/_dispatch.py

@@ -0,0 +1,69 @@
+"""Dispatch mixin — actually sends alerts/recoveries via channels."""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from detectkit.alerting.channels.base import AlertData, BaseAlertChannel
+from detectkit.alerting.orchestrator._base import _OrchestratorBase
+from detectkit.utils.datetime_utils import now_utc_naive
+
+
+class _DispatchMixin(_OrchestratorBase):
+    def send_alerts(
+        self,
+        alert_data: AlertData,
+        channels: List[BaseAlertChannel],
+        template: Optional[str] = None,
+    ) -> Dict[str, bool]:
+        """Send *alert_data* to every channel; record success per-channel.
+
+        Updates ``last_alert_sent`` (and increments the counter) when at
+        least one channel succeeded — this is what powers cooldown and
+        recovery detection.
+        """
+        results = self._dispatch(channels, alert_data, template, "alert")
+
+        if any(results.values()) and self.internal:
+            self.internal.update_alert_timestamp(
+                metric_name=self.metric_name,
+                alert_config_id=self.alert_config_id,
+                timestamp=now_utc_naive(),
+                increment_count=True,
+            )
+        return results
+
+    def send_recovery(
+        self,
+        alert_data: AlertData,
+        channels: List[BaseAlertChannel],
+        template: Optional[str] = None,
+    ) -> Dict[str, bool]:
+        """Send a recovery notification and stamp ``last_recovery_sent``."""
+        results = self._dispatch(channels, alert_data, template, "recovery")
+
+        if any(results.values()) and self.internal:
+            self.internal.update_recovery_timestamp(
+                metric_name=self.metric_name,
+                alert_config_id=self.alert_config_id,
+                timestamp=now_utc_naive(),
+            )
+        return results
+
+    @staticmethod
+    def _dispatch(
+        channels: List[BaseAlertChannel],
+        alert_data: AlertData,
+        template: Optional[str],
+        kind: str,
+    ) -> Dict[str, bool]:
+        results: Dict[str, bool] = {}
+        for channel in channels:
+            channel_name = channel.__class__.__name__
+            try:
+                results[channel_name] = bool(channel.send(alert_data, template))
+            except Exception as exc:
+                # One bad channel must not abort the others.
+                print(f"Error sending {kind} via {channel_name}: {exc}")
+                results[channel_name] = False
+        return results

detectkit-0.4.1/detectkit/alerting/orchestrator/_recovery.py

@@ -0,0 +1,203 @@
+"""Recovery decision and reconstruction logic."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import List, Optional, Tuple
+
+import numpy as np
+
+from detectkit.alerting.channels.base import AlertData
+from detectkit.alerting.orchestrator._base import _OrchestratorBase
+from detectkit.alerting.orchestrator._types import (
+    DetectionRecord,
+    _direction_from_metadata,
+    _parse_detection_metadata,
+)
+
+
+class _RecoveryMixin(_OrchestratorBase):
+    def should_send_recovery(
+        self,
+        recent_detections: List[DetectionRecord],
+    ) -> Tuple[bool, Optional[AlertData]]:
+        """Decide whether to send a recovery notification.
+
+        Conditions (all must hold):
+            1. A previous alert has been sent (``last_alert_sent`` exists).
+            2. The metric has actually recovered (no blocking anomalies).
+            3. We haven't already notified recovery for this incident.
+        """
+        if not self.internal:
+            return False, None
+
+        last_alert = self.internal.get_last_alert_timestamp(
+            self.metric_name, self.alert_config_id
+        )
+        if not last_alert:
+            return False, None
+
+        last_recovery = self.internal.get_last_recovery_timestamp(
+            self.metric_name, self.alert_config_id
+        )
+        if last_recovery and last_recovery >= last_alert:
+            return False, None  # already notified for this incident
+
+        if not self._check_recovery_since_last_alert(last_alert):
+            return False, None
+
+        recovery_data = self._build_recovery_data(recent_detections)
+        if not recovery_data:
+            return False, None
+        return True, recovery_data
+
+    def _check_recovery_since_last_alert(
+        self, last_alert_timestamp: datetime
+    ) -> bool:
+        """Return ``True`` when the metric has recovered since *last_alert_timestamp*.
+
+        Direction-aware: a "down"-only alert is not blocked by a fresh
+        "up" anomaly, since the alert condition no longer holds.
+        """
+        if not self.internal:
+            return False
+
+        last_point = self.get_last_complete_point()
+        # +5 for safety margin so we don't truncate the consecutive window.
+        num_points = self.conditions.consecutive_anomalies + 5
+
+        recent_detections = self.internal.get_recent_detections(
+            metric_name=self.metric_name,
+            last_point=last_point,
+            num_points=num_points,
+            created_after=last_alert_timestamp,
+        )
+        if not recent_detections:
+            # No fresh detections at all → assume recovery.
+            return True
+
+        records: List[DetectionRecord] = []
+        for det in recent_detections:
+            metadata_list = (
+                det.get("detection_metadata_list")
+                or [None] * len(det["detector_ids"])
+            )
+            for i in range(len(det["detector_ids"])):
+                is_anomaly = det["is_anomaly_flags"][i]
+                metadata = _parse_detection_metadata(metadata_list[i])
+                records.append(
+                    DetectionRecord(
+                        timestamp=np.datetime64(det["timestamp"]),
+                        detector_name=det["detector_names"][i],
+                        detector_id=det["detector_ids"][i],
+                        detector_params=det["detector_params_list"][i],
+                        value=det["value"],
+                        is_anomaly=is_anomaly,
+                        confidence_lower=det["confidence_lowers"][i],
+                        confidence_upper=det["confidence_uppers"][i],
+                        direction=_direction_from_metadata(metadata, is_anomaly),
+                        severity=0.0,  # not used for the recovery check
+                        detection_metadata=metadata,
+                    )
+                )
+
+        detections_by_time = self._group_by_timestamp(records)
+        timestamps_sorted = sorted(detections_by_time.keys(), reverse=True)
+        latest_anomalies = [
+            d for d in detections_by_time[timestamps_sorted[0]] if d.is_anomaly
+        ]
+
+        direction_condition = self.conditions.direction
+        if direction_condition == "down":
+            blocking = [d for d in latest_anomalies if d.direction == "down"]
+        elif direction_condition == "up":
+            blocking = [d for d in latest_anomalies if d.direction == "up"]
+        elif direction_condition == "same":
+            trigger_direction = self._get_alert_trigger_direction(
+                last_alert_timestamp
+            )
+            if trigger_direction is None:
+                blocking = latest_anomalies  # conservative fallback
+            else:
+                blocking = [
+                    d for d in latest_anomalies if d.direction == trigger_direction
+                ]
+        else:  # "any" / unknown — preserve historical behaviour
+            blocking = latest_anomalies
+
+        return len(blocking) == 0
+
+    def _get_alert_trigger_direction(
+        self, last_alert_timestamp: datetime
+    ) -> Optional[str]:
+        """Return the direction of the anomaly that triggered the last alert."""
+        if not self.internal:
+            return None
+
+        trigger_detections = self.internal.get_recent_detections(
+            metric_name=self.metric_name,
+            last_point=last_alert_timestamp,
+            num_points=1,
+        )
+        if not trigger_detections:
+            return None
+
+        det = trigger_detections[0]
+        metadata_list = (
+            det.get("detection_metadata_list")
+            or [None] * len(det["detector_ids"])
+        )
+        for i in range(len(det["detector_ids"])):
+            if not det["is_anomaly_flags"][i]:
+                continue
+            direction = _direction_from_metadata(metadata_list[i], True)
+            if direction in ("up", "down"):
+                return direction
+        return None
+
+    def _build_recovery_data(
+        self,
+        detections: List[DetectionRecord],
+    ) -> Optional[AlertData]:
+        """Construct the AlertData payload sent as a recovery notification."""
+        if not detections:
+            return None
+
+        # ``detections`` is oldest→newest, so the latest point lives at [-1].
+        latest = detections[-1]
+
+        # Prefer the latest CI so the message reflects the *current* interval.
+        # Fall back to the last anomalous point only if the latest row has no
+        # CI (e.g. missing-data / insufficient-data placeholders).
+        recovery_ci_lower = latest.confidence_lower
+        recovery_ci_upper = latest.confidence_upper
+        recovery_detector_name = latest.detector_name
+        recovery_detector_params = latest.detector_params
+
+        if recovery_ci_lower is None or recovery_ci_upper is None:
+            last_anomalous = next(
+                (d for d in reversed(detections) if d.is_anomaly), None
+            )
+            if last_anomalous:
+                recovery_detector_name = last_anomalous.detector_name
+                recovery_detector_params = last_anomalous.detector_params
+                recovery_ci_lower = last_anomalous.confidence_lower
+                recovery_ci_upper = last_anomalous.confidence_upper
+
+        return AlertData(
+            metric_name=self.metric_name,
+            timestamp=latest.timestamp,
+            timezone=self.timezone_display,
+            value=latest.value,
+            confidence_lower=recovery_ci_lower,
+            confidence_upper=recovery_ci_upper,
+            detector_name=recovery_detector_name,
+            detector_params=recovery_detector_params,
+            direction="none",
+            severity=0.0,
+            detection_metadata={},
+            consecutive_count=0,
+            is_recovery=True,
+            description=self.description,
+            mentions=self.mentions,
+        )