detectkit 0.3.5__tar.gz → 0.3.6__tar.gz

{detectkit-0.3.5/detectkit.egg-info → detectkit-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detectkit
-Version: 0.3.5
+Version: 0.3.6
 Summary: Metric monitoring with automatic anomaly detection
 Author: detectkit team
 License: MIT
@@ -74,6 +74,13 @@ Published to PyPI: https://pypi.org/project/detectkit/
 
 Complete rewrite with modern architecture and full documentation (2025).
 
+### What's New in v0.3.6
+
+✅ **Recovery Notifications** - Know when your metrics stabilize
+- `notify_on_recovery: true` sends a message when anomaly resolves
+- Custom `template_recovery` for recovery message format
+- One notification per incident, no duplicates
+
 ### What's New in v0.3.0
 
 🎯 **Alert Cooldown** - Prevent alert spam from persistent anomalies

{detectkit-0.3.5 → detectkit-0.3.6}/README.md

@@ -12,6 +12,13 @@ Published to PyPI: https://pypi.org/project/detectkit/
 
 Complete rewrite with modern architecture and full documentation (2025).
 
+### What's New in v0.3.6
+
+✅ **Recovery Notifications** - Know when your metrics stabilize
+- `notify_on_recovery: true` sends a message when anomaly resolves
+- Custom `template_recovery` for recovery message format
+- One notification per incident, no duplicates
+
 ### What's New in v0.3.0
 
 🎯 **Alert Cooldown** - Prevent alert spam from persistent anomalies

{detectkit-0.3.5 → detectkit-0.3.6}/detectkit/alerting/channels/base.py

@@ -46,6 +46,7 @@ class AlertData:
     severity: float
     detection_metadata: Dict[str, Any]
     consecutive_count: int = 1
+    is_recovery: bool = False
 
 
 class BaseAlertChannel(ABC):
@@ -99,6 +100,7 @@ class BaseAlertChannel(ABC):
         self,
         alert_data: AlertData,
         template: Optional[str] = None,
+        recovery_template: Optional[str] = None,
     ) -> str:
         """
         Format alert message from template.
@@ -127,7 +129,10 @@ class BaseAlertChannel(ABC):
             >>> message = channel.format_message(alert_data, template)
         """
         if template is None:
-            template = self.get_default_template()
+            if alert_data.is_recovery:
+                template = recovery_template or self.get_default_recovery_template()
+            else:
+                template = self.get_default_template()
 
         # Format timestamp to string
         from datetime import datetime
@@ -149,6 +154,8 @@ class BaseAlertChannel(ABC):
             confidence_str = "N/A"
 
         # Format message
+        status = "RECOVERED" if alert_data.is_recovery else "ANOMALY"
+
         try:
             message = template.format(
                 metric_name=alert_data.metric_name,
@@ -163,6 +170,7 @@ class BaseAlertChannel(ABC):
                 direction=alert_data.direction,
                 severity=alert_data.severity,
                 consecutive_count=alert_data.consecutive_count,
+                status=status,
             )
         except KeyError as e:
             # If template has unknown variables, fall back to default
@@ -188,6 +196,22 @@ class BaseAlertChannel(ABC):
             "Severity: {severity:.2f}"
         )
 
+    def get_default_recovery_template(self) -> str:
+        """
+        Get default recovery message template.
+
+        Returns:
+            Default recovery template string
+        """
+        return (
+            "Metric recovered: {metric_name}\n"
+            "Time: {timestamp}\n"
+            "Value: {value}\n"
+            "Confidence interval: {confidence_interval}\n"
+            "Detector: {detector_name}\n"
+            "Status: metric returned to normal"
+        )
+
     def __repr__(self) -> str:
         """String representation of channel."""
         return f"{self.__class__.__name__}()"

{detectkit-0.3.5 → detectkit-0.3.6}/detectkit/alerting/orchestrator.py

@@ -521,6 +521,124 @@ class AlertOrchestrator:
         # Recovery = consecutive dropped below threshold
         return consecutive < self.conditions.consecutive_anomalies
 
+    def should_send_recovery(
+        self,
+        recent_detections: List[DetectionRecord],
+    ) -> tuple[bool, Optional[AlertData]]:
+        """
+        Determine if recovery notification should be sent.
+
+        Recovery is sent when:
+        1. A previous alert was sent (last_alert_sent exists)
+        2. Metric has recovered (consecutive anomalies < threshold)
+        3. Recovery hasn't already been sent for this incident
+           (last_recovery_sent > last_alert_sent would mean already notified)
+
+        Args:
+            recent_detections: List of recent detection records (sorted by time, newest first)
+
+        Returns:
+            Tuple of (should_send, recovery_alert_data)
+        """
+        if not self.internal:
+            return False, None
+
+        # Check if there was a previous alert
+        last_alert = self.internal.get_last_alert_timestamp(self.metric_name)
+        if not last_alert:
+            return False, None  # Never alerted, nothing to recover from
+
+        # Check if recovery already sent for this incident
+        last_recovery = self.internal.get_last_recovery_timestamp(self.metric_name)
+        if last_recovery and last_recovery >= last_alert:
+            return False, None  # Already sent recovery for this alert
+
+        # Check if metric actually recovered
+        has_recovery = self._check_recovery_since_last_alert(last_alert)
+        if not has_recovery:
+            return False, None  # Still in anomaly state
+
+        # Build recovery AlertData from latest normal point
+        recovery_data = self._build_recovery_data(recent_detections)
+        if not recovery_data:
+            return False, None
+
+        return True, recovery_data
+
+    def _build_recovery_data(
+        self,
+        detections: List[DetectionRecord],
+    ) -> Optional[AlertData]:
+        """
+        Build AlertData for recovery notification from latest detection.
+
+        Args:
+            detections: Recent detection records
+
+        Returns:
+            AlertData with is_recovery=True, or None if no data
+        """
+        if not detections:
+            return None
+
+        # Use the latest detection point for recovery info
+        latest = detections[0]
+
+        return AlertData(
+            metric_name=self.metric_name,
+            timestamp=latest.timestamp,
+            timezone=self.timezone_display,
+            value=latest.value,
+            confidence_lower=latest.confidence_lower,
+            confidence_upper=latest.confidence_upper,
+            detector_name=latest.detector_name,
+            detector_params=latest.detector_params,
+            direction="none",
+            severity=0.0,
+            detection_metadata={},
+            consecutive_count=0,
+            is_recovery=True,
+        )
+
+    def send_recovery(
+        self,
+        alert_data: AlertData,
+        channels: List[BaseAlertChannel],
+        template: Optional[str] = None,
+    ) -> Dict[str, bool]:
+        """
+        Send recovery notifications through all configured channels.
+
+        Args:
+            alert_data: Recovery alert data (is_recovery=True)
+            channels: List of alert channels
+            template: Optional custom recovery message template
+
+        Returns:
+            Dict mapping channel name to success status
+        """
+        results = {}
+
+        for channel in channels:
+            try:
+                success = channel.send(alert_data, template)
+                channel_name = channel.__class__.__name__
+                results[channel_name] = success
+            except Exception as e:
+                channel_name = channel.__class__.__name__
+                print(f"Error sending recovery via {channel_name}: {e}")
+                results[channel_name] = False
+
+        # Update recovery timestamp after sending
+        if any(results.values()) and self.internal:
+            from datetime import timezone as tz
+            self.internal.update_recovery_timestamp(
+                metric_name=self.metric_name,
+                timestamp=datetime.now(tz.utc).replace(tzinfo=None),
+            )
+
+        return results
+
     def __repr__(self) -> str:
         """String representation."""
         return (

{detectkit-0.3.5 → detectkit-0.3.6}/detectkit/config/metric_config.py

@@ -181,6 +181,17 @@ class AlertConfig(BaseModel):
                     "Only applies if alert_cooldown is set. "
                     "True = cooldown resets on recovery, False = strict cooldown independent of recovery."
     )
+    notify_on_recovery: bool = Field(
+        default=False,
+        description="Send notification when metric recovers from anomaly state. "
+                    "Recovery is detected when consecutive anomalies drop below threshold "
+                    "after an alert was previously sent."
+    )
+    template_recovery: Optional[str] = Field(
+        default=None,
+        description="Custom template for recovery notification message. "
+                    "Supports same variables as anomaly templates plus {status}."
+    )
 
     @field_validator("consecutive_anomalies")
     @classmethod

{detectkit-0.3.5 → detectkit-0.3.6}/detectkit/database/internal_tables.py

@@ -976,3 +976,86 @@ class InternalTablesManager:
 
         # ClickHouse ALTER TABLE UPDATE is async, return 1 (optimistic)
         return 1
+
+    def get_last_recovery_timestamp(
+        self,
+        metric_name: str
+    ) -> Optional[datetime]:
+        """
+        Get timestamp of last sent recovery notification for a metric.
+
+        Args:
+            metric_name: Metric identifier
+
+        Returns:
+            Timestamp of last sent recovery, or None if never sent
+        """
+        full_table_name = self._manager.get_full_table_name(
+            TABLE_TASKS, use_internal=True
+        )
+
+        query = f"""
+        SELECT last_recovery_sent
+        FROM {full_table_name}
+        WHERE metric_name = %(metric_name)s
+          AND detector_id = 'pipeline'
+          AND process_type = 'pipeline'
+        LIMIT 1
+        """
+
+        results = self._manager.execute_query(
+            query,
+            params={"metric_name": metric_name}
+        )
+
+        if not results or not results[0].get("last_recovery_sent"):
+            return None
+
+        last_sent = results[0]["last_recovery_sent"]
+
+        if hasattr(last_sent, 'tzinfo') and last_sent.tzinfo is not None:
+            last_sent = last_sent.replace(tzinfo=None)
+
+        return last_sent
+
+    def update_recovery_timestamp(
+        self,
+        metric_name: str,
+        timestamp: datetime,
+    ) -> int:
+        """
+        Update last_recovery_sent timestamp after sending recovery notification.
+
+        Args:
+            metric_name: Metric identifier
+            timestamp: Timestamp when recovery was sent
+
+        Returns:
+            Number of rows updated (typically 1)
+        """
+        full_table_name = self._manager.get_full_table_name(
+            TABLE_TASKS, use_internal=True
+        )
+
+        if hasattr(timestamp, 'tzinfo') and timestamp.tzinfo is not None:
+            timestamp = timestamp.replace(tzinfo=None)
+
+        update_query = f"""
+        ALTER TABLE {full_table_name}
+        UPDATE
+            last_recovery_sent = %(timestamp)s,
+            updated_at = %(timestamp)s
+        WHERE metric_name = %(metric_name)s
+          AND detector_id = 'pipeline'
+          AND process_type = 'pipeline'
+        """
+
+        self._manager.execute_query(
+            update_query,
+            params={
+                "metric_name": metric_name,
+                "timestamp": timestamp
+            }
+        )
+
+        return 1

{detectkit-0.3.5 → detectkit-0.3.6}/detectkit/database/tables.py

@@ -128,6 +128,11 @@ def get_tasks_table_model() -> TableModel:
                 nullable=True
             ),
             ColumnDefinition("alert_count", "UInt32", default="0"),
+            ColumnDefinition(
+                "last_recovery_sent",
+                "Nullable(DateTime64(3, 'UTC'))",
+                nullable=True
+            ),
         ],
         primary_key=["metric_name", "detector_id", "process_type"],
         engine="MergeTree",

{detectkit-0.3.5 → detectkit-0.3.6}/detectkit/orchestration/task_manager.py

@@ -643,7 +643,36 @@ class TaskManager:
             else:
                 click.echo(click.style("  └─ No valid alert channels available", fg="yellow"))
         else:
-            click.echo("  └─ No alert needed (conditions not met)")
+            # Check recovery notification
+            if alerting_config.notify_on_recovery:
+                should_recover, recovery_data = orchestrator.should_send_recovery(
+                    recent_detections
+                )
+
+                if should_recover:
+                    click.echo(click.style(f"  │ ✓ Recovery detected! Sending to {len(alerting_config.channels)} channel(s)...", fg="green", bold=True))
+
+                    channels = self._create_alert_channels(alerting_config.channels)
+
+                    if channels:
+                        results = orchestrator.send_recovery(
+                            recovery_data,
+                            channels,
+                            template=alerting_config.template_recovery,
+                        )
+                        recovery_sent = sum(1 for success in results.values() if success)
+
+                        for channel_name, success in results.items():
+                            status = click.style("✓", fg="green") if success else click.style("✗", fg="red")
+                            click.echo(f"  │   {status} {channel_name}")
+
+                        click.echo(click.style(f"  └─ Sent {recovery_sent}/{len(channels)} recovery notifications", fg="green"))
+                    else:
+                        click.echo(click.style("  └─ No valid alert channels available", fg="yellow"))
+                else:
+                    click.echo("  └─ No alert needed (conditions not met)")
+            else:
+                click.echo("  └─ No alert needed (conditions not met)")
 
         return {"alerts_sent": alerts_sent}
 

{detectkit-0.3.5 → detectkit-0.3.6/detectkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detectkit
-Version: 0.3.5
+Version: 0.3.6
 Summary: Metric monitoring with automatic anomaly detection
 Author: detectkit team
 License: MIT
@@ -74,6 +74,13 @@ Published to PyPI: https://pypi.org/project/detectkit/
 
 Complete rewrite with modern architecture and full documentation (2025).
 
+### What's New in v0.3.6
+
+✅ **Recovery Notifications** - Know when your metrics stabilize
+- `notify_on_recovery: true` sends a message when anomaly resolves
+- Custom `template_recovery` for recovery message format
+- One notification per incident, no duplicates
+
 ### What's New in v0.3.0
 
 🎯 **Alert Cooldown** - Prevent alert spam from persistent anomalies

{detectkit-0.3.5 → detectkit-0.3.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "detectkit"
-version = "0.3.5"
+version = "0.3.6"
 description = "Metric monitoring with automatic anomaly detection"
 readme = "README.md"
 requires-python = ">=3.10"