detectkit 0.2.8__tar.gz → 0.3.1__tar.gz

{detectkit-0.2.8/detectkit.egg-info → detectkit-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detectkit
-Version: 0.2.8
+Version: 0.3.1
 Summary: Metric monitoring with automatic anomaly detection
 Author: detectkit team
 License: MIT
@@ -68,12 +68,19 @@ Dynamic: license-file
 
 ## Status
 
-✅ **Production Ready** - Version 0.1.2
+✅ **Production Ready** - Version 0.3.0
 
 Published to PyPI: https://pypi.org/project/detectkit/
 
 Complete rewrite with modern architecture and full documentation (2025).
 
+### What's New in v0.3.0
+
+🎯 **Alert Cooldown** - Prevent alert spam from persistent anomalies
+- Configure minimum time between alerts (`alert_cooldown: "30min"`)
+- Automatic recovery detection (`cooldown_reset_on_recovery: true`)
+- Stops duplicate alerts during long-running issues
+
 ## Features
 
 - ✅ **Pure numpy arrays** - No pandas dependency in core logic
@@ -225,13 +232,14 @@ This project is currently in active development. Contributions are welcome once
 
 ## Changelog
 
-### 0.1.0 (2025-11-07)
-- Initial release with complete rewrite
-- ✅ Core foundation: models, database, config
-- ✅ Metric loading with gap filling and seasonality extraction
-- ✅ Statistical detectors (Z-Score, MAD, IQR, Manual Bounds)
-- ✅ Alert channels (Webhook, Mattermost, Slack)
-- ✅ Alert orchestration with consecutive anomaly logic
-- ✅ Task manager for pipeline execution
-- ✅ CLI commands (dtk init, dtk run)
-- 📊 287 unit tests, 87% coverage
+See [CHANGELOG.md](CHANGELOG.md) for complete version history.
+
+### Recent Releases
+
+**[0.3.0]** (2025-11-10) - Alert cooldown system, spam prevention
+**[0.2.8]** (2025-11-10) - Fix incomplete interval detection
+**[0.2.7]** (2025-11-10) - Add _dtk_metrics table
+**[0.2.0]** (2025-11-06) - Detector preprocessing and value weighting
+**[0.1.0]** (2025-11-03) - Initial release
+
+[Full changelog →](CHANGELOG.md)

{detectkit-0.2.8 → detectkit-0.3.1}/README.md

@@ -6,12 +6,19 @@
 
 ## Status
 
-✅ **Production Ready** - Version 0.1.2
+✅ **Production Ready** - Version 0.3.0
 
 Published to PyPI: https://pypi.org/project/detectkit/
 
 Complete rewrite with modern architecture and full documentation (2025).
 
+### What's New in v0.3.0
+
+🎯 **Alert Cooldown** - Prevent alert spam from persistent anomalies
+- Configure minimum time between alerts (`alert_cooldown: "30min"`)
+- Automatic recovery detection (`cooldown_reset_on_recovery: true`)
+- Stops duplicate alerts during long-running issues
+
 ## Features
 
 - ✅ **Pure numpy arrays** - No pandas dependency in core logic
@@ -163,13 +170,14 @@ This project is currently in active development. Contributions are welcome once
 
 ## Changelog
 
-### 0.1.0 (2025-11-07)
-- Initial release with complete rewrite
-- ✅ Core foundation: models, database, config
-- ✅ Metric loading with gap filling and seasonality extraction
-- ✅ Statistical detectors (Z-Score, MAD, IQR, Manual Bounds)
-- ✅ Alert channels (Webhook, Mattermost, Slack)
-- ✅ Alert orchestration with consecutive anomaly logic
-- ✅ Task manager for pipeline execution
-- ✅ CLI commands (dtk init, dtk run)
-- 📊 287 unit tests, 87% coverage
+See [CHANGELOG.md](CHANGELOG.md) for complete version history.
+
+### Recent Releases
+
+**[0.3.0]** (2025-11-10) - Alert cooldown system, spam prevention
+**[0.2.8]** (2025-11-10) - Fix incomplete interval detection
+**[0.2.7]** (2025-11-10) - Add _dtk_metrics table
+**[0.2.0]** (2025-11-06) - Detector preprocessing and value weighting
+**[0.1.0]** (2025-11-03) - Initial release
+
+[Full changelog →](CHANGELOG.md)

{detectkit-0.2.8 → detectkit-0.3.1}/detectkit/alerting/orchestrator.py

@@ -73,6 +73,8 @@ class AlertOrchestrator:
         interval: Interval,
         conditions: Optional[AlertConditions] = None,
         timezone_display: str = "UTC",
+        internal=None,  # InternalTablesManager (optional, for cooldown tracking)
+        alert_config=None,  # AlertConfig (optional, for cooldown settings)
     ):
         """
         Initialize alert orchestrator.
@@ -82,11 +84,15 @@ class AlertOrchestrator:
             interval: Metric interval
             conditions: Alert conditions (defaults to AlertConditions())
             timezone_display: Timezone for alert display (default: UTC)
+            internal: InternalTablesManager instance (optional, for cooldown tracking)
+            alert_config: AlertConfig instance (optional, for cooldown settings)
         """
         self.metric_name = metric_name
         self.interval = interval
         self.conditions = conditions or AlertConditions()
         self.timezone_display = timezone_display
+        self.internal = internal
+        self.alert_config = alert_config
 
     def should_alert(
         self,
@@ -106,11 +112,16 @@ class AlertOrchestrator:
         Logic:
             1. Check if enough detectors triggered (min_detectors)
             2. Check consecutive anomalies with direction matching
-            3. Return decision and formatted AlertData
+            3. Check alert cooldown (if configured)
+            4. Return decision and formatted AlertData
         """
         if not recent_detections:
             return False, None
 
+        # NEW: Check cooldown FIRST (before expensive checks)
+        if self._is_in_cooldown():
+            return False, None
+
         # Group detections by timestamp
         detections_by_time = self._group_by_timestamp(recent_detections)
 
@@ -316,6 +327,15 @@ class AlertOrchestrator:
                 print(f"Error sending alert via {channel_name}: {e}")
                 results[channel_name] = False
 
+        # NEW: Update alert timestamp after sending (for cooldown tracking)
+        if any(results.values()) and self.internal:
+            # At least one channel succeeded - update timestamp
+            self.internal.update_alert_timestamp(
+                metric_name=self.metric_name,
+                timestamp=datetime.utcnow(),
+                increment_count=True
+            )
+
         return results
 
     def get_last_complete_point(self, now: Optional[datetime] = None) -> datetime:
@@ -357,6 +377,150 @@ class AlertOrchestrator:
 
         return datetime.fromtimestamp(last_complete_seconds, tz=timezone.utc)
 
+    def _is_in_cooldown(self) -> bool:
+        """
+        Check if alert is currently in cooldown period.
+
+        Returns:
+            True if in cooldown (should NOT send alert), False otherwise
+
+        Logic:
+            1. If alert_cooldown not configured → return False (no cooldown)
+            2. Get last_alert_sent timestamp from database
+            3. If never sent → return False (no cooldown)
+            4. Calculate elapsed time since last alert
+            5. If cooldown_reset_on_recovery=True:
+               - Check if recovery happened since last alert
+               - If yes → return False (cooldown reset)
+            6. If elapsed < cooldown_interval → return True (in cooldown)
+            7. Otherwise → return False (cooldown expired)
+        """
+        # No cooldown configured
+        if not self.alert_config or not self.alert_config.alert_cooldown:
+            return False
+
+        # No internal manager (can't check cooldown)
+        if not self.internal:
+            return False
+
+        # Get last alert timestamp
+        last_sent = self.internal.get_last_alert_timestamp(self.metric_name)
+
+        if not last_sent:
+            return False  # Never sent alert before
+
+        # Parse cooldown interval
+        from detectkit.core.interval import Interval
+        cooldown_interval = Interval(self.alert_config.alert_cooldown)
+        cooldown_seconds = cooldown_interval.seconds
+
+        # Calculate elapsed time
+        now = datetime.utcnow()
+        elapsed = (now - last_sent).total_seconds()
+
+        # Check recovery reset (if enabled)
+        if self.alert_config.cooldown_reset_on_recovery:
+            # Check if recovery happened since last alert
+            has_recovery = self._check_recovery_since_last_alert(last_sent)
+
+            if has_recovery:
+                return False  # Cooldown reset by recovery
+
+        # Check if still in cooldown
+        return elapsed < cooldown_seconds
+
+    def _check_recovery_since_last_alert(
+        self,
+        last_alert_timestamp: datetime
+    ) -> bool:
+        """
+        Check if recovery happened since last alert was sent.
+
+        Recovery means: consecutive anomalies count dropped below threshold,
+        indicating the metric returned to normal state.
+
+        Args:
+            last_alert_timestamp: Timestamp when last alert was sent
+
+        Returns:
+            True if recovery detected, False otherwise
+
+        Logic:
+            1. Load detections created after last_alert_timestamp
+            2. Count consecutive anomalies using same logic as should_alert()
+            3. If consecutive < required → recovery happened
+            4. If consecutive >= required → still in anomaly state
+        """
+        if not self.internal:
+            return False
+
+        # Get last complete point
+        last_point = self.get_last_complete_point()
+
+        # Load detections created AFTER last alert
+        # We need enough points to check consecutive anomalies
+        num_points = self.conditions.consecutive_anomalies + 5  # +5 for margin
+
+        recent_detections = self.internal.get_recent_detections(
+            metric_name=self.metric_name,
+            last_point=last_point,
+            num_points=num_points,
+            created_after=last_alert_timestamp  # Only detections AFTER last alert
+        )
+
+        if not recent_detections:
+            # No new detections → assume recovery
+            return True
+
+        # Convert to DetectionRecord format
+        detection_records = []
+        for det in recent_detections:
+            # Group has multiple detectors per timestamp
+            for i in range(len(det["detector_ids"])):
+                # Parse detection metadata
+                try:
+                    import json
+                    metadata = json.loads(det["detector_params_list"][i])
+                except:
+                    metadata = {}
+
+                # Determine direction
+                value = det["value"]
+                conf_lower = det["confidence_lowers"][i]
+                conf_upper = det["confidence_uppers"][i]
+
+                if value < conf_lower:
+                    direction = "down"
+                elif value > conf_upper:
+                    direction = "up"
+                else:
+                    direction = "none"
+
+                record = DetectionRecord(
+                    timestamp=np.datetime64(det["timestamp"]),
+                    detector_name=det["detector_names"][i],
+                    detector_id=det["detector_ids"][i],
+                    detector_params=det["detector_params_list"][i],
+                    value=value,
+                    is_anomaly=det["is_anomaly_flags"][i],
+                    confidence_lower=conf_lower,
+                    confidence_upper=conf_upper,
+                    direction=direction,
+                    severity=0.0,  # Not used for recovery check
+                    detection_metadata=metadata
+                )
+                detection_records.append(record)
+
+        # Count consecutive anomalies (same logic as should_alert)
+        consecutive = self._count_consecutive_anomalies(
+            detections=detection_records,
+            min_detectors=self.conditions.min_detectors,
+            direction=self.conditions.direction
+        )
+
+        # Recovery = consecutive dropped below threshold
+        return consecutive < self.conditions.consecutive_anomalies
+
     def __repr__(self) -> str:
         """String representation."""
         return (

{detectkit-0.2.8 → detectkit-0.3.1}/detectkit/cli/commands/run.py

@@ -369,17 +369,26 @@ def find_metrics_by_tag(metrics_dir: Path, tag: str) -> List[Path]:
 
     matching_metrics = []
 
-    for metric_file in metrics_dir.glob("**/*.yml"):
-        try:
-            with open(metric_file) as f:
-                config = yaml.safe_load(f)
-
-            if config and "tags" in config:
-                if tag in config["tags"]:
-                    matching_metrics.append(metric_file)
-        except Exception:
-            # Skip files that can't be parsed
-            continue
+    # Search both .yml and .yaml extensions (consistent with find_metric_by_name)
+    for pattern in ["**/*.yml", "**/*.yaml"]:
+        for metric_file in metrics_dir.glob(pattern):
+            try:
+                with open(metric_file) as f:
+                    config = yaml.safe_load(f)
+
+                if config and "tags" in config:
+                    if tag in config["tags"]:
+                        matching_metrics.append(metric_file)
+            except Exception as e:
+                # Warn about unparseable files but continue searching
+                click.echo(
+                    click.style(
+                        f"Warning: Skipping {metric_file.relative_to(metrics_dir.parent)}: {e}",
+                        fg="yellow"
+                    ),
+                    err=True
+                )
+                continue
 
     return matching_metrics
 
@@ -406,8 +415,15 @@ def find_metric_by_name(metrics_dir: Path, name: str) -> Optional[Path]:
 
                 if config and config.get("name") == name:
                     return metric_file
-            except Exception:
-                # Skip files that can't be parsed
+            except Exception as e:
+                # Warn about unparseable files but continue searching
+                click.echo(
+                    click.style(
+                        f"Warning: Skipping {metric_file.relative_to(metrics_dir.parent)}: {e}",
+                        fg="yellow"
+                    ),
+                    err=True
+                )
                 continue
 
     return None

{detectkit-0.2.8 → detectkit-0.3.1}/detectkit/config/metric_config.py

@@ -141,6 +141,8 @@ class AlertConfig(BaseModel):
         no_data_alert: Whether to alert when data is missing
         template_single: Custom template for single anomaly alert
         template_consecutive: Custom template for consecutive anomalies alert
+        alert_cooldown: Minimum interval between alerts (e.g., "30min", 1800 seconds)
+        cooldown_reset_on_recovery: Whether to reset cooldown when anomaly recovers
     """
 
     enabled: bool = Field(default=True, description="Enable alerting")
@@ -168,6 +170,17 @@ class AlertConfig(BaseModel):
     template_consecutive: Optional[str] = Field(
         default=None, description="Custom template for consecutive anomalies"
     )
+    alert_cooldown: Optional[Union[str, int]] = Field(
+        default=None,
+        description="Minimum interval between alerts (e.g., '30min', 1800). "
+                    "If None, no cooldown is applied (alerts sent every time conditions are met)."
+    )
+    cooldown_reset_on_recovery: bool = Field(
+        default=True,
+        description="Reset cooldown timer when anomaly recovers to normal. "
+                    "Only applies if alert_cooldown is set. "
+                    "True = cooldown resets on recovery, False = strict cooldown independent of recovery."
+    )
 
     @field_validator("consecutive_anomalies")
     @classmethod

{detectkit-0.2.8 → detectkit-0.3.1}/detectkit/database/internal_tables.py

@@ -841,3 +841,125 @@ class InternalTablesManager:
             key_columns={"metric_name": metric_config.name},
             data=data
         )
+
+    def get_last_alert_timestamp(
+        self,
+        metric_name: str
+    ) -> Optional[datetime]:
+        """
+        Get timestamp of last sent alert for a metric.
+
+        Used for alert cooldown tracking - prevents sending alerts
+        too frequently for the same metric.
+
+        Args:
+            metric_name: Metric identifier
+
+        Returns:
+            Timestamp of last sent alert, or None if never sent
+
+        Example:
+            >>> last_sent = internal.get_last_alert_timestamp("cpu_usage")
+            >>> if last_sent:
+            ...     elapsed = (datetime.utcnow() - last_sent).total_seconds()
+            ...     print(f"Last alert sent {elapsed}s ago")
+        """
+        full_table_name = self._manager.get_full_table_name(
+            TABLE_TASKS, use_internal=True
+        )
+
+        # Query for pipeline task (detector_id="pipeline", process_type="pipeline")
+        query = f"""
+        SELECT last_alert_sent
+        FROM {full_table_name}
+        WHERE metric_name = %(metric_name)s
+          AND detector_id = 'pipeline'
+          AND process_type = 'pipeline'
+        LIMIT 1
+        """
+
+        results = self._manager.execute_query(
+            query,
+            params={"metric_name": metric_name}
+        )
+
+        if not results or not results[0]["last_alert_sent"]:
+            return None
+
+        last_sent = results[0]["last_alert_sent"]
+
+        # Normalize to naive datetime if needed
+        if hasattr(last_sent, 'tzinfo') and last_sent.tzinfo is not None:
+            last_sent = last_sent.replace(tzinfo=None)
+
+        return last_sent
+
+    def update_alert_timestamp(
+        self,
+        metric_name: str,
+        timestamp: datetime,
+        increment_count: bool = True
+    ) -> int:
+        """
+        Update last_alert_sent timestamp and optionally increment alert_count.
+
+        Called after successfully sending an alert to track cooldown state.
+
+        Args:
+            metric_name: Metric identifier
+            timestamp: Timestamp when alert was sent (typically datetime.utcnow())
+            increment_count: Whether to increment alert_count (default: True)
+
+        Returns:
+            Number of rows updated (typically 1)
+
+        Example:
+            >>> # After sending alert
+            >>> internal.update_alert_timestamp(
+            ...     "cpu_usage",
+            ...     datetime.utcnow(),
+            ...     increment_count=True
+            ... )
+        """
+        full_table_name = self._manager.get_full_table_name(
+            TABLE_TASKS, use_internal=True
+        )
+
+        # Normalize timestamp to naive if needed
+        if hasattr(timestamp, 'tzinfo') and timestamp.tzinfo is not None:
+            timestamp = timestamp.replace(tzinfo=None)
+
+        if increment_count:
+            # Update with alert_count increment
+            update_query = f"""
+            ALTER TABLE {full_table_name}
+            UPDATE
+                last_alert_sent = %(timestamp)s,
+                alert_count = alert_count + 1,
+                updated_at = %(timestamp)s
+            WHERE metric_name = %(metric_name)s
+              AND detector_id = 'pipeline'
+              AND process_type = 'pipeline'
+            """
+        else:
+            # Update without alert_count increment
+            update_query = f"""
+            ALTER TABLE {full_table_name}
+            UPDATE
+                last_alert_sent = %(timestamp)s,
+                updated_at = %(timestamp)s
+            WHERE metric_name = %(metric_name)s
+              AND detector_id = 'pipeline'
+              AND process_type = 'pipeline'
+            """
+
+        self._manager.execute_query(
+            update_query,
+            params={
+                "metric_name": metric_name,
+                "timestamp": timestamp
+            }
+        )
+
+        # ClickHouse ALTER TABLE UPDATE is async, return 1 (optimistic)
+        return 1

{detectkit-0.2.8 → detectkit-0.3.1}/detectkit/database/tables.py

@@ -97,12 +97,15 @@ def get_tasks_table_model() -> TableModel:
         - last_processed_timestamp: Last successfully processed timestamp
         - error_message: Error message if failed (nullable)
         - timeout_seconds: Task timeout in seconds
+        - last_alert_sent: Timestamp of last sent alert (nullable, for cooldown tracking)
+        - alert_count: Number of alerts sent for this metric (for statistics)
 
     Primary Key: (metric_name, detector_id, process_type)
 
-    This table serves dual purpose:
+    This table serves multiple purposes:
     1. Locking: Only one process can run for a given (metric, detector, type)
     2. Resume: Stores last_processed_timestamp to resume from interruptions
+    3. Alert cooldown: Tracks last_alert_sent timestamp to prevent alert spam
     """
     return TableModel(
         columns=[
@@ -119,6 +122,12 @@ def get_tasks_table_model() -> TableModel:
             ),
             ColumnDefinition("error_message", "Nullable(String)", nullable=True),
             ColumnDefinition("timeout_seconds", "Int32"),
+            ColumnDefinition(
+                "last_alert_sent",
+                "Nullable(DateTime64(3, 'UTC'))",
+                nullable=True
+            ),
+            ColumnDefinition("alert_count", "UInt32", default="0"),
         ],
         primary_key=["metric_name", "detector_id", "process_type"],
         engine="MergeTree",

{detectkit-0.2.8 → detectkit-0.3.1}/detectkit/orchestration/task_manager.py

@@ -577,6 +577,8 @@ class TaskManager:
                 consecutive_anomalies=alerting_config.consecutive_anomalies,
             ),
             timezone_display="UTC",
+            internal=self.internal,  # For cooldown tracking
+            alert_config=alerting_config,  # For cooldown settings
         )
 
         # Get last complete point

{detectkit-0.2.8 → detectkit-0.3.1/detectkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detectkit
-Version: 0.2.8
+Version: 0.3.1
 Summary: Metric monitoring with automatic anomaly detection
 Author: detectkit team
 License: MIT
@@ -68,12 +68,19 @@ Dynamic: license-file
 
 ## Status
 
-✅ **Production Ready** - Version 0.1.2
+✅ **Production Ready** - Version 0.3.0
 
 Published to PyPI: https://pypi.org/project/detectkit/
 
 Complete rewrite with modern architecture and full documentation (2025).
 
+### What's New in v0.3.0
+
+🎯 **Alert Cooldown** - Prevent alert spam from persistent anomalies
+- Configure minimum time between alerts (`alert_cooldown: "30min"`)
+- Automatic recovery detection (`cooldown_reset_on_recovery: true`)
+- Stops duplicate alerts during long-running issues
+
 ## Features
 
 - ✅ **Pure numpy arrays** - No pandas dependency in core logic
@@ -225,13 +232,14 @@ This project is currently in active development. Contributions are welcome once
 
 ## Changelog
 
-### 0.1.0 (2025-11-07)
-- Initial release with complete rewrite
-- ✅ Core foundation: models, database, config
-- ✅ Metric loading with gap filling and seasonality extraction
-- ✅ Statistical detectors (Z-Score, MAD, IQR, Manual Bounds)
-- ✅ Alert channels (Webhook, Mattermost, Slack)
-- ✅ Alert orchestration with consecutive anomaly logic
-- ✅ Task manager for pipeline execution
-- ✅ CLI commands (dtk init, dtk run)
-- 📊 287 unit tests, 87% coverage
+See [CHANGELOG.md](CHANGELOG.md) for complete version history.
+
+### Recent Releases
+
+**[0.3.0]** (2025-11-10) - Alert cooldown system, spam prevention
+**[0.2.8]** (2025-11-10) - Fix incomplete interval detection
+**[0.2.7]** (2025-11-10) - Add _dtk_metrics table
+**[0.2.0]** (2025-11-06) - Detector preprocessing and value weighting
+**[0.1.0]** (2025-11-03) - Initial release
+
+[Full changelog →](CHANGELOG.md)

{detectkit-0.2.8 → detectkit-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "detectkit"
-version = "0.2.8"
+version = "0.3.1"
 description = "Metric monitoring with automatic anomaly detection"
 readme = "README.md"
 requires-python = ">=3.10"