detectkit 0.2.4__py3-none-any.whl

detectkit/detectors/statistical/mad.py

@@ -0,0 +1,478 @@
+"""
+Median Absolute Deviation (MAD) anomaly detector.
+
+MAD is a robust statistical method for outlier detection that:
+- Uses median (robust to outliers) instead of mean
+- Measures deviation from median using MAD instead of std
+- Less sensitive to extreme values than Z-Score
+
+Formula:
+- median_val = median(values)
+- mad_val = median(|values - median_val|)
+- lower_bound = median_val - threshold × mad_val
+- upper_bound = median_val + threshold × mad_val
+
+Seasonality support:
+- Groups data by seasonality components
+- Computes global statistics (entire window)
+- Computes component statistics (per group)
+- Applies multipliers to adjust confidence intervals
+"""
+
+from typing import Any, Dict, List, Optional, Tuple, Union
+import json
+
+import numpy as np
+
+from detectkit.detectors.base import BaseDetector, DetectionResult
+
+
+class MADDetector(BaseDetector):
+    """
+    Median Absolute Deviation detector for anomaly detection.
+
+    Detects anomalies by comparing values against confidence intervals
+    based on median and MAD (median absolute deviation).
+
+    Parameters:
+        threshold (float): Number of MAD units from median (default: 3.0)
+            - 3.0 is standard (similar to 3-sigma in Z-Score)
+            - Higher = less sensitive (fewer anomalies)
+            - Lower = more sensitive (more anomalies)
+
+        window_size (int): Historical window size in points (default: 100)
+            - Uses last N points to compute statistics
+            - Larger = more stable but less responsive
+            - Smaller = more responsive but less stable
+
+        min_samples (int): Minimum samples required for detection (default: 30)
+            - Skip detection if window has fewer valid points
+            - Ensures statistical reliability
+
+    Example:
+        >>> detector = MADDetector(threshold=3.0, window_size=100)
+        >>> results = detector.detect(data)
+        >>> for r in results:
+        ...     if r.is_anomaly:
+        ...         print(f"Anomaly: {r.value} outside [{r.confidence_lower}, {r.confidence_upper}]")
+    """
+
+    def __init__(
+        self,
+        threshold: float = 3.0,
+        window_size: int = 100,
+        min_samples: int = 30,
+        seasonality_components: Optional[List[Union[str, List[str]]]] = None,
+        min_samples_per_group: int = 10,
+        input_type: str = "values",
+        smoothing: Optional[str] = None,
+        smoothing_alpha: float = 0.3,
+        smoothing_window: int = 10,
+        window_weights: Optional[str] = None,
+        weight_decay: float = 0.95,
+    ):
+        """
+        Initialize MAD detector with parameters.
+
+        Args:
+            threshold: Number of MAD units from median
+            window_size: Historical window size in points
+            min_samples: Minimum total samples required
+            seasonality_components: Optional list of seasonality groups
+                Examples:
+                - ["day_of_week"] - single component
+                - [["day_of_week", "hour"]] - combined group
+                - ["day", ["hour", "minute"]] - separate + combined
+            min_samples_per_group: Minimum samples per seasonality group
+            input_type: Input transformation type (values, changes, absolute_changes, log_changes)
+            smoothing: Smoothing method (None, ema, sma)
+            smoothing_alpha: EMA smoothing factor (0 < alpha <= 1)
+            smoothing_window: SMA window size
+            window_weights: Weighting method (None, exponential, linear)
+            weight_decay: Decay factor for exponential weights (0 < decay < 1)
+        """
+        super().__init__(
+            threshold=threshold,
+            window_size=window_size,
+            min_samples=min_samples,
+            seasonality_components=seasonality_components,
+            min_samples_per_group=min_samples_per_group,
+            input_type=input_type,
+            smoothing=smoothing,
+            smoothing_alpha=smoothing_alpha,
+            smoothing_window=smoothing_window,
+            window_weights=window_weights,
+            weight_decay=weight_decay,
+        )
+
+    def _validate_params(self):
+        """Validate detector parameters."""
+        threshold = self.params.get("threshold")
+        if threshold is None or threshold <= 0:
+            raise ValueError("threshold must be positive")
+
+        window_size = self.params.get("window_size")
+        if window_size is None or window_size < 1:
+            raise ValueError("window_size must be at least 1")
+
+        min_samples = self.params.get("min_samples")
+        if min_samples is None or min_samples < 1:
+            raise ValueError("min_samples must be at least 1")
+
+        if min_samples > window_size:
+            raise ValueError("min_samples cannot exceed window_size")
+
+    def _parse_seasonality_data(
+        self, seasonality_data: np.ndarray, seasonality_columns: List[str]
+    ) -> Dict[str, np.ndarray]:
+        """
+        Parse seasonality JSON strings into structured data.
+
+        Args:
+            seasonality_data: Array of JSON strings
+            seasonality_columns: List of column names
+
+        Returns:
+            Dict with column names as keys, numpy arrays as values
+
+        Example:
+            Input: ['{"day": 1, "hour": 10}', '{"day": 1, "hour": 11}']
+            Output: {"day": array([1, 1]), "hour": array([10, 11])}
+        """
+        if len(seasonality_data) == 0:
+            return {}
+
+        # Parse all JSON strings
+        parsed_data = {col: [] for col in seasonality_columns}
+
+        for json_str in seasonality_data:
+            if json_str is None or json_str == "{}":
+                # Empty seasonality - add None for all columns
+                for col in seasonality_columns:
+                    parsed_data[col].append(None)
+            else:
+                try:
+                    data_dict = json.loads(json_str)
+                    for col in seasonality_columns:
+                        parsed_data[col].append(data_dict.get(col))
+                except (json.JSONDecodeError, TypeError):
+                    # Invalid JSON - add None
+                    for col in seasonality_columns:
+                        parsed_data[col].append(None)
+
+        # Convert to numpy arrays
+        return {col: np.array(vals) for col, vals in parsed_data.items()}
+
+    def _create_seasonality_mask(
+        self,
+        seasonality_dict: Dict[str, np.ndarray],
+        window_start: int,
+        current_idx: int,
+        group_columns: List[str],
+    ) -> np.ndarray:
+        """
+        Create boolean mask for seasonality group.
+
+        Args:
+            seasonality_dict: Parsed seasonality data
+            window_start: Start index of window
+            current_idx: Current point index
+            group_columns: List of columns to group by (e.g., ["day", "hour"])
+
+        Returns:
+            Boolean mask for window indices matching current point's seasonality
+
+        Example:
+            Current point: day=1, hour=10
+            Group columns: ["day", "hour"]
+            Returns: mask where (day==1) AND (hour==10)
+        """
+        if not group_columns or not seasonality_dict:
+            # No grouping - return all True
+            window_size = current_idx - window_start
+            return np.ones(window_size, dtype=bool)
+
+        # Get current point's seasonality values
+        current_values = {}
+        for col in group_columns:
+            if col in seasonality_dict:
+                current_values[col] = seasonality_dict[col][current_idx]
+            else:
+                # Column not found - no filtering
+                return np.ones(current_idx - window_start, dtype=bool)
+
+        # Create combined mask (AND of all columns)
+        mask = np.ones(current_idx - window_start, dtype=bool)
+
+        for col in group_columns:
+            current_val = current_values[col]
+            window_vals = seasonality_dict[col][window_start:current_idx]
+            mask &= (window_vals == current_val)
+
+        return mask
+
+    def detect(self, data: Dict[str, np.ndarray]) -> list[DetectionResult]:
+        """
+        Perform MAD-based anomaly detection with seasonality support.
+
+        Algorithm (TECHNICAL_SPEC.md section 8):
+        1. Apply preprocessing (input_type transformation and smoothing)
+        2. Parse seasonality data
+        3. For each point:
+           - Compute global statistics (entire window)
+           - Apply weighting if specified
+           - For each seasonality group:
+             * Create mask matching current point's seasonality
+             * Compute group statistics
+             * Calculate multipliers
+           - Apply all multipliers to adjust intervals
+           - Detect anomalies
+
+        Args:
+            data: Dictionary with keys:
+                - timestamp: np.array of datetime64[ms]
+                - value: np.array of float64 (may contain NaN)
+                - seasonality_data: np.array of JSON strings
+                - seasonality_columns: list of column names
+
+        Returns:
+            List of DetectionResult for each point
+        """
+        timestamps = data["timestamp"]
+        values = data["value"]  # ORIGINAL values (always kept)
+        seasonality_data = data.get("seasonality_data", np.array([]))
+        seasonality_columns = data.get("seasonality_columns", [])
+
+        threshold = self.params["threshold"]
+        window_size = self.params["window_size"]
+        min_samples = self.params["min_samples"]
+        seasonality_components = self.params.get("seasonality_components")
+        min_samples_per_group = self.params.get("min_samples_per_group", 10)
+
+        # STEP 0: Preprocessing (smoothing + input_type transformation)
+        # Order matters: smoothing first, then input_type
+        smoothed_values = self._apply_smoothing(values)
+        processed_values = self._preprocess_input(smoothed_values)
+
+        # Parse seasonality data once
+        seasonality_dict = {}
+        if len(seasonality_data) > 0 and seasonality_columns:
+            seasonality_dict = self._parse_seasonality_data(
+                seasonality_data, seasonality_columns
+            )
+
+        results = []
+        n_points = len(timestamps)
+
+        for i in range(n_points):
+            current_val = values[i]  # ORIGINAL value
+            current_processed = processed_values[i]  # PROCESSED value
+            current_ts = timestamps[i]
+
+            # Skip NaN values (in processed)
+            if np.isnan(current_processed):
+                results.append(
+                    DetectionResult(
+                        timestamp=current_ts,
+                        value=current_val,
+                        processed_value=current_processed,
+                        is_anomaly=False,
+                        detection_metadata={"reason": "missing_data"},
+                    )
+                )
+                continue
+
+            # Get historical window (not including current point)
+            window_start = max(0, i - window_size)
+            window_processed = processed_values[window_start:i]
+
+            # Filter out NaN values from window
+            valid_mask = ~np.isnan(window_processed)
+            window_valid = window_processed[valid_mask]
+
+            # Check if we have enough samples
+            if len(window_valid) < min_samples:
+                results.append(
+                    DetectionResult(
+                        timestamp=current_ts,
+                        value=current_val,
+                        processed_value=current_processed,
+                        is_anomaly=False,
+                        detection_metadata={
+                            "reason": "insufficient_data",
+                            "window_size": int(len(window_valid)),
+                            "min_samples": min_samples,
+                        },
+                    )
+                )
+                continue
+
+            # Compute weights for window (if specified)
+            weights = self._compute_weights(len(window_valid))
+
+            # STEP 1: Compute GLOBAL statistics (entire window)
+            # Use weighted statistics if weights are not uniform
+            from detectkit.utils import weighted_median, weighted_mad
+
+            global_median = weighted_median(window_valid, weights)
+            global_mad = weighted_mad(window_valid, weights, center=global_median)
+
+            # Initialize adjusted statistics with global values
+            adjusted_median = global_median
+            adjusted_mad = global_mad
+
+            # STEP 2: Apply seasonality adjustments
+            multipliers_applied = []
+
+            if seasonality_components and seasonality_dict:
+                # Process each seasonality group
+                for group in seasonality_components:
+                    # Normalize to list (handle both str and List[str])
+                    group_cols = [group] if isinstance(group, str) else group
+
+                    # Create mask for this group
+                    season_mask = self._create_seasonality_mask(
+                        seasonality_dict, window_start, i, group_cols
+                    )
+
+                    # Apply mask to window (only valid values + seasonality match)
+                    # Both valid_mask and season_mask are same size as window_processed
+                    combined_mask = valid_mask & season_mask
+
+                    group_values = window_processed[combined_mask]
+
+                    # Check if enough samples in group
+                    if len(group_values) < min_samples_per_group:
+                        # Insufficient data - skip this group (multiplier = 1.0)
+                        multipliers_applied.append({
+                            "group": group_cols,
+                            "median_multiplier": 1.0,
+                            "mad_multiplier": 1.0,
+                            "reason": "insufficient_group_data",
+                            "group_size": int(len(group_values)),
+                        })
+                        continue
+
+                    # Compute group statistics with weights
+                    group_weights = self._compute_weights(len(group_values))
+                    group_median = weighted_median(group_values, group_weights)
+                    group_mad = weighted_mad(group_values, group_weights, center=group_median)
+
+                    # Calculate multipliers
+                    if global_median != 0:
+                        median_multiplier = group_median / global_median
+                    else:
+                        median_multiplier = 1.0
+
+                    if global_mad != 0:
+                        mad_multiplier = group_mad / global_mad
+                    else:
+                        mad_multiplier = 1.0
+
+                    # Apply multipliers
+                    adjusted_median *= median_multiplier
+                    adjusted_mad *= mad_multiplier
+
+                    multipliers_applied.append({
+                        "group": group_cols,
+                        "median_multiplier": float(median_multiplier),
+                        "mad_multiplier": float(mad_multiplier),
+                        "group_size": int(len(group_values)),
+                    })
+
+            # STEP 3: Build confidence interval
+            if adjusted_mad == 0:
+                # All values identical - any deviation is anomalous
+                confidence_lower = adjusted_median - 1e-10
+                confidence_upper = adjusted_median + 1e-10
+            else:
+                confidence_lower = adjusted_median - threshold * adjusted_mad
+                confidence_upper = adjusted_median + threshold * adjusted_mad
+
+            # STEP 4: Check if current PROCESSED value is anomalous
+            is_anomaly = (current_processed < confidence_lower) or (current_processed > confidence_upper)
+
+            # Build metadata
+            metadata = {
+                "global_median": float(global_median),
+                "global_mad": float(global_mad),
+                "adjusted_median": float(adjusted_median),
+                "adjusted_mad": float(adjusted_mad),
+                "window_size": int(len(window_valid)),
+            }
+
+            # Add preprocessing info if used
+            if self.params.get("smoothing") or self.params.get("input_type") != "values":
+                metadata["preprocessing"] = {
+                    "input_type": self.params.get("input_type", "values"),
+                    "smoothing": self.params.get("smoothing"),
+                }
+                if self.params.get("smoothing"):
+                    metadata["preprocessing"]["smoothed_value"] = float(smoothed_values[i])
+
+            if seasonality_components and multipliers_applied:
+                metadata["seasonality_groups"] = multipliers_applied
+
+            if is_anomaly:
+                if current_processed < confidence_lower:
+                    direction = "below"
+                    distance = confidence_lower - current_processed
+                else:
+                    direction = "above"
+                    distance = current_processed - confidence_upper
+
+                # Severity: how many adjusted MAD units away
+                severity = distance / adjusted_mad if adjusted_mad > 0 else float("inf")
+
+                metadata.update({
+                    "direction": direction,
+                    "severity": float(severity),
+                    "distance": float(distance),
+                })
+
+            results.append(
+                DetectionResult(
+                    timestamp=current_ts,
+                    value=current_val,  # ORIGINAL value
+                    processed_value=current_processed,  # PROCESSED value
+                    is_anomaly=is_anomaly,
+                    confidence_lower=float(confidence_lower),
+                    confidence_upper=float(confidence_upper),
+                    detection_metadata=metadata,
+                )
+            )
+
+        return results
+
+    def _get_non_default_params(self) -> Dict[str, Any]:
+        """
+        Get parameters that differ from defaults.
+
+        Excludes execution parameters (seasonality_components, min_samples_per_group)
+        from detector ID hash.
+        """
+        defaults = {
+            "threshold": 3.0,
+            "window_size": 100,
+            "min_samples": 30,
+            "min_samples_per_group": 10,
+            "input_type": "values",
+            "smoothing": None,
+            "smoothing_alpha": 0.3,
+            "smoothing_window": 10,
+            "window_weights": None,
+            "weight_decay": 0.95,
+        }
+        # Execution parameters that don't affect detector ID
+        execution_params = {
+            "seasonality_components",
+            "min_samples_per_group",
+            "smoothing_alpha",  # Only affects smoothing, not algorithm
+            "smoothing_window",  # Only affects smoothing, not algorithm
+            "weight_decay",  # Only affects weighting, not algorithm
+        }
+
+        return {
+            k: v for k, v in self.params.items()
+            if v != defaults.get(k) and k not in execution_params
+        }

detectkit/detectors/statistical/manual_bounds.py

@@ -0,0 +1,206 @@
+"""
+Manual Bounds anomaly detector.
+
+Simple detector that uses user-specified thresholds for anomaly detection.
+Useful when domain knowledge exists about acceptable ranges.
+
+Examples:
+- CPU usage should be <= 90%
+- Response time should be <= 1000ms
+- Queue size should be >= 0 and <= 10000
+"""
+
+from typing import Any, Dict, Optional
+
+import numpy as np
+
+from detectkit.detectors.base import BaseDetector, DetectionResult
+
+
+class ManualBoundsDetector(BaseDetector):
+    """
+    Manual threshold detector for anomaly detection.
+
+    Detects anomalies by comparing values against user-specified bounds.
+    Does not use historical data - purely threshold-based.
+
+    Parameters:
+        lower_bound (float | None): Minimum acceptable value (default: None = no lower limit)
+            - Values below this are anomalous
+            - None means no lower bound
+
+        upper_bound (float | None): Maximum acceptable value (default: None = no upper limit)
+            - Values above this are anomalous
+            - None means no upper bound
+
+    At least one bound must be specified.
+
+    Example:
+        >>> # Detect values above 100
+        >>> detector = ManualBoundsDetector(upper_bound=100.0)
+        >>> results = detector.detect(data)
+
+        >>> # Detect values outside [10, 90]
+        >>> detector = ManualBoundsDetector(lower_bound=10.0, upper_bound=90.0)
+        >>> results = detector.detect(data)
+    """
+
+    def __init__(
+        self,
+        lower_bound: Optional[float] = None,
+        upper_bound: Optional[float] = None,
+        input_type: str = "values",
+    ):
+        """
+        Initialize Manual Bounds detector with thresholds.
+
+        Args:
+            lower_bound: Minimum acceptable value (None = no lower limit)
+            upper_bound: Maximum acceptable value (None = no upper limit)
+            input_type: Input transformation type (values, changes, absolute_changes, log_changes)
+        """
+        super().__init__(
+            lower_bound=lower_bound,
+            upper_bound=upper_bound,
+            input_type=input_type,
+        )
+
+    def _validate_params(self):
+        """Validate detector parameters."""
+        lower_bound = self.params.get("lower_bound")
+        upper_bound = self.params.get("upper_bound")
+
+        # At least one bound must be specified
+        if lower_bound is None and upper_bound is None:
+            raise ValueError("At least one of lower_bound or upper_bound must be specified")
+
+        # If both specified, lower must be less than upper
+        if lower_bound is not None and upper_bound is not None:
+            if lower_bound >= upper_bound:
+                raise ValueError("lower_bound must be less than upper_bound")
+
+    def detect(self, data: Dict[str, np.ndarray]) -> list[DetectionResult]:
+        """
+        Perform threshold-based anomaly detection.
+
+        Simply checks if each value is outside the specified bounds.
+        Does not use historical window - purely threshold-based.
+
+        Args:
+            data: Dictionary with keys:
+                - timestamp: np.array of datetime64[ms]
+                - value: np.array of float64 (may contain NaN)
+                - seasonality_data: np.array of JSON strings (not used)
+                - seasonality_columns: list of column names (not used)
+
+        Returns:
+            List of DetectionResult for each point
+
+        Notes:
+            - NaN values are skipped (marked as non-anomalous)
+            - No historical window needed
+            - No minimum samples requirement
+        """
+        timestamps = data["timestamp"]
+        values = data["value"]  # ORIGINAL values (always kept)
+        lower_bound = self.params.get("lower_bound")
+        upper_bound = self.params.get("upper_bound")
+
+        # STEP 0: Preprocessing (input_type transformation only, no smoothing)
+        # Note: ManualBounds doesn't use smoothing or weights (no historical window)
+        processed_values = self._preprocess_input(values)
+
+        results = []
+        n_points = len(timestamps)
+
+        for i in range(n_points):
+            current_val = values[i]  # ORIGINAL value
+            current_processed = processed_values[i]  # PROCESSED value
+            current_ts = timestamps[i]
+
+            # Skip NaN values (in processed)
+            if np.isnan(current_processed):
+                results.append(
+                    DetectionResult(
+                        timestamp=current_ts,
+                        value=current_val,
+                        processed_value=current_processed,
+                        is_anomaly=False,
+                        detection_metadata={"reason": "missing_data"},
+                    )
+                )
+                continue
+
+            # Check bounds on PROCESSED value
+            is_anomaly = False
+            direction = None
+            distance = 0.0
+
+            if lower_bound is not None and current_processed < lower_bound:
+                is_anomaly = True
+                direction = "below"
+                distance = lower_bound - current_processed
+
+            if upper_bound is not None and current_processed > upper_bound:
+                is_anomaly = True
+                direction = "above"
+                distance = current_processed - upper_bound
+
+            # Prepare metadata
+            metadata = {}
+
+            # Add preprocessing info if used
+            if self.params.get("input_type") != "values":
+                metadata["preprocessing"] = {
+                    "input_type": self.params.get("input_type", "values"),
+                }
+
+            if is_anomaly:
+                metadata["direction"] = direction
+                metadata["distance"] = float(distance)
+
+                # Severity: relative distance from bound
+                if direction == "below":
+                    # How far below as percentage of range
+                    if upper_bound is not None:
+                        bound_range = upper_bound - lower_bound
+                        severity = distance / bound_range if bound_range > 0 else float("inf")
+                    else:
+                        # No upper bound, just use absolute distance
+                        severity = distance
+                else:  # above
+                    if lower_bound is not None:
+                        bound_range = upper_bound - lower_bound
+                        severity = distance / bound_range if bound_range > 0 else float("inf")
+                    else:
+                        severity = distance
+
+                metadata["severity"] = float(severity)
+
+            results.append(
+                DetectionResult(
+                    timestamp=current_ts,
+                    value=current_val,  # ORIGINAL value
+                    processed_value=current_processed,  # PROCESSED value
+                    is_anomaly=is_anomaly,
+                    confidence_lower=lower_bound,
+                    confidence_upper=upper_bound,
+                    detection_metadata=metadata,
+                )
+            )
+
+        return results
+
+    def _get_non_default_params(self) -> Dict[str, Any]:
+        """
+        Get parameters that differ from defaults.
+
+        Returns all specified bounds plus input_type if not default.
+        """
+        defaults = {
+            "input_type": "values",
+        }
+        return {
+            k: v for k, v in self.params.items()
+            if v is not None and v != defaults.get(k)
+        }