detectkit 0.1.0__py3-none-any.whl

detectkit/database/manager.py

@@ -0,0 +1,324 @@
+"""
+Base database manager interface.
+
+Provides universal methods for database operations WITHOUT hardcoding
+specific table logic (e.g., _dtk_datapoints, _dtk_detections).
+
+The manager is database-agnostic and provides generic operations:
+- execute_query(): Run SQL and return results
+- create_table(): Create table from TableModel
+- table_exists(): Check if table exists
+- insert_batch(): Insert batch of data
+- get_last_timestamp(): Get last timestamp for a metric
+"""
+
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+
+from detectkit.core.models import TableModel
+
+
+class BaseDatabaseManager(ABC):
+    """
+    Universal database manager interface.
+
+    This class provides GENERIC methods for database operations.
+    It does NOT hardcode logic for internal tables (_dtk_datapoints, etc.).
+
+    Internal table management is handled by higher-level classes that
+    use these generic methods.
+
+    Key Design Principles:
+    1. Universal methods (not table-specific)
+    2. Works with any table via table_name parameter
+    3. Type conversion handled internally
+    4. Connection pooling and error handling
+    """
+
+    @abstractmethod
+    def execute_query(
+        self,
+        query: str,
+        params: Optional[Dict[str, Any]] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Execute SQL query and return results as list of dictionaries.
+
+        Args:
+            query: SQL query to execute
+            params: Optional query parameters for parameterized queries
+
+        Returns:
+            List of dictionaries where each dict represents a row
+
+        Raises:
+            DatabaseError: If query execution fails
+
+        Example:
+            >>> results = manager.execute_query(
+            ...     "SELECT * FROM metrics WHERE name = %(name)s",
+            ...     {"name": "cpu_usage"}
+            ... )
+            >>> for row in results:
+            ...     print(row['timestamp'], row['value'])
+        """
+        pass
+
+    @abstractmethod
+    def create_table(
+        self,
+        table_name: str,
+        table_model: TableModel,
+        if_not_exists: bool = True
+    ) -> None:
+        """
+        Create table from TableModel definition.
+
+        Converts database-agnostic TableModel into database-specific DDL.
+
+        Args:
+            table_name: Name of table to create
+            table_model: Table schema definition
+            if_not_exists: Add IF NOT EXISTS clause
+
+        Raises:
+            DatabaseError: If table creation fails
+
+        Example:
+            >>> model = TableModel(
+            ...     columns=[
+            ...         ColumnDefinition("id", "Int32"),
+            ...         ColumnDefinition("value", "Float64", nullable=True),
+            ...     ],
+            ...     primary_key=["id"],
+            ...     engine="MergeTree",
+            ...     order_by=["id"]
+            ... )
+            >>> manager.create_table("my_metrics", model)
+        """
+        pass
+
+    @abstractmethod
+    def table_exists(
+        self,
+        table_name: str,
+        schema: Optional[str] = None
+    ) -> bool:
+        """
+        Check if table exists in database.
+
+        Args:
+            table_name: Name of table to check
+            schema: Optional schema/database name (if None, use default)
+
+        Returns:
+            True if table exists, False otherwise
+
+        Example:
+            >>> if not manager.table_exists("_dtk_datapoints"):
+            ...     manager.create_table("_dtk_datapoints", datapoints_model)
+        """
+        pass
+
+    @abstractmethod
+    def insert_batch(
+        self,
+        table_name: str,
+        data: Dict[str, np.ndarray],
+        conflict_strategy: str = "ignore"
+    ) -> int:
+        """
+        Insert batch of data into table.
+
+        Universal method that works with any table - NOT specific to
+        internal tables.
+
+        Args:
+            table_name: Name of table to insert into
+            data: Dictionary mapping column names to numpy arrays
+                 All arrays must have same length
+            conflict_strategy: How to handle conflicts:
+                - "ignore": Skip rows with duplicate primary keys
+                - "replace": Replace existing rows
+                - "fail": Raise error on conflict
+
+        Returns:
+            Number of rows inserted (may be less than input if conflicts ignored)
+
+        Raises:
+            ValueError: If arrays have different lengths
+            DatabaseError: If insertion fails
+
+        Example:
+            >>> data = {
+            ...     "metric_name": np.array(["cpu", "cpu"]),
+            ...     "timestamp": np.array([dt1, dt2]),
+            ...     "value": np.array([0.5, 0.6]),
+            ... }
+            >>> rows_inserted = manager.insert_batch(
+            ...     "_dtk_datapoints", data, conflict_strategy="ignore"
+            ... )
+        """
+        pass
+
+    @abstractmethod
+    def get_last_timestamp(
+        self,
+        table_name: str,
+        metric_name: str,
+        timestamp_column: str = "timestamp"
+    ) -> Optional[datetime]:
+        """
+        Get last timestamp for a specific metric in a table.
+
+        Universal method that works with any table containing metric_name
+        and timestamp columns.
+
+        Args:
+            table_name: Table to query
+            metric_name: Value to filter by metric_name column
+            timestamp_column: Name of timestamp column (default: "timestamp")
+
+        Returns:
+            Last timestamp or None if no data found
+
+        Example:
+            >>> last_ts = manager.get_last_timestamp(
+            ...     "_dtk_datapoints", "cpu_usage"
+            ... )
+            >>> if last_ts:
+            ...     print(f"Last data point at {last_ts}")
+        """
+        pass
+
+    @abstractmethod
+    def upsert_task_status(
+        self,
+        metric_name: str,
+        detector_id: str,
+        process_type: str,
+        status: str,
+        last_processed_timestamp: Optional[datetime] = None,
+        error_message: Optional[str] = None,
+        timeout_seconds: int = 3600
+    ) -> None:
+        """
+        Update or insert task status (for locking and idempotency).
+
+        This method is critical for:
+        1. Task locking: Prevent concurrent runs of same task
+        2. Idempotency: Store last_processed_timestamp to resume from interruptions
+
+        Implementation varies by database:
+        - ClickHouse: DELETE + INSERT (no native UPSERT)
+        - PostgreSQL: INSERT ... ON CONFLICT DO UPDATE
+        - MySQL: INSERT ... ON DUPLICATE KEY UPDATE
+
+        Args:
+            metric_name: Metric identifier
+            detector_id: Detector identifier (or "load" for loading tasks)
+            process_type: Type of process ("load" or "detect")
+            status: Task status ("running", "completed", "failed")
+            last_processed_timestamp: Last successfully processed timestamp
+            error_message: Error message if status is "failed"
+            timeout_seconds: Task timeout in seconds
+
+        Example:
+            >>> # Start task
+            >>> manager.upsert_task_status(
+            ...     "cpu_usage", "load", "load", "running",
+            ...     timeout_seconds=3600
+            ... )
+            >>> # Update progress
+            >>> manager.upsert_task_status(
+            ...     "cpu_usage", "load", "load", "running",
+            ...     last_processed_timestamp=datetime(2024, 1, 1, 12, 0)
+            ... )
+            >>> # Complete task
+            >>> manager.upsert_task_status(
+            ...     "cpu_usage", "load", "load", "completed",
+            ...     last_processed_timestamp=datetime(2024, 1, 1, 23, 59)
+            ... )
+        """
+        pass
+
+    @property
+    @abstractmethod
+    def internal_location(self) -> str:
+        """
+        Get full location path for internal tables.
+
+        Format depends on database:
+        - ClickHouse: "database_name"
+        - PostgreSQL: "schema_name"
+
+        Returns:
+            Full path to internal schema/database
+
+        Example:
+            >>> manager.internal_location
+            'detectk_internal'
+        """
+        pass
+
+    @property
+    @abstractmethod
+    def data_location(self) -> str:
+        """
+        Get full location path for user data tables.
+
+        Format depends on database:
+        - ClickHouse: "database_name"
+        - PostgreSQL: "schema_name"
+
+        Returns:
+            Full path to data schema/database
+
+        Example:
+            >>> manager.data_location
+            'analytics'
+        """
+        pass
+
+    def get_full_table_name(
+        self,
+        table_name: str,
+        use_internal: bool = True
+    ) -> str:
+        """
+        Get fully qualified table name.
+
+        Args:
+            table_name: Table name
+            use_internal: If True, use internal_location, else data_location
+
+        Returns:
+            Fully qualified table name
+
+        Example:
+            >>> manager.get_full_table_name("_dtk_datapoints", use_internal=True)
+            'detectk_internal._dtk_datapoints'
+        """
+        location = self.internal_location if use_internal else self.data_location
+        return f"{location}.{table_name}"
+
+    @abstractmethod
+    def close(self) -> None:
+        """
+        Close database connection and cleanup resources.
+
+        Example:
+            >>> manager.close()
+        """
+        pass
+
+    def __enter__(self):
+        """Context manager entry."""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit - close connection."""
+        self.close()

detectkit/database/tables.py

@@ -0,0 +1,134 @@
+"""
+Internal table models for detectk.
+
+Defines schemas for internal tables:
+- _dtk_datapoints: Metric data points
+- _dtk_detections: Anomaly detections
+- _dtk_tasks: Task status and locking
+"""
+
+from detectkit.core.models import ColumnDefinition, TableModel
+
+
+def get_datapoints_table_model() -> TableModel:
+    """
+    Get TableModel for _dtk_datapoints table.
+
+    Schema:
+        - metric_name: Metric identifier
+        - timestamp: Data point timestamp (UTC, millisecond precision)
+        - value: Metric value (nullable for missing data)
+        - seasonality_data: JSON with seasonality components (hour, day_of_week, etc.)
+        - interval_seconds: Interval in seconds
+        - seasonality_columns: Comma-separated list of seasonality columns used
+        - created_at: When record was created (UTC, millisecond precision)
+
+    Primary Key: (metric_name, timestamp)
+    """
+    return TableModel(
+        columns=[
+            ColumnDefinition("metric_name", "String"),
+            ColumnDefinition("timestamp", "DateTime64(3, 'UTC')"),
+            ColumnDefinition("value", "Nullable(Float64)", nullable=True),
+            ColumnDefinition("seasonality_data", "String"),
+            ColumnDefinition("interval_seconds", "Int32"),
+            ColumnDefinition("seasonality_columns", "String"),
+            ColumnDefinition("created_at", "DateTime64(3, 'UTC')"),
+        ],
+        primary_key=["metric_name", "timestamp"],
+        engine="ReplacingMergeTree(created_at)",
+        order_by=["metric_name", "timestamp"],
+    )
+
+
+def get_detections_table_model() -> TableModel:
+    """
+    Get TableModel for _dtk_detections table.
+
+    Schema:
+        - metric_name: Metric identifier
+        - detector_id: Detector identifier (hash of class + params)
+        - timestamp: Detection timestamp (UTC, millisecond precision)
+        - is_anomaly: Whether point is anomalous
+        - confidence_lower: Lower confidence bound
+        - confidence_upper: Upper confidence bound
+        - value: Actual metric value
+        - detector_params: JSON with sorted detector parameters
+        - detection_metadata: JSON with missing_ratio, severity, direction, etc.
+        - created_at: When detection was performed (UTC, millisecond precision)
+
+    Primary Key: (metric_name, detector_id, timestamp)
+    """
+    return TableModel(
+        columns=[
+            ColumnDefinition("metric_name", "String"),
+            ColumnDefinition("detector_id", "String"),
+            ColumnDefinition("timestamp", "DateTime64(3, 'UTC')"),
+            ColumnDefinition("is_anomaly", "Bool"),
+            ColumnDefinition("confidence_lower", "Nullable(Float64)", nullable=True),
+            ColumnDefinition("confidence_upper", "Nullable(Float64)", nullable=True),
+            ColumnDefinition("value", "Nullable(Float64)", nullable=True),
+            ColumnDefinition("detector_params", "String"),
+            ColumnDefinition("detection_metadata", "String"),
+            ColumnDefinition("created_at", "DateTime64(3, 'UTC')"),
+        ],
+        primary_key=["metric_name", "detector_id", "timestamp"],
+        engine="ReplacingMergeTree(created_at)",
+        order_by=["metric_name", "detector_id", "timestamp"],
+    )
+
+
+def get_tasks_table_model() -> TableModel:
+    """
+    Get TableModel for _dtk_tasks table.
+
+    Schema:
+        - metric_name: Metric identifier
+        - detector_id: Detector identifier (or "load" for loading tasks)
+        - process_type: Type of process ("load" or "detect")
+        - status: Task status ("running", "completed", "failed")
+        - started_at: When task started (UTC, millisecond precision)
+        - updated_at: Last update timestamp (UTC, millisecond precision)
+        - last_processed_timestamp: Last successfully processed timestamp
+        - error_message: Error message if failed (nullable)
+        - timeout_seconds: Task timeout in seconds
+
+    Primary Key: (metric_name, detector_id, process_type)
+
+    This table serves dual purpose:
+    1. Locking: Only one process can run for a given (metric, detector, type)
+    2. Resume: Stores last_processed_timestamp to resume from interruptions
+    """
+    return TableModel(
+        columns=[
+            ColumnDefinition("metric_name", "String"),
+            ColumnDefinition("detector_id", "String"),
+            ColumnDefinition("process_type", "String"),
+            ColumnDefinition("status", "String"),
+            ColumnDefinition("started_at", "DateTime64(3, 'UTC')"),
+            ColumnDefinition("updated_at", "DateTime64(3, 'UTC')"),
+            ColumnDefinition(
+                "last_processed_timestamp",
+                "Nullable(DateTime64(3, 'UTC'))",
+                nullable=True
+            ),
+            ColumnDefinition("error_message", "Nullable(String)", nullable=True),
+            ColumnDefinition("timeout_seconds", "Int32"),
+        ],
+        primary_key=["metric_name", "detector_id", "process_type"],
+        engine="MergeTree",
+        order_by=["metric_name", "detector_id", "process_type"],
+    )
+
+
+# Table names as constants
+TABLE_DATAPOINTS = "_dtk_datapoints"
+TABLE_DETECTIONS = "_dtk_detections"
+TABLE_TASKS = "_dtk_tasks"
+
+# Map of table names to model factories
+INTERNAL_TABLES = {
+    TABLE_DATAPOINTS: get_datapoints_table_model,
+    TABLE_DETECTIONS: get_detections_table_model,
+    TABLE_TASKS: get_tasks_table_model,
+}

detectkit/detectors/__init__.py

@@ -0,0 +1,6 @@
+"""Anomaly detectors for detectkit."""
+
+from detectkit.detectors.base import BaseDetector, DetectionResult
+from detectkit.detectors.factory import DetectorFactory
+
+__all__ = ["BaseDetector", "DetectionResult", "DetectorFactory"]

detectkit/detectors/base.py

@@ -0,0 +1,222 @@
+"""
+Base detector interface for anomaly detection.
+
+All detectors must inherit from BaseDetector and implement:
+- _validate_params() - parameter validation
+- detect() - main detection method
+- _get_non_default_params() - for hash generation
+"""
+
+import hashlib
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+import numpy as np
+
+try:
+    import orjson
+    HAS_ORJSON = True
+except ImportError:
+    import json
+    HAS_ORJSON = False
+
+
+def json_dumps_sorted(obj):
+    """JSON dumps with sorted keys - handles both orjson and standard json."""
+    if HAS_ORJSON:
+        return orjson.dumps(obj, option=orjson.OPT_SORT_KEYS).decode('utf-8')
+    else:
+        return json.dumps(obj, sort_keys=True)
+
+
+@dataclass
+class DetectionResult:
+    """
+    Result of anomaly detection for a single data point.
+
+    Attributes:
+        timestamp: Data point timestamp
+        value: Actual metric value
+        is_anomaly: Whether point is anomalous
+        confidence_lower: Lower bound of confidence interval (if available)
+        confidence_upper: Upper bound of confidence interval (if available)
+        detection_metadata: Additional metadata (severity, direction, etc.)
+    """
+
+    timestamp: np.datetime64
+    value: float
+    is_anomaly: bool
+    confidence_lower: Optional[float] = None
+    confidence_upper: Optional[float] = None
+    detection_metadata: Optional[Dict[str, Any]] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for database storage."""
+        return {
+            "timestamp": self.timestamp,
+            "value": self.value,
+            "is_anomaly": self.is_anomaly,
+            "confidence_lower": self.confidence_lower,
+            "confidence_upper": self.confidence_upper,
+            "detection_metadata": json_dumps_sorted(self.detection_metadata or {}),
+        }
+
+
+class BaseDetector(ABC):
+    """
+    Abstract base class for anomaly detectors.
+
+    All detectors must:
+    1. Validate parameters in _validate_params()
+    2. Implement detect() to return DetectionResult for each point
+    3. Implement _get_non_default_params() for hash generation
+
+    The detector_id (hash) is used for:
+    - Storing detections in _dtk_detections table
+    - Task locking in _dtk_tasks table
+
+    Example:
+        >>> class MyDetector(BaseDetector):
+        ...     def __init__(self, threshold: float = 3.0):
+        ...         super().__init__(threshold=threshold)
+        ...
+        ...     def _validate_params(self):
+        ...         if self.params["threshold"] <= 0:
+        ...             raise ValueError("threshold must be positive")
+        ...
+        ...     def detect(self, data):
+        ...         # Detection logic here
+        ...         pass
+        ...
+        ...     def _get_non_default_params(self):
+        ...         defaults = {"threshold": 3.0}
+        ...         return {k: v for k, v in self.params.items() if v != defaults.get(k)}
+    """
+
+    def __init__(self, **params):
+        """
+        Initialize detector with parameters.
+
+        Args:
+            **params: Detector-specific parameters
+        """
+        self.params = params
+        self._validate_params()
+
+    @abstractmethod
+    def _validate_params(self):
+        """
+        Validate detector parameters.
+
+        Should raise ValueError if parameters are invalid.
+
+        Example:
+            >>> def _validate_params(self):
+            ...     if self.params.get("threshold", 0) <= 0:
+            ...         raise ValueError("threshold must be positive")
+        """
+        pass
+
+    @abstractmethod
+    def detect(self, data: Dict[str, np.ndarray]) -> list[DetectionResult]:
+        """
+        Perform anomaly detection on metric data.
+
+        Args:
+            data: Dictionary from MetricLoader.load() with keys:
+                - timestamp: np.array of datetime64[ms]
+                - value: np.array of float64 (may contain NaN for missing data)
+                - seasonality_data: np.array of JSON strings
+                - seasonality_columns: list of column names
+
+        Returns:
+            List of DetectionResult for each data point
+
+        Notes:
+            - Handle NaN values appropriately (missing data)
+            - Use seasonality_data if detector supports it
+            - confidence_lower/upper are optional (only if detector provides them)
+            - detection_metadata can include: severity, direction, missing_ratio, etc.
+
+        Example:
+            >>> results = detector.detect(data)
+            >>> for result in results:
+            ...     if result.is_anomaly:
+            ...         print(f"Anomaly at {result.timestamp}: {result.value}")
+        """
+        pass
+
+    def get_detector_id(self) -> str:
+        """
+        Generate unique detector ID (hash).
+
+        Hash is based on:
+        - Detector class name
+        - Non-default parameters (sorted)
+
+        This ensures:
+        - Same detector with same params = same ID
+        - Different params = different ID (allows parallel runs)
+
+        Returns:
+            16-character hex string (first 16 chars of SHA256)
+
+        Example:
+            >>> detector1 = MADDetector(threshold=3.0)
+            >>> detector2 = MADDetector(threshold=3.0)
+            >>> detector1.get_detector_id() == detector2.get_detector_id()
+            True
+            >>> detector3 = MADDetector(threshold=2.5)
+            >>> detector1.get_detector_id() != detector3.get_detector_id()
+            True
+        """
+        non_default_params = self._get_non_default_params()
+        sorted_params = sorted(non_default_params.items())
+        hash_string = self.__class__.__name__ + str(sorted_params)
+        return hashlib.sha256(hash_string.encode()).hexdigest()[:16]
+
+    def get_detector_params(self) -> str:
+        """
+        Get detector parameters as JSON string.
+
+        Returns JSON with sorted keys for consistency.
+        Used for storing in _dtk_detections.detector_params.
+
+        Returns:
+            JSON string with sorted parameters
+
+        Example:
+            >>> detector = MADDetector(threshold=3.0, min_samples=30)
+            >>> detector.get_detector_params()
+            '{"min_samples": 30, "threshold": 3.0}'
+        """
+        non_default_params = self._get_non_default_params()
+        return json_dumps_sorted(non_default_params)
+
+    @abstractmethod
+    def _get_non_default_params(self) -> Dict[str, Any]:
+        """
+        Get parameters that differ from defaults.
+
+        Used for hash generation and parameter storage.
+        Only non-default parameters are included to ensure
+        consistent hashing across different instantiations.
+
+        Returns:
+            Dictionary of non-default parameters
+
+        Example:
+            >>> def _get_non_default_params(self):
+            ...     defaults = {"threshold": 3.0, "min_samples": 30}
+            ...     return {
+            ...         k: v for k, v in self.params.items()
+            ...         if v != defaults.get(k)
+            ...     }
+        """
+        pass
+
+    def __repr__(self) -> str:
+        """String representation of detector."""
+        params_str = ", ".join(f"{k}={v}" for k, v in self.params.items())
+        return f"{self.__class__.__name__}({params_str})"