detectkit 0.2.4__tar.gz → 0.2.6__tar.gz

{detectkit-0.2.4/detectkit.egg-info → detectkit-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detectkit
-Version: 0.2.4
+Version: 0.2.6
 Summary: Metric monitoring with automatic anomaly detection
 Author: detectkit team
 License: MIT

{detectkit-0.2.4 → detectkit-0.2.6}/detectkit/cli/commands/run.py

@@ -12,6 +12,7 @@ import click
 
 from detectkit.config.metric_config import MetricConfig
 from detectkit.config.profile import ProfilesConfig
+from detectkit.config.project_config import ProjectConfig
 from detectkit.config.validator import validate_metric_uniqueness
 from detectkit.database.internal_tables import InternalTablesManager
 from detectkit.orchestration.task_manager import PipelineStep, TaskManager
@@ -63,7 +64,18 @@ def run_command(
     click.echo(f"Project root: {project_root}")
 
     # Load project config
-    # project_config = load_project_config(project_root)
+    project_config_path = project_root / "detectkit_project.yml"
+    try:
+        project_config = ProjectConfig.from_yaml_file(project_config_path)
+    except Exception as e:
+        click.echo(
+            click.style(
+                f"Error loading detectkit_project.yml: {e}",
+                fg="red",
+                bold=True,
+            )
+        )
+        return
 
     # Select metrics based on selector
     # Returns list of (path, config) tuples with uniqueness validation
@@ -169,6 +181,7 @@ def run_command(
         internal_manager=internal_manager,
         db_manager=db_manager,
         profiles_config=profiles_config,
+        project_config=project_config,
     )
 
     # Process each metric
@@ -457,6 +470,7 @@ def process_metric(
             to_date=to_date,
             full_refresh=full_refresh,
             force=force,
+            metric_file_path=str(metric_path),
         )
 
         # Display results - task_manager already printed details

{detectkit-0.2.4 → detectkit-0.2.6}/detectkit/config/project_config.py

@@ -33,6 +33,7 @@ class ProjectTablesConfig(BaseModel):
         datapoints: Default datapoints table name
         detections: Default detections table name
         tasks: Default tasks table name
+        metrics: Default metrics configuration table name
     """
 
     datapoints: str = Field(
@@ -42,6 +43,7 @@ class ProjectTablesConfig(BaseModel):
         default="_dtk_detections", description="Default detections table"
     )
     tasks: str = Field(default="_dtk_tasks", description="Default tasks table")
+    metrics: str = Field(default="_dtk_metrics", description="Default metrics config table")
 
 
 class ProjectTimeoutsConfig(BaseModel):
@@ -95,6 +97,7 @@ class ProjectConfig(BaseModel):
           datapoints: "_dtk_datapoints"
           detections: "_dtk_detections"
           tasks: "_dtk_tasks"
+          metrics: "_dtk_metrics"
 
         timeouts:
           load: 3600

{detectkit-0.2.4 → detectkit-0.2.6}/detectkit/database/clickhouse_manager.py

@@ -300,7 +300,20 @@ class ClickHouseDatabaseManager(BaseDatabaseManager):
         result = self.execute_query(query, {"metric_name": metric_name})
 
         if result and result[0]["last_ts"]:
-            return result[0]["last_ts"]
+            last_ts = result[0]["last_ts"]
+
+            # ClickHouse returns epoch (1970-01-01 00:00:00) for NULL datetime
+            # Detect this and treat as None to avoid loading from 1970
+            epoch = datetime(1970, 1, 1, 0, 0, 0)
+
+            # Handle both timezone-aware and naive datetimes
+            if last_ts.tzinfo is not None:
+                epoch = epoch.replace(tzinfo=last_ts.tzinfo)
+
+            if last_ts == epoch:
+                return None
+
+            return last_ts
 
         return None
 
@@ -377,6 +390,42 @@ class ClickHouseDatabaseManager(BaseDatabaseManager):
             conflict_strategy="ignore"
         )
 
+    def upsert_record(
+        self,
+        table_name: str,
+        key_columns: Dict[str, Any],
+        data: Dict[str, np.ndarray]
+    ) -> int:
+        """
+        Upsert record in ClickHouse using DELETE + INSERT pattern.
+
+        ClickHouse doesn't have native UPSERT, so we explicitly delete
+        the old record (if exists) and then insert the new one.
+
+        Args:
+            table_name: Fully qualified table name
+            key_columns: Dict of column names to values for WHERE clause
+            data: Dict of column names to numpy arrays for INSERT
+
+        Returns:
+            Number of rows inserted (typically 1)
+        """
+        # Step 1: DELETE existing record (if any)
+        where_parts = [f"{col} = %({col})s" for col in key_columns.keys()]
+        delete_query = f"""
+        ALTER TABLE {table_name}
+        DELETE WHERE {' AND '.join(where_parts)}
+        """
+
+        self._client.execute(delete_query, key_columns)
+
+        # Step 2: INSERT new record
+        return self.insert_batch(
+            table_name,
+            data,
+            conflict_strategy="ignore"
+        )
+
     @property
     def internal_location(self) -> str:
         """Get internal database name."""

{detectkit-0.2.4 → detectkit-0.2.6}/detectkit/database/internal_tables.py

@@ -2,12 +2,13 @@
 Internal tables manager for detectk.
 
 High-level wrapper over BaseDatabaseManager for working with internal tables
-(_dtk_datapoints, _dtk_detections, _dtk_tasks).
+(_dtk_datapoints, _dtk_detections, _dtk_tasks, _dtk_metrics).
 
 This class provides convenient methods that use the UNIVERSAL BaseDatabaseManager
 methods underneath. It does NOT duplicate logic - just provides semantic wrappers.
 """
 
+import json
 from datetime import datetime, timezone
 from typing import Dict, List, Optional
 
@@ -18,6 +19,7 @@ from detectkit.database.tables import (
     INTERNAL_TABLES,
     TABLE_DATAPOINTS,
     TABLE_DETECTIONS,
+    TABLE_METRICS,
     TABLE_TASKS,
 )
 
@@ -722,3 +724,103 @@ class InternalTablesManager:
             status="running",
             last_processed_timestamp=last_processed_timestamp,
         )
+
+    def upsert_metric_config(
+        self,
+        metric_config,  # MetricConfig type (avoiding circular import)
+        file_path: str,
+        table_name_override: Optional[str] = None
+    ) -> int:
+        """
+        Save or update metric configuration metadata to _dtk_metrics table.
+
+        This table is INFORMATIONAL ONLY - used by analysts for dashboards.
+        It does NOT affect library logic.
+
+        Updated on every dtk run via DELETE + INSERT pattern for guaranteed uniqueness.
+
+        Args:
+            metric_config: MetricConfig instance
+            file_path: Path to .yml config file
+            table_name_override: Optional override for table name (from ProjectConfig)
+
+        Returns:
+            Number of rows inserted (typically 1)
+
+        Example:
+            >>> internal.upsert_metric_config(
+            ...     metric_config=config,
+            ...     file_path="metrics/cpu_usage.yml",
+            ...     table_name_override="_dtk_metrics"
+            ... )
+        """
+        # Get table name (use override if provided, else default)
+        table_name = table_name_override or TABLE_METRICS
+        full_table_name = self._manager.get_full_table_name(
+            table_name, use_internal=True
+        )
+
+        # Get current UTC time (naive for numpy compatibility)
+        now = datetime.now(timezone.utc).replace(tzinfo=None)
+
+        # Parse loading_start_time if provided
+        loading_start_time_dt = None
+        if metric_config.loading_start_time:
+            try:
+                from datetime import datetime as dt
+                loading_start_time_dt = dt.strptime(
+                    metric_config.loading_start_time,
+                    "%Y-%m-%d %H:%M:%S"
+                ).replace(tzinfo=None)
+            except (ValueError, AttributeError):
+                # If parsing fails, leave as None
+                pass
+
+        # Extract alert configuration
+        is_alert_enabled = 0
+        timezone_str = None
+        direction = None
+        consecutive_anomalies = 3
+        no_data_alert = 0
+        min_detectors = 1
+
+        if metric_config.alerting:
+            is_alert_enabled = 1 if metric_config.alerting.enabled else 0
+            timezone_str = metric_config.alerting.timezone
+            direction = metric_config.alerting.direction
+            consecutive_anomalies = metric_config.alerting.consecutive_anomalies
+            no_data_alert = 1 if metric_config.alerting.no_data_alert else 0
+            min_detectors = metric_config.alerting.min_detectors
+
+        # Prepare data for INSERT
+        data = {
+            "metric_name": np.array([metric_config.name]),
+            "description": np.array([getattr(metric_config, 'description', None)]),
+            "path": np.array([file_path]),
+            "interval": np.array([str(metric_config.interval)]),
+            "loading_start_time": np.array(
+                [loading_start_time_dt], dtype="datetime64[ms]"
+            ) if loading_start_time_dt else np.array([None]),
+            "loading_batch_size": np.array(
+                [metric_config.loading_batch_size], dtype=np.uint32
+            ),
+            "is_alert_enabled": np.array([is_alert_enabled], dtype=np.uint8),
+            "timezone": np.array([timezone_str]),
+            "direction": np.array([direction]),
+            "consecutive_anomalies": np.array(
+                [consecutive_anomalies], dtype=np.uint32
+            ),
+            "no_data_alert": np.array([no_data_alert], dtype=np.uint8),
+            "min_detectors": np.array([min_detectors], dtype=np.uint32),
+            "tags": np.array([json.dumps(metric_config.tags or [])]),
+            "enabled": np.array([1 if metric_config.enabled else 0], dtype=np.uint8),
+            "created_at": np.array([now], dtype="datetime64[ms]"),
+            "updated_at": np.array([now], dtype="datetime64[ms]"),
+        }
+
+        # Use upsert_record for DELETE + INSERT pattern
+        return self._manager.upsert_record(
+            table_name=full_table_name,
+            key_columns={"metric_name": metric_config.name},
+            data=data
+        )

{detectkit-0.2.4 → detectkit-0.2.6}/detectkit/database/manager.py

@@ -245,6 +245,54 @@ class BaseDatabaseManager(ABC):
         """
         pass
 
+    @abstractmethod
+    def upsert_record(
+        self,
+        table_name: str,
+        key_columns: Dict[str, Any],
+        data: Dict[str, np.ndarray]
+    ) -> int:
+        """
+        Delete record by key columns, then insert new record.
+
+        This is a universal database-agnostic upsert pattern that guarantees
+        uniqueness by explicitly deleting old record before inserting new one.
+
+        Use this when ReplacingMergeTree or native UPSERT is not suitable
+        (e.g., for informational tables where guaranteed uniqueness is required).
+
+        Implementation varies by database:
+        - ClickHouse: ALTER TABLE ... DELETE + INSERT
+        - PostgreSQL: DELETE + INSERT (in transaction)
+        - MySQL: DELETE + INSERT (in transaction)
+
+        Args:
+            table_name: Fully qualified table name
+            key_columns: Dict of column names to values for WHERE clause
+                        (e.g., {"metric_name": "cpu_usage"})
+            data: Dict of column names to numpy arrays for INSERT
+                  (must include all key columns)
+
+        Returns:
+            Number of rows inserted (typically 1)
+
+        Raises:
+            DatabaseError: If operation fails
+
+        Example:
+            >>> manager.upsert_record(
+            ...     table_name="detectk_internal._dtk_metrics",
+            ...     key_columns={"metric_name": "cpu_usage"},
+            ...     data={
+            ...         "metric_name": np.array(["cpu_usage"]),
+            ...         "interval": np.array(["10min"]),
+            ...         "enabled": np.array([1]),
+            ...         ...
+            ...     }
+            ... )
+        """
+        pass
+
     @property
     @abstractmethod
     def internal_location(self) -> str:

{detectkit-0.2.4 → detectkit-0.2.6}/detectkit/database/tables.py

@@ -5,6 +5,7 @@ Defines schemas for internal tables:
 - _dtk_datapoints: Metric data points
 - _dtk_detections: Anomaly detections
 - _dtk_tasks: Task status and locking
+- _dtk_metrics: Metric configuration metadata (informational)
 """
 
 from detectkit.core.models import ColumnDefinition, TableModel
@@ -125,14 +126,74 @@ def get_tasks_table_model() -> TableModel:
     )
 
 
+def get_metrics_table_model() -> TableModel:
+    """
+    Get TableModel for _dtk_metrics table.
+
+    This table stores metric configuration metadata for analytics dashboards.
+    It is INFORMATIONAL ONLY - does not affect library logic.
+    Updated on every dtk run via DELETE + INSERT pattern.
+
+    Schema:
+        - metric_name: Metric identifier (PRIMARY KEY)
+        - description: Optional metric description
+        - path: Path to .yml config file
+        - interval: Interval as string ("10min", "1h", etc.)
+        - loading_start_time: Start time for initial data loading
+        - loading_batch_size: Batch size for loading operations
+        - is_alert_enabled: Whether alerting is enabled (0/1)
+        - timezone: Timezone for alerts (e.g., "Europe/Moscow")
+        - direction: Required anomaly direction ("same", "any", "up", "down")
+        - consecutive_anomalies: Consecutive anomalies to trigger alert
+        - no_data_alert: Whether to alert on missing data (0/1)
+        - min_detectors: Minimum detectors that must agree
+        - tags: JSON array of tags
+        - enabled: Whether metric is enabled for processing (0/1)
+        - created_at: First time config was saved (UTC, millisecond precision)
+        - updated_at: Last config update (UTC, millisecond precision)
+
+    Primary Key: (metric_name)
+    Engine: MergeTree (uses DELETE + INSERT for guaranteed uniqueness)
+    """
+    return TableModel(
+        columns=[
+            ColumnDefinition("metric_name", "String"),
+            ColumnDefinition("description", "Nullable(String)", nullable=True),
+            ColumnDefinition("path", "String"),
+            ColumnDefinition("interval", "String"),
+            ColumnDefinition(
+                "loading_start_time",
+                "Nullable(DateTime64(3, 'UTC'))",
+                nullable=True
+            ),
+            ColumnDefinition("loading_batch_size", "UInt32"),
+            ColumnDefinition("is_alert_enabled", "UInt8"),
+            ColumnDefinition("timezone", "Nullable(String)", nullable=True),
+            ColumnDefinition("direction", "Nullable(String)", nullable=True),
+            ColumnDefinition("consecutive_anomalies", "UInt32"),
+            ColumnDefinition("no_data_alert", "UInt8"),
+            ColumnDefinition("min_detectors", "UInt32"),
+            ColumnDefinition("tags", "String"),
+            ColumnDefinition("enabled", "UInt8"),
+            ColumnDefinition("created_at", "DateTime64(3, 'UTC')"),
+            ColumnDefinition("updated_at", "DateTime64(3, 'UTC')"),
+        ],
+        primary_key=["metric_name"],
+        engine="MergeTree",
+        order_by=["metric_name"],
+    )
+
+
 # Table names as constants
 TABLE_DATAPOINTS = "_dtk_datapoints"
 TABLE_DETECTIONS = "_dtk_detections"
 TABLE_TASKS = "_dtk_tasks"
+TABLE_METRICS = "_dtk_metrics"
 
 # Map of table names to model factories
 INTERNAL_TABLES = {
     TABLE_DATAPOINTS: get_datapoints_table_model,
     TABLE_DETECTIONS: get_detections_table_model,
     TABLE_TASKS: get_tasks_table_model,
+    TABLE_METRICS: get_metrics_table_model,
 }

{detectkit-0.2.4 → detectkit-0.2.6}/detectkit/orchestration/task_manager.py

@@ -73,6 +73,7 @@ class TaskManager:
         internal_manager: InternalTablesManager,
         db_manager,  # BaseDatabaseManager
         profiles_config=None,  # ProfilesConfig (optional for backward compatibility)
+        project_config=None,  # ProjectConfig (for table name overrides)
     ):
         """
         Initialize task manager.
@@ -81,10 +82,12 @@ class TaskManager:
             internal_manager: Manager for internal detectk tables
             db_manager: Database manager for metric data
             profiles_config: Profiles configuration (for alert channels)
+            project_config: Project configuration (for table name overrides)
         """
         self.internal = internal_manager
         self.db_manager = db_manager
         self.profiles_config = profiles_config
+        self.project_config = project_config
 
     def run_metric(
         self,
@@ -94,6 +97,7 @@ class TaskManager:
         to_date: Optional[datetime] = None,
         full_refresh: bool = False,
         force: bool = False,
+        metric_file_path: Optional[str] = None,
     ) -> Dict[str, any]:
         """
         Run metric processing pipeline.
@@ -105,6 +109,7 @@ class TaskManager:
             to_date: End date for data loading (optional)
             full_refresh: Delete all existing data and reload from scratch
             force: Ignore task locks
+            metric_file_path: Path to metric .yml file (for _dtk_metrics table)
 
         Returns:
             Dict with execution results:
@@ -139,7 +144,19 @@ class TaskManager:
         }
 
         try:
-            # Step 0: Acquire lock
+            # Step 0a: Save metric configuration to _dtk_metrics (informational)
+            if metric_file_path:
+                metrics_table_name = None
+                if self.project_config and hasattr(self.project_config, 'tables'):
+                    metrics_table_name = self.project_config.tables.metrics
+
+                self.internal.upsert_metric_config(
+                    metric_config=config,
+                    file_path=metric_file_path,
+                    table_name_override=metrics_table_name
+                )
+
+            # Step 0b: Acquire lock
             if not force:
                 # Default timeout: 1 hour (can be overridden via ProjectConfig in future)
                 timeout_seconds = 3600

{detectkit-0.2.4 → detectkit-0.2.6/detectkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: detectkit
-Version: 0.2.4
+Version: 0.2.6
 Summary: Metric monitoring with automatic anomaly detection
 Author: detectkit team
 License: MIT

{detectkit-0.2.4 → detectkit-0.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "detectkit"
-version = "0.2.4"
+version = "0.2.6"
 description = "Metric monitoring with automatic anomaly detection"
 readme = "README.md"
 requires-python = ">=3.10"