detectkit 0.2.4__py3-none-any.whl

detectkit/config/profile.py

@@ -0,0 +1,285 @@
+"""
+Profile configuration for detectk.
+
+Manages database connections and locations (similar to dbt profiles).
+"""
+
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import yaml
+from pydantic import BaseModel, Field, field_validator
+
+from detectkit.database.clickhouse_manager import ClickHouseDatabaseManager
+from detectkit.database.manager import BaseDatabaseManager
+
+
+class ProfileConfig(BaseModel):
+    """
+    Single profile configuration.
+
+    Defines connection parameters and database locations for a specific
+    environment (dev, prod, etc.).
+
+    Attributes:
+        type: Database type ("clickhouse", "postgres", "mysql")
+        host: Database host
+        port: Database port
+        user: Database user
+        password: Database password
+        internal_database: Database/schema for internal tables
+        internal_schema: Schema for internal tables (PostgreSQL only)
+        data_database: Database for user data tables
+        data_schema: Schema for user data (PostgreSQL only)
+        settings: Additional database-specific settings
+    """
+
+    type: str = Field(..., description="Database type")
+    host: str = Field(default="localhost", description="Database host")
+    port: int = Field(..., description="Database port")
+    user: str = Field(default="default", description="Database user")
+    password: str = Field(default="", description="Database password")
+
+    # Internal location for _dtk_* tables
+    internal_database: Optional[str] = Field(
+        default=None,
+        description="Database for internal tables (ClickHouse/MySQL)"
+    )
+    internal_schema: Optional[str] = Field(
+        default=None,
+        description="Schema for internal tables (PostgreSQL)"
+    )
+
+    # Data location for user tables
+    data_database: Optional[str] = Field(
+        default=None,
+        description="Database for user data tables (ClickHouse/MySQL)"
+    )
+    data_schema: Optional[str] = Field(
+        default=None,
+        description="Schema for user data (PostgreSQL)"
+    )
+
+    settings: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional database settings"
+    )
+
+    @field_validator("type")
+    @classmethod
+    def validate_type(cls, v: str) -> str:
+        """Validate database type."""
+        allowed_types = {"clickhouse", "postgres", "mysql"}
+        if v not in allowed_types:
+            raise ValueError(
+                f"Invalid database type: {v}. "
+                f"Allowed types: {', '.join(allowed_types)}"
+            )
+        return v
+
+    @field_validator("port")
+    @classmethod
+    def validate_port(cls, v: int) -> int:
+        """Validate port number."""
+        if not (1 <= v <= 65535):
+            raise ValueError(f"Port must be between 1 and 65535, got {v}")
+        return v
+
+    def get_internal_location(self) -> str:
+        """
+        Get internal location (database or schema).
+
+        Returns:
+            Internal database/schema name
+
+        Raises:
+            ValueError: If location not configured
+        """
+        if self.type == "clickhouse":
+            if not self.internal_database:
+                raise ValueError("internal_database must be set for ClickHouse")
+            return self.internal_database
+        elif self.type == "postgres":
+            if not self.internal_schema:
+                raise ValueError("internal_schema must be set for PostgreSQL")
+            return self.internal_schema
+        elif self.type == "mysql":
+            if not self.internal_database:
+                raise ValueError("internal_database must be set for MySQL")
+            return self.internal_database
+        else:
+            raise ValueError(f"Unsupported database type: {self.type}")
+
+    def get_data_location(self) -> str:
+        """
+        Get data location (database or schema).
+
+        Returns:
+            Data database/schema name
+
+        Raises:
+            ValueError: If location not configured
+        """
+        if self.type == "clickhouse":
+            if not self.data_database:
+                raise ValueError("data_database must be set for ClickHouse")
+            return self.data_database
+        elif self.type == "postgres":
+            if not self.data_schema:
+                raise ValueError("data_schema must be set for PostgreSQL")
+            return self.data_schema
+        elif self.type == "mysql":
+            if not self.data_database:
+                raise ValueError("data_database must be set for MySQL")
+            return self.data_database
+        else:
+            raise ValueError(f"Unsupported database type: {self.type}")
+
+    def create_manager(self) -> BaseDatabaseManager:
+        """
+        Create database manager from profile configuration.
+
+        Returns:
+            Database manager instance
+
+        Raises:
+            NotImplementedError: If database type not yet implemented
+        """
+        if self.type == "clickhouse":
+            return ClickHouseDatabaseManager(
+                host=self.host,
+                port=self.port,
+                user=self.user,
+                password=self.password,
+                internal_database=self.get_internal_location(),
+                data_database=self.get_data_location(),
+                settings=self.settings,
+            )
+        elif self.type == "postgres":
+            raise NotImplementedError("PostgreSQL support coming soon")
+        elif self.type == "mysql":
+            raise NotImplementedError("MySQL support coming soon")
+        else:
+            raise ValueError(f"Unsupported database type: {self.type}")
+
+
+class ProfilesConfig(BaseModel):
+    """
+    Container for multiple profile configurations.
+
+    Loaded from profiles.yml file.
+
+    Attributes:
+        profiles: Dictionary mapping profile names to configurations
+        default_profile: Name of default profile to use
+        alert_channels: Dictionary mapping channel names to configurations
+    """
+
+    profiles: Dict[str, ProfileConfig]
+    default_profile: Optional[str] = None
+    alert_channels: Dict[str, Dict[str, Any]] = Field(
+        default_factory=dict, description="Alert channel configurations"
+    )
+
+    @field_validator("default_profile")
+    @classmethod
+    def validate_default_profile(cls, v: Optional[str], info) -> Optional[str]:
+        """Validate default profile exists."""
+        if v is not None:
+            profiles = info.data.get("profiles", {})
+            if v not in profiles:
+                raise ValueError(
+                    f"default_profile '{v}' not found in profiles. "
+                    f"Available profiles: {', '.join(profiles.keys())}"
+                )
+        return v
+
+    @classmethod
+    def from_yaml(cls, path: Path) -> "ProfilesConfig":
+        """
+        Load profiles from YAML file.
+
+        Args:
+            path: Path to profiles.yml
+
+        Returns:
+            ProfilesConfig instance
+
+        Raises:
+            FileNotFoundError: If file doesn't exist
+            ValueError: If YAML is invalid
+        """
+        if not path.exists():
+            raise FileNotFoundError(f"Profiles file not found: {path}")
+
+        with open(path, "r") as f:
+            data = yaml.safe_load(f)
+
+        if not data:
+            raise ValueError("Profiles file is empty")
+
+        return cls.model_validate(data)
+
+    def get_profile(self, name: Optional[str] = None) -> ProfileConfig:
+        """
+        Get profile configuration by name.
+
+        Args:
+            name: Profile name (if None, use default_profile)
+
+        Returns:
+            ProfileConfig instance
+
+        Raises:
+            ValueError: If profile not found or no default set
+        """
+        if name is None:
+            if self.default_profile is None:
+                raise ValueError(
+                    "No profile name specified and no default_profile set. "
+                    f"Available profiles: {', '.join(self.profiles.keys())}"
+                )
+            name = self.default_profile
+
+        if name not in self.profiles:
+            raise ValueError(
+                f"Profile '{name}' not found. "
+                f"Available profiles: {', '.join(self.profiles.keys())}"
+            )
+
+        return self.profiles[name]
+
+    def create_manager(self, profile_name: Optional[str] = None) -> BaseDatabaseManager:
+        """
+        Create database manager for a profile.
+
+        Args:
+            profile_name: Profile name (if None, use default)
+
+        Returns:
+            Database manager instance
+        """
+        profile = self.get_profile(profile_name)
+        return profile.create_manager()
+
+    def get_alert_channel_config(self, channel_name: str) -> Dict[str, Any]:
+        """
+        Get alert channel configuration by name.
+
+        Args:
+            channel_name: Channel name
+
+        Returns:
+            Channel configuration dictionary
+
+        Raises:
+            ValueError: If channel not found
+        """
+        if channel_name not in self.alert_channels:
+            available = ", ".join(sorted(self.alert_channels.keys()))
+            raise ValueError(
+                f"Alert channel '{channel_name}' not found. "
+                f"Available channels: {available}"
+            )
+
+        return self.alert_channels[channel_name]

detectkit/config/project_config.py

@@ -0,0 +1,164 @@
+"""
+Project configuration models.
+
+Defines configuration structure for detectkit_project.yml.
+"""
+
+from pathlib import Path
+from typing import Dict, Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class ProjectPathsConfig(BaseModel):
+    """
+    Project directory paths configuration.
+
+    Attributes:
+        metrics: Directory containing metric YAML files
+        sql: Directory containing SQL query files
+        templates: Directory containing alert templates
+    """
+
+    metrics: str = Field(default="metrics", description="Metrics directory")
+    sql: str = Field(default="sql", description="SQL files directory")
+    templates: str = Field(default="templates", description="Templates directory")
+
+
+class ProjectTablesConfig(BaseModel):
+    """
+    Default internal table names for the project.
+
+    Attributes:
+        datapoints: Default datapoints table name
+        detections: Default detections table name
+        tasks: Default tasks table name
+    """
+
+    datapoints: str = Field(
+        default="_dtk_datapoints", description="Default datapoints table"
+    )
+    detections: str = Field(
+        default="_dtk_detections", description="Default detections table"
+    )
+    tasks: str = Field(default="_dtk_tasks", description="Default tasks table")
+
+
+class ProjectTimeoutsConfig(BaseModel):
+    """
+    Default timeout values for operations (in seconds).
+
+    Attributes:
+        load: Timeout for data loading operations
+        detect: Timeout for detection operations
+        alert: Timeout for alerting operations
+    """
+
+    load: int = Field(default=3600, description="Load timeout (seconds)")
+    detect: int = Field(default=7200, description="Detect timeout (seconds)")
+    alert: int = Field(default=300, description="Alert timeout (seconds)")
+
+    @field_validator("load", "detect", "alert")
+    @classmethod
+    def validate_timeout(cls, v: int) -> int:
+        """Validate timeout value."""
+        if v < 1:
+            raise ValueError("Timeout must be at least 1 second")
+        if v > 86400:  # 24 hours
+            raise ValueError("Timeout cannot exceed 24 hours (86400 seconds)")
+        return v
+
+
+class ProjectConfig(BaseModel):
+    """
+    Project configuration loaded from detectkit_project.yml.
+
+    Attributes:
+        name: Project name
+        version: Project version
+        paths: Directory paths configuration
+        tables: Default table names
+        timeouts: Operation timeouts
+        default_profile: Default database profile to use
+
+    Example YAML:
+        ```yaml
+        name: "my_analytics_project"
+        version: "1.0"
+
+        paths:
+          metrics: "metrics"
+          sql: "sql"
+          templates: "templates"
+
+        tables:
+          datapoints: "_dtk_datapoints"
+          detections: "_dtk_detections"
+          tasks: "_dtk_tasks"
+
+        timeouts:
+          load: 3600
+          detect: 7200
+          alert: 300
+
+        default_profile: "clickhouse_prod"
+        ```
+    """
+
+    name: str = Field(..., description="Project name")
+    version: str = Field(default="1.0", description="Project version")
+    paths: ProjectPathsConfig = Field(
+        default_factory=ProjectPathsConfig, description="Directory paths"
+    )
+    tables: ProjectTablesConfig = Field(
+        default_factory=ProjectTablesConfig, description="Default table names"
+    )
+    timeouts: ProjectTimeoutsConfig = Field(
+        default_factory=ProjectTimeoutsConfig, description="Operation timeouts"
+    )
+    default_profile: str = Field(..., description="Default database profile")
+
+    @field_validator("name")
+    @classmethod
+    def validate_name(cls, v: str) -> str:
+        """Validate project name."""
+        if not v:
+            raise ValueError("Project name cannot be empty")
+        # Allow alphanumeric, underscore, dash, space
+        if not all(c.isalnum() or c in ("_", "-", " ") for c in v):
+            raise ValueError(
+                "Project name can only contain alphanumeric characters, "
+                "underscores, dashes, and spaces"
+            )
+        return v
+
+    @classmethod
+    def from_yaml_file(cls, path: Path) -> "ProjectConfig":
+        """
+        Load project configuration from YAML file.
+
+        Args:
+            path: Path to detectkit_project.yml
+
+        Returns:
+            ProjectConfig instance
+
+        Raises:
+            FileNotFoundError: If file doesn't exist
+            ValueError: If YAML is invalid
+
+        Example:
+            >>> config = ProjectConfig.from_yaml_file(Path("detectkit_project.yml"))
+        """
+        import yaml
+
+        if not path.exists():
+            raise FileNotFoundError(f"Project config file not found: {path}")
+
+        with open(path, "r") as f:
+            data = yaml.safe_load(f)
+
+        if not data:
+            raise ValueError(f"Empty project config file: {path}")
+
+        return cls.model_validate(data)

detectkit/config/validator.py

@@ -0,0 +1,124 @@
+"""
+Metric configuration validation.
+
+This module provides validation functions for metric configurations,
+ensuring data integrity and preventing configuration errors.
+"""
+
+from pathlib import Path
+from typing import List, Tuple
+
+from detectkit.config.metric_config import MetricConfig
+
+
+def validate_metric_uniqueness(metric_paths: List[Path]) -> List[Tuple[Path, MetricConfig]]:
+    """
+    Load all metrics and validate that metric names are unique.
+
+    This validation is CRITICAL for data integrity because duplicate metric names
+    would cause:
+    - Data corruption (mixed data in _dtk_datapoints table)
+    - Task blocking (lock conflicts in _dtk_tasks table)
+    - Wrong anomaly detection (detectors receive mixed data from different sources)
+    - Data loss (ReplacingMergeTree ignores duplicate inserts)
+
+    Args:
+        metric_paths: List of paths to metric YAML files
+
+    Returns:
+        List of (path, config) tuples for all valid metrics
+
+    Raises:
+        ValueError: If duplicate metric names are found, with clear error message
+            showing which files have conflicting names
+        ValidationError: If any metric config fails to parse
+
+    Example:
+        >>> paths = [Path("metrics/api/cpu.yml"), Path("metrics/system/cpu.yml")]
+        >>> validate_metric_uniqueness(paths)
+        ValueError: Duplicate metric name 'cpu_usage' found:
+          - metrics/api/cpu.yml
+          - metrics/system/cpu.yml
+
+        Metric names must be unique across the project.
+        Please rename one of the metrics.
+    """
+    configs: List[Tuple[Path, MetricConfig]] = []
+    seen_names: dict[str, Path] = {}
+
+    for metric_path in metric_paths:
+        # Load and parse config
+        try:
+            config = MetricConfig.from_yaml_file(metric_path)
+        except Exception as e:
+            raise ValueError(
+                f"Failed to parse metric config at {metric_path}:\n{e}"
+            ) from e
+
+        # Check for duplicate metric names
+        if config.name in seen_names:
+            conflicting_path = seen_names[config.name]
+            raise ValueError(
+                f"Duplicate metric name '{config.name}' found:\n"
+                f"  - {conflicting_path}\n"
+                f"  - {metric_path}\n\n"
+                f"Metric names must be unique across the project.\n"
+                f"Please rename one of the metrics to avoid data corruption."
+            )
+
+        seen_names[config.name] = metric_path
+        configs.append((metric_path, config))
+
+    return configs
+
+
+def validate_project_metrics(project_root: Path) -> List[Tuple[Path, MetricConfig]]:
+    """
+    Load and validate all metrics in the project.
+
+    This is a convenience function that:
+    1. Finds all *.yml and *.yaml files in the metrics/ directory (recursively)
+    2. Validates uniqueness of metric names
+    3. Returns validated list of (path, config) tuples
+
+    Args:
+        project_root: Path to project root directory (contains metrics/ folder)
+
+    Returns:
+        List of (path, config) tuples for all valid metrics
+
+    Raises:
+        ValueError: If duplicate metric names found or configs fail validation
+        FileNotFoundError: If metrics/ directory doesn't exist
+
+    Example:
+        >>> from pathlib import Path
+        >>> project_root = Path("/path/to/project")
+        >>> metrics = validate_project_metrics(project_root)
+        >>> for path, config in metrics:
+        ...     print(f"{config.name}: {path}")
+    """
+    metrics_dir = project_root / "metrics"
+
+    if not metrics_dir.exists():
+        raise FileNotFoundError(
+            f"Metrics directory not found: {metrics_dir}\n"
+            f"Expected structure:\n"
+            f"  {project_root}/\n"
+            f"    metrics/\n"
+            f"      your_metric.yml\n"
+        )
+
+    # Find all metric files recursively
+    metric_paths = []
+    for pattern in ["**/*.yml", "**/*.yaml"]:
+        metric_paths.extend(metrics_dir.glob(pattern))
+
+    if not metric_paths:
+        raise ValueError(
+            f"No metric files found in {metrics_dir}\n"
+            f"Expected at least one *.yml or *.yaml file."
+        )
+
+    # Validate uniqueness
+    return validate_metric_uniqueness(metric_paths)

detectkit/core/__init__.py

@@ -0,0 +1,6 @@
+"""Core functionality for detectk."""
+
+from detectkit.core.interval import Interval
+from detectkit.core.models import ColumnDefinition, TableModel
+
+__all__ = ["Interval", "ColumnDefinition", "TableModel"]