detectkit 0.1.0__py3-none-any.whl

detectkit/config/profile.py

@@ -0,0 +1,285 @@
+"""
+Profile configuration for detectk.
+
+Manages database connections and locations (similar to dbt profiles).
+"""
+
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import yaml
+from pydantic import BaseModel, Field, field_validator
+
+from detectkit.database.clickhouse_manager import ClickHouseDatabaseManager
+from detectkit.database.manager import BaseDatabaseManager
+
+
+class ProfileConfig(BaseModel):
+    """
+    Single profile configuration.
+
+    Defines connection parameters and database locations for a specific
+    environment (dev, prod, etc.).
+
+    Attributes:
+        type: Database type ("clickhouse", "postgres", "mysql")
+        host: Database host
+        port: Database port
+        user: Database user
+        password: Database password
+        internal_database: Database/schema for internal tables
+        internal_schema: Schema for internal tables (PostgreSQL only)
+        data_database: Database for user data tables
+        data_schema: Schema for user data (PostgreSQL only)
+        settings: Additional database-specific settings
+    """
+
+    type: str = Field(..., description="Database type")
+    host: str = Field(default="localhost", description="Database host")
+    port: int = Field(..., description="Database port")
+    user: str = Field(default="default", description="Database user")
+    password: str = Field(default="", description="Database password")
+
+    # Internal location for _dtk_* tables
+    internal_database: Optional[str] = Field(
+        default=None,
+        description="Database for internal tables (ClickHouse/MySQL)"
+    )
+    internal_schema: Optional[str] = Field(
+        default=None,
+        description="Schema for internal tables (PostgreSQL)"
+    )
+
+    # Data location for user tables
+    data_database: Optional[str] = Field(
+        default=None,
+        description="Database for user data tables (ClickHouse/MySQL)"
+    )
+    data_schema: Optional[str] = Field(
+        default=None,
+        description="Schema for user data (PostgreSQL)"
+    )
+
+    settings: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional database settings"
+    )
+
+    @field_validator("type")
+    @classmethod
+    def validate_type(cls, v: str) -> str:
+        """Validate database type."""
+        allowed_types = {"clickhouse", "postgres", "mysql"}
+        if v not in allowed_types:
+            raise ValueError(
+                f"Invalid database type: {v}. "
+                f"Allowed types: {', '.join(allowed_types)}"
+            )
+        return v
+
+    @field_validator("port")
+    @classmethod
+    def validate_port(cls, v: int) -> int:
+        """Validate port number."""
+        if not (1 <= v <= 65535):
+            raise ValueError(f"Port must be between 1 and 65535, got {v}")
+        return v
+
+    def get_internal_location(self) -> str:
+        """
+        Get internal location (database or schema).
+
+        Returns:
+            Internal database/schema name
+
+        Raises:
+            ValueError: If location not configured
+        """
+        if self.type == "clickhouse":
+            if not self.internal_database:
+                raise ValueError("internal_database must be set for ClickHouse")
+            return self.internal_database
+        elif self.type == "postgres":
+            if not self.internal_schema:
+                raise ValueError("internal_schema must be set for PostgreSQL")
+            return self.internal_schema
+        elif self.type == "mysql":
+            if not self.internal_database:
+                raise ValueError("internal_database must be set for MySQL")
+            return self.internal_database
+        else:
+            raise ValueError(f"Unsupported database type: {self.type}")
+
+    def get_data_location(self) -> str:
+        """
+        Get data location (database or schema).
+
+        Returns:
+            Data database/schema name
+
+        Raises:
+            ValueError: If location not configured
+        """
+        if self.type == "clickhouse":
+            if not self.data_database:
+                raise ValueError("data_database must be set for ClickHouse")
+            return self.data_database
+        elif self.type == "postgres":
+            if not self.data_schema:
+                raise ValueError("data_schema must be set for PostgreSQL")
+            return self.data_schema
+        elif self.type == "mysql":
+            if not self.data_database:
+                raise ValueError("data_database must be set for MySQL")
+            return self.data_database
+        else:
+            raise ValueError(f"Unsupported database type: {self.type}")
+
+    def create_manager(self) -> BaseDatabaseManager:
+        """
+        Create database manager from profile configuration.
+
+        Returns:
+            Database manager instance
+
+        Raises:
+            NotImplementedError: If database type not yet implemented
+        """
+        if self.type == "clickhouse":
+            return ClickHouseDatabaseManager(
+                host=self.host,
+                port=self.port,
+                user=self.user,
+                password=self.password,
+                internal_database=self.get_internal_location(),
+                data_database=self.get_data_location(),
+                settings=self.settings,
+            )
+        elif self.type == "postgres":
+            raise NotImplementedError("PostgreSQL support coming soon")
+        elif self.type == "mysql":
+            raise NotImplementedError("MySQL support coming soon")
+        else:
+            raise ValueError(f"Unsupported database type: {self.type}")
+
+
+class ProfilesConfig(BaseModel):
+    """
+    Container for multiple profile configurations.
+
+    Loaded from profiles.yml file.
+
+    Attributes:
+        profiles: Dictionary mapping profile names to configurations
+        default_profile: Name of default profile to use
+        alert_channels: Dictionary mapping channel names to configurations
+    """
+
+    profiles: Dict[str, ProfileConfig]
+    default_profile: Optional[str] = None
+    alert_channels: Dict[str, Dict[str, Any]] = Field(
+        default_factory=dict, description="Alert channel configurations"
+    )
+
+    @field_validator("default_profile")
+    @classmethod
+    def validate_default_profile(cls, v: Optional[str], info) -> Optional[str]:
+        """Validate default profile exists."""
+        if v is not None:
+            profiles = info.data.get("profiles", {})
+            if v not in profiles:
+                raise ValueError(
+                    f"default_profile '{v}' not found in profiles. "
+                    f"Available profiles: {', '.join(profiles.keys())}"
+                )
+        return v
+
+    @classmethod
+    def from_yaml(cls, path: Path) -> "ProfilesConfig":
+        """
+        Load profiles from YAML file.
+
+        Args:
+            path: Path to profiles.yml
+
+        Returns:
+            ProfilesConfig instance
+
+        Raises:
+            FileNotFoundError: If file doesn't exist
+            ValueError: If YAML is invalid
+        """
+        if not path.exists():
+            raise FileNotFoundError(f"Profiles file not found: {path}")
+
+        with open(path, "r") as f:
+            data = yaml.safe_load(f)
+
+        if not data:
+            raise ValueError("Profiles file is empty")
+
+        return cls.model_validate(data)
+
+    def get_profile(self, name: Optional[str] = None) -> ProfileConfig:
+        """
+        Get profile configuration by name.
+
+        Args:
+            name: Profile name (if None, use default_profile)
+
+        Returns:
+            ProfileConfig instance
+
+        Raises:
+            ValueError: If profile not found or no default set
+        """
+        if name is None:
+            if self.default_profile is None:
+                raise ValueError(
+                    "No profile name specified and no default_profile set. "
+                    f"Available profiles: {', '.join(self.profiles.keys())}"
+                )
+            name = self.default_profile
+
+        if name not in self.profiles:
+            raise ValueError(
+                f"Profile '{name}' not found. "
+                f"Available profiles: {', '.join(self.profiles.keys())}"
+            )
+
+        return self.profiles[name]
+
+    def create_manager(self, profile_name: Optional[str] = None) -> BaseDatabaseManager:
+        """
+        Create database manager for a profile.
+
+        Args:
+            profile_name: Profile name (if None, use default)
+
+        Returns:
+            Database manager instance
+        """
+        profile = self.get_profile(profile_name)
+        return profile.create_manager()
+
+    def get_alert_channel_config(self, channel_name: str) -> Dict[str, Any]:
+        """
+        Get alert channel configuration by name.
+
+        Args:
+            channel_name: Channel name
+
+        Returns:
+            Channel configuration dictionary
+
+        Raises:
+            ValueError: If channel not found
+        """
+        if channel_name not in self.alert_channels:
+            available = ", ".join(sorted(self.alert_channels.keys()))
+            raise ValueError(
+                f"Alert channel '{channel_name}' not found. "
+                f"Available channels: {available}"
+            )
+
+        return self.alert_channels[channel_name]

detectkit/config/project_config.py

@@ -0,0 +1,164 @@
+"""
+Project configuration models.
+
+Defines configuration structure for detectkit_project.yml.
+"""
+
+from pathlib import Path
+from typing import Dict, Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class ProjectPathsConfig(BaseModel):
+    """
+    Project directory paths configuration.
+
+    Attributes:
+        metrics: Directory containing metric YAML files
+        sql: Directory containing SQL query files
+        templates: Directory containing alert templates
+    """
+
+    metrics: str = Field(default="metrics", description="Metrics directory")
+    sql: str = Field(default="sql", description="SQL files directory")
+    templates: str = Field(default="templates", description="Templates directory")
+
+
+class ProjectTablesConfig(BaseModel):
+    """
+    Default internal table names for the project.
+
+    Attributes:
+        datapoints: Default datapoints table name
+        detections: Default detections table name
+        tasks: Default tasks table name
+    """
+
+    datapoints: str = Field(
+        default="_dtk_datapoints", description="Default datapoints table"
+    )
+    detections: str = Field(
+        default="_dtk_detections", description="Default detections table"
+    )
+    tasks: str = Field(default="_dtk_tasks", description="Default tasks table")
+
+
+class ProjectTimeoutsConfig(BaseModel):
+    """
+    Default timeout values for operations (in seconds).
+
+    Attributes:
+        load: Timeout for data loading operations
+        detect: Timeout for detection operations
+        alert: Timeout for alerting operations
+    """
+
+    load: int = Field(default=3600, description="Load timeout (seconds)")
+    detect: int = Field(default=7200, description="Detect timeout (seconds)")
+    alert: int = Field(default=300, description="Alert timeout (seconds)")
+
+    @field_validator("load", "detect", "alert")
+    @classmethod
+    def validate_timeout(cls, v: int) -> int:
+        """Validate timeout value."""
+        if v < 1:
+            raise ValueError("Timeout must be at least 1 second")
+        if v > 86400:  # 24 hours
+            raise ValueError("Timeout cannot exceed 24 hours (86400 seconds)")
+        return v
+
+
+class ProjectConfig(BaseModel):
+    """
+    Project configuration loaded from detectkit_project.yml.
+
+    Attributes:
+        name: Project name
+        version: Project version
+        paths: Directory paths configuration
+        tables: Default table names
+        timeouts: Operation timeouts
+        default_profile: Default database profile to use
+
+    Example YAML:
+        ```yaml
+        name: "my_analytics_project"
+        version: "1.0"
+
+        paths:
+          metrics: "metrics"
+          sql: "sql"
+          templates: "templates"
+
+        tables:
+          datapoints: "_dtk_datapoints"
+          detections: "_dtk_detections"
+          tasks: "_dtk_tasks"
+
+        timeouts:
+          load: 3600
+          detect: 7200
+          alert: 300
+
+        default_profile: "clickhouse_prod"
+        ```
+    """
+
+    name: str = Field(..., description="Project name")
+    version: str = Field(default="1.0", description="Project version")
+    paths: ProjectPathsConfig = Field(
+        default_factory=ProjectPathsConfig, description="Directory paths"
+    )
+    tables: ProjectTablesConfig = Field(
+        default_factory=ProjectTablesConfig, description="Default table names"
+    )
+    timeouts: ProjectTimeoutsConfig = Field(
+        default_factory=ProjectTimeoutsConfig, description="Operation timeouts"
+    )
+    default_profile: str = Field(..., description="Default database profile")
+
+    @field_validator("name")
+    @classmethod
+    def validate_name(cls, v: str) -> str:
+        """Validate project name."""
+        if not v:
+            raise ValueError("Project name cannot be empty")
+        # Allow alphanumeric, underscore, dash, space
+        if not all(c.isalnum() or c in ("_", "-", " ") for c in v):
+            raise ValueError(
+                "Project name can only contain alphanumeric characters, "
+                "underscores, dashes, and spaces"
+            )
+        return v
+
+    @classmethod
+    def from_yaml_file(cls, path: Path) -> "ProjectConfig":
+        """
+        Load project configuration from YAML file.
+
+        Args:
+            path: Path to detectkit_project.yml
+
+        Returns:
+            ProjectConfig instance
+
+        Raises:
+            FileNotFoundError: If file doesn't exist
+            ValueError: If YAML is invalid
+
+        Example:
+            >>> config = ProjectConfig.from_yaml_file(Path("detectkit_project.yml"))
+        """
+        import yaml
+
+        if not path.exists():
+            raise FileNotFoundError(f"Project config file not found: {path}")
+
+        with open(path, "r") as f:
+            data = yaml.safe_load(f)
+
+        if not data:
+            raise ValueError(f"Empty project config file: {path}")
+
+        return cls.model_validate(data)

detectkit/core/__init__.py

@@ -0,0 +1,6 @@
+"""Core functionality for detectk."""
+
+from detectkit.core.interval import Interval
+from detectkit.core.models import ColumnDefinition, TableModel
+
+__all__ = ["Interval", "ColumnDefinition", "TableModel"]

detectkit/core/interval.py

@@ -0,0 +1,132 @@
+"""
+Interval parsing and handling.
+
+Supports:
+- Integer seconds: 600
+- String format: "10min", "1h", "1d"
+"""
+
+import re
+from typing import Union
+
+
+class Interval:
+    """
+    Represents a time interval in seconds.
+
+    Supports parsing from:
+    - Integer (seconds): 600
+    - String: "10min", "1h", "1d", "30s"
+
+    Examples:
+        >>> interval = Interval("10min")
+        >>> interval.seconds
+        600
+        >>> interval = Interval(3600)
+        >>> interval.seconds
+        3600
+    """
+
+    UNITS = {
+        's': 1,
+        'sec': 1,
+        'second': 1,
+        'seconds': 1,
+        'm': 60,
+        'min': 60,
+        'minute': 60,
+        'minutes': 60,
+        'h': 3600,
+        'hour': 3600,
+        'hours': 3600,
+        'd': 86400,
+        'day': 86400,
+        'days': 86400,
+    }
+
+    def __init__(self, value: Union[int, str]):
+        """
+        Initialize interval from integer or string.
+
+        Args:
+            value: Interval as integer (seconds) or string ("10min")
+
+        Raises:
+            ValueError: If string format is invalid
+        """
+        if isinstance(value, int):
+            if value <= 0:
+                raise ValueError(f"Interval must be positive, got {value}")
+            self._seconds = value
+        elif isinstance(value, str):
+            self._seconds = self._parse_string(value)
+        else:
+            raise TypeError(f"Interval must be int or str, got {type(value)}")
+
+    def _parse_string(self, s: str) -> int:
+        """
+        Parse interval string.
+
+        Args:
+            s: String like "10min", "1h", "30s"
+
+        Returns:
+            Interval in seconds
+
+        Raises:
+            ValueError: If format is invalid
+        """
+        s = s.strip().lower()
+
+        # Match pattern: digits followed by unit
+        match = re.match(r'^(\d+)([a-z]+)$', s)
+        if not match:
+            raise ValueError(
+                f"Invalid interval format: '{s}'. "
+                f"Expected format: <number><unit> (e.g., '10min', '1h')"
+            )
+
+        value_str, unit = match.groups()
+        value = int(value_str)
+
+        if value <= 0:
+            raise ValueError(f"Interval value must be positive, got {value}")
+
+        if unit not in self.UNITS:
+            raise ValueError(
+                f"Unknown time unit: '{unit}'. "
+                f"Supported units: {', '.join(sorted(set(self.UNITS.keys())))}"
+            )
+
+        return value * self.UNITS[unit]
+
+    @property
+    def seconds(self) -> int:
+        """Get interval in seconds."""
+        return self._seconds
+
+    def __eq__(self, other) -> bool:
+        """Check equality based on seconds."""
+        if isinstance(other, Interval):
+            return self._seconds == other._seconds
+        return False
+
+    def __hash__(self) -> int:
+        """Hash based on seconds."""
+        return hash(self._seconds)
+
+    def __repr__(self) -> str:
+        """String representation."""
+        return f"Interval({self._seconds})"
+
+    def __str__(self) -> str:
+        """User-friendly string representation."""
+        # Try to represent in human-readable format
+        if self._seconds % 86400 == 0:
+            return f"{self._seconds // 86400}d"
+        elif self._seconds % 3600 == 0:
+            return f"{self._seconds // 3600}h"
+        elif self._seconds % 60 == 0:
+            return f"{self._seconds // 60}min"
+        else:
+            return f"{self._seconds}s"

detectkit/core/models.py

@@ -0,0 +1,106 @@
+"""
+Core data models for detectk.
+
+Defines table schemas and column definitions for database abstraction.
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, List, Optional
+
+
+@dataclass
+class ColumnDefinition:
+    """
+    Definition of a table column.
+
+    Attributes:
+        name: Column name
+        type: SQL type (database-specific)
+        nullable: Whether column can be NULL
+        default: Default value for column
+    """
+
+    name: str
+    type: str
+    nullable: bool = False
+    default: Optional[Any] = None
+
+    def __post_init__(self):
+        """Validate column definition."""
+        if not self.name:
+            raise ValueError("Column name cannot be empty")
+        if not self.type:
+            raise ValueError("Column type cannot be empty")
+
+
+@dataclass
+class TableModel:
+    """
+    Model for table schema definition.
+
+    This is used by BaseDatabaseManager.create_table() to create tables
+    in a database-agnostic way.
+
+    Attributes:
+        columns: List of column definitions
+        primary_key: List of column names forming primary key
+        engine: Database engine (ClickHouse-specific, e.g., "MergeTree")
+        order_by: Columns for ORDER BY clause (ClickHouse-specific)
+        indexes: Additional indexes to create
+
+    Example:
+        >>> model = TableModel(
+        ...     columns=[
+        ...         ColumnDefinition("id", "Int32"),
+        ...         ColumnDefinition("name", "String"),
+        ...     ],
+        ...     primary_key=["id"],
+        ...     engine="MergeTree",
+        ...     order_by=["id"]
+        ... )
+    """
+
+    columns: List[ColumnDefinition]
+    primary_key: List[str]
+    engine: Optional[str] = None
+    order_by: Optional[List[str]] = None
+    indexes: List[str] = field(default_factory=list)
+
+    def __post_init__(self):
+        """Validate table model."""
+        if not self.columns:
+            raise ValueError("Table must have at least one column")
+
+        if not self.primary_key:
+            raise ValueError("Table must have a primary key")
+
+        # Validate primary key columns exist
+        column_names = {col.name for col in self.columns}
+        for pk_col in self.primary_key:
+            if pk_col not in column_names:
+                raise ValueError(
+                    f"Primary key column '{pk_col}' not found in table columns"
+                )
+
+        # Validate order_by columns exist (if specified)
+        if self.order_by:
+            for order_col in self.order_by:
+                if order_col not in column_names:
+                    raise ValueError(
+                        f"ORDER BY column '{order_col}' not found in table columns"
+                    )
+
+    def get_column(self, name: str) -> Optional[ColumnDefinition]:
+        """
+        Get column definition by name.
+
+        Args:
+            name: Column name
+
+        Returns:
+            ColumnDefinition or None if not found
+        """
+        for col in self.columns:
+            if col.name == name:
+                return col
+        return None