ch-migrate-cli 0.5.0__py3-none-any.whl

ch_migrate/config.py

@@ -0,0 +1,116 @@
+"""Configuration loading for ch-migrate-cli."""
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from ch_migrate.secrets import get_secret
+
+
+def load_config(config_path: Path) -> dict[str, Any]:
+    """
+    Load YAML configuration file.
+
+    Args:
+        config_path: Path to config.yaml
+
+    Returns:
+        Parsed configuration dictionary
+
+    Raises:
+        FileNotFoundError: If config file doesn't exist
+    """
+    if not config_path.exists():
+        raise FileNotFoundError(f"Configuration file not found: {config_path}")
+
+    with open(config_path) as f:
+        return yaml.safe_load(f)  # type: ignore[no-any-return]
+
+
+def get_env_config(env_name: str, config_path: Path) -> dict[str, Any]:
+    """
+    Get configuration for a specific environment, merging defaults.
+
+    Loads non-secret config from YAML, secrets from environment variables.
+    Supports both new field names (migration_user, migration_password) and
+    legacy names (user, password) for backward compatibility.
+
+    Args:
+        env_name: Environment name (dev, staging, production, etc.)
+        config_path: Path to config.yaml
+
+    Returns:
+        Complete environment configuration with secrets
+
+    Raises:
+        ValueError: If environment not found or required secrets missing
+    """
+    config = load_config(config_path)
+
+    environments = config.get("environments", {})
+    if env_name not in environments:
+        available = ", ".join(environments.keys()) or "(none)"
+        raise ValueError(f"Unknown environment: {env_name}. Available: {available}")
+
+    # Merge defaults with environment-specific config
+    defaults = config.get("defaults", {})
+    env_config = {**defaults, **environments[env_name]}
+
+    # Add project name from config if available
+    project_config = config.get("project", {})
+    if isinstance(project_config, dict) and "name" in project_config:
+        env_config["project"] = project_config["name"]
+
+    # Get SSM config if present (for secret lookups)
+    ssm_config = environments[env_name].get("ssm", {})
+    if ssm_config:
+        env_config["ssm"] = ssm_config
+
+    # Get AWS region for SSM lookups (optional, uses AWS default if not set)
+    aws_region = env_config.get("aws_region")
+
+    # Load secrets using unified get_secret() - uses SSM if configured, otherwise env vars
+    # Migration password is required
+    password = get_secret(
+        env_name,
+        "migration_password",
+        ssm_path=ssm_config.get("migration_password"),
+        aws_region=aws_region,
+        required=True,
+    )
+    env_config["password"] = password  # Legacy field for backward compat
+
+    # Optional: admin password (for bootstrap)
+    env_config["admin_password"] = get_secret(
+        env_name,
+        "admin_password",
+        ssm_path=ssm_config.get("admin_password"),
+        aws_region=aws_region,
+        required=False,
+    )
+
+    # Optional: dict_reader password (for dictionaries)
+    env_config["dict_reader_password"] = get_secret(
+        env_name,
+        "dict_reader_password",
+        ssm_path=ssm_config.get("dict_reader_password"),
+        aws_region=aws_region,
+        required=False,
+    )
+
+    # Optional: MCP user password (for read-only access)
+    env_config["mcp_password"] = get_secret(
+        env_name,
+        "mcp_password",
+        ssm_path=ssm_config.get("mcp_password"),
+        aws_region=aws_region,
+        required=False,
+    )
+
+    # Pass through top-level hooks section (used by env.py for pre/post migrate)
+    hooks_config = config.get("hooks")
+    if hooks_config:
+        env_config["hooks"] = hooks_config
+
+    return env_config

ch_migrate/connection.py

@@ -0,0 +1,83 @@
+"""Shared ClickHouse connection helpers for CLI commands."""
+
+from __future__ import annotations
+
+import io
+import sys
+from contextlib import contextmanager
+from typing import Any
+
+
+@contextmanager
+def _suppress_stderr():
+    """Suppress stderr during clickhouse_connect operations.
+
+    clickhouse_connect prints "Unexpected Http Driver Exception" directly
+    to stderr on connection failures, bypassing the logging framework.
+    """
+    old_stderr = sys.stderr
+    sys.stderr = io.StringIO()
+    try:
+        yield
+    finally:
+        sys.stderr = old_stderr
+
+
+def get_client(env_config: dict[str, Any]) -> Any:
+    """Create a clickhouse_connect client using migration user credentials.
+
+    Args:
+        env_config: Environment config dict from get_env_config().
+
+    Returns:
+        A clickhouse_connect Client instance.
+    """
+    import clickhouse_connect
+
+    secure = env_config.get("secure", True)
+    return clickhouse_connect.get_client(
+        host=env_config["host"],
+        port=env_config.get("port", 8443 if secure else 8123),
+        username=env_config.get("migration_user") or env_config.get("user", ""),
+        password=env_config.get("password", ""),
+        secure=secure,
+        interface="https" if secure else "http",
+        connect_timeout=10,
+        send_receive_timeout=15,
+    )
+
+
+def get_current_heads(env_config: dict[str, Any]) -> set[str]:
+    """Query alembic_version table for current head revision(s).
+
+    Alembic stores only the current head(s) in alembic_version, not
+    every historically applied revision. Use resolve_applied_revisions()
+    to expand these into the full set of applied revisions.
+
+    Args:
+        env_config: Environment config dict from get_env_config().
+
+    Returns:
+        Set of current head revision ID strings (usually just one).
+    """
+    from clickhouse_connect.driver.binding import quote_identifier
+
+    with _suppress_stderr():
+        client = get_client(env_config)
+        try:
+            db = env_config["database"]
+            engines = client.query(
+                "SELECT engine FROM system.tables "
+                "WHERE database = {db:String} AND name = 'alembic_version'",
+                parameters={"db": db},
+            ).result_rows
+            if not engines:
+                return set()
+            # Legacy tables require FINAL; the official plain MergeTree rejects it.
+            final = " FINAL" if engines[0][0].endswith("ReplacingMergeTree") else ""
+            result = client.query(
+                f"SELECT version_num FROM {quote_identifier(db)}.alembic_version{final}"
+            )
+            return {row[0] for row in result.result_rows}
+        finally:
+            client.close()

ch_migrate/deps.py

@@ -0,0 +1,123 @@
+"""Dependency graph analysis and migration validation."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any
+
+from ch_migrate.introspect import (
+    DependencyGraph,
+    DepType,
+    ObjectNode,
+    get_dependencies,
+)
+
+
+@dataclass
+class MigrationWarning:
+    severity: str  # "error" or "warning"
+    message: str
+    affected_objects: list[str]
+
+
+# Patterns that detect destructive operations
+_RE_DROP_TABLE = re.compile(
+    r"DROP\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:`?(\w+)`?\.)?`?(\w+)`?",
+    re.IGNORECASE,
+)
+
+_RE_DROP_VIEW = re.compile(
+    r"DROP\s+(?:MATERIALIZED\s+)?VIEW\s+(?:IF\s+EXISTS\s+)?(?:`?(\w+)`?\.)?`?(\w+)`?",
+    re.IGNORECASE,
+)
+
+_RE_DROP_DICT = re.compile(
+    r"DROP\s+DICTIONARY\s+(?:IF\s+EXISTS\s+)?(?:`?(\w+)`?\.)?`?(\w+)`?",
+    re.IGNORECASE,
+)
+
+
+def validate_migration(sql: str, graph: DependencyGraph) -> list[MigrationWarning]:
+    """Check if migration SQL would break dependencies in the graph.
+
+    Args:
+        sql: The migration SQL to validate.
+        graph: A live DependencyGraph from introspect.get_dependencies().
+
+    Returns:
+        List of warnings about potential dependency breakage.
+    """
+    warnings: list[MigrationWarning] = []
+
+    # Check DROP TABLE
+    for m in _RE_DROP_TABLE.finditer(sql):
+        table_name = m.group(2)
+        if table_name in graph.nodes:
+            affected = graph.affected_by_drop(table_name)
+            if affected:
+                # Distinguish schema vs data_flow impact
+                schema_deps = []
+                data_flow_deps = []
+                for edge in graph.edges:
+                    if edge.source == table_name:
+                        if edge.dep_type == DepType.SCHEMA:
+                            schema_deps.append(edge.target)
+                        elif edge.dep_type == DepType.DATA_FLOW:
+                            data_flow_deps.append(edge.target)
+
+                if schema_deps:
+                    warnings.append(MigrationWarning(
+                        severity="error",
+                        message=f"DROP TABLE {table_name} breaks schema dependencies: {', '.join(schema_deps)}",
+                        affected_objects=schema_deps,
+                    ))
+                if data_flow_deps:
+                    warnings.append(MigrationWarning(
+                        severity="warning",
+                        message=f"DROP TABLE {table_name} breaks data flow to: {', '.join(data_flow_deps)}",
+                        affected_objects=data_flow_deps,
+                    ))
+
+    # Check DROP VIEW / DROP MATERIALIZED VIEW
+    for m in _RE_DROP_VIEW.finditer(sql):
+        view_name = m.group(2)
+        if view_name in graph.nodes:
+            affected = graph.affected_by_drop(view_name)
+            if affected:
+                affected_names = [n.name for n in affected]
+                warnings.append(MigrationWarning(
+                    severity="warning",
+                    message=f"DROP VIEW {view_name} affects: {', '.join(affected_names)}",
+                    affected_objects=affected_names,
+                ))
+
+    # Check DROP DICTIONARY
+    for m in _RE_DROP_DICT.finditer(sql):
+        dict_name = m.group(2)
+        if dict_name in graph.nodes:
+            affected = graph.affected_by_drop(dict_name)
+            if affected:
+                affected_names = [n.name for n in affected]
+                warnings.append(MigrationWarning(
+                    severity="warning",
+                    message=f"DROP DICTIONARY {dict_name} affects: {', '.join(affected_names)}",
+                    affected_objects=affected_names,
+                ))
+
+    return warnings
+
+
+def build_dependency_graph(client: Any, database: str) -> DependencyGraph:
+    """Build a dependency graph from the live database.
+
+    Convenience wrapper around introspect.get_dependencies().
+
+    Args:
+        client: clickhouse-connect client.
+        database: Database name.
+
+    Returns:
+        A DependencyGraph with nodes and typed edges.
+    """
+    return get_dependencies(client, database)

ch_migrate/diff.py

@@ -0,0 +1,232 @@
+"""Schema comparison: field-by-field structured diff between two Schema objects."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Literal
+
+from ch_migrate.introspect import (
+    ColumnDefinition,
+    Schema,
+    TableDefinition,
+)
+
+
+class DiffStatus(str, Enum):
+    IN_SYNC = "in_sync"
+    MODIFIED = "modified"
+    LOCAL_ONLY = "local_only"
+    REMOTE_ONLY = "remote_only"
+
+
+@dataclass
+class FieldDiff:
+    field_name: str
+    local_value: str | None
+    remote_value: str | None
+    message: str
+
+
+@dataclass
+class SchemaDiff:
+    name: str
+    obj_type: str  # "table", "view", "materialized_view", "dictionary"
+    status: DiffStatus
+    field_diffs: list[FieldDiff] = field(default_factory=list)
+
+
+def _compare_columns(
+    local_cols: list[ColumnDefinition],
+    remote_cols: list[ColumnDefinition],
+) -> list[FieldDiff]:
+    """Compare column lists field-by-field."""
+    diffs: list[FieldDiff] = []
+    local_map = {c.name: c for c in local_cols}
+    remote_map = {c.name: c for c in remote_cols}
+
+    all_names = dict.fromkeys([c.name for c in local_cols] + [c.name for c in remote_cols])
+
+    for name in all_names:
+        local_col = local_map.get(name)
+        remote_col = remote_map.get(name)
+
+        if local_col and not remote_col:
+            diffs.append(FieldDiff(
+                field_name=f"column '{name}'",
+                local_value=local_col.type,
+                remote_value=None,
+                message=f"column '{name}' {local_col.type} is in the snapshot but not in the database",
+            ))
+        elif remote_col and not local_col:
+            diffs.append(FieldDiff(
+                field_name=f"column '{name}'",
+                local_value=None,
+                remote_value=remote_col.type,
+                message=f"column '{name}' {remote_col.type} is in the database but not in the snapshot",
+            ))
+        elif local_col and remote_col:
+            if local_col.type != remote_col.type:
+                diffs.append(FieldDiff(
+                    field_name=f"column '{name}' type",
+                    local_value=local_col.type,
+                    remote_value=remote_col.type,
+                    message=f"column '{name}' type differs: {local_col.type} vs {remote_col.type}",
+                ))
+            if local_col.default_kind != remote_col.default_kind:
+                diffs.append(FieldDiff(
+                    field_name=f"column '{name}' default_kind",
+                    local_value=local_col.default_kind,
+                    remote_value=remote_col.default_kind,
+                    message=f"column '{name}' default kind differs",
+                ))
+            if local_col.default_expr != remote_col.default_expr:
+                diffs.append(FieldDiff(
+                    field_name=f"column '{name}' default_expr",
+                    local_value=local_col.default_expr,
+                    remote_value=remote_col.default_expr,
+                    message=f"column '{name}' default expression differs",
+                ))
+            if local_col.codec != remote_col.codec:
+                diffs.append(FieldDiff(
+                    field_name=f"column '{name}' codec",
+                    local_value=local_col.codec,
+                    remote_value=remote_col.codec,
+                    message=f"column '{name}' codec differs: {local_col.codec} vs {remote_col.codec}",
+                ))
+
+    return diffs
+
+
+def _compare_tables(local: TableDefinition, remote: TableDefinition) -> list[FieldDiff]:
+    """Field-by-field comparison of two TableDefinitions."""
+    diffs: list[FieldDiff] = []
+
+    # Engine
+    if local.engine != remote.engine:
+        diffs.append(FieldDiff(
+            field_name="engine",
+            local_value=local.engine,
+            remote_value=remote.engine,
+            message=f"engine differs: {local.engine} vs {remote.engine}",
+        ))
+
+    # Columns
+    diffs.extend(_compare_columns(local.columns, remote.columns))
+
+    # ORDER BY
+    if local.order_by != remote.order_by:
+        diffs.append(FieldDiff(
+            field_name="order_by",
+            local_value=", ".join(local.order_by),
+            remote_value=", ".join(remote.order_by),
+            message=f"ORDER BY differs",
+        ))
+
+    # PARTITION BY
+    if local.partition_by != remote.partition_by:
+        diffs.append(FieldDiff(
+            field_name="partition_by",
+            local_value=local.partition_by,
+            remote_value=remote.partition_by,
+            message=f"PARTITION BY differs",
+        ))
+
+    # TTL
+    if local.ttl != remote.ttl:
+        diffs.append(FieldDiff(
+            field_name="ttl",
+            local_value=local.ttl,
+            remote_value=remote.ttl,
+            message=f"TTL differs",
+        ))
+
+    # Settings
+    if local.settings != remote.settings:
+        diffs.append(FieldDiff(
+            field_name="settings",
+            local_value=str(local.settings),
+            remote_value=str(remote.settings),
+            message=f"SETTINGS differ",
+        ))
+
+    return diffs
+
+
+def _normalize_ddl(raw: str) -> str:
+    """Normalize raw DDL for fallback string comparison."""
+    import re
+    s = raw.strip()
+    s = re.sub(r"\s+", " ", s)
+    return s
+
+
+def _compare_raw_ddl(local_ddl: str, remote_ddl: str) -> list[FieldDiff]:
+    """Fallback: normalized string comparison for unparseable objects."""
+    if _normalize_ddl(local_ddl) != _normalize_ddl(remote_ddl):
+        return [FieldDiff(
+            field_name="raw_ddl",
+            local_value=local_ddl[:200] if local_ddl else None,
+            remote_value=remote_ddl[:200] if remote_ddl else None,
+            message="DDL definition differs (raw comparison)",
+        )]
+    return []
+
+
+def compare_schemas(local: Schema, live: Schema) -> list[SchemaDiff]:
+    """Compare two Schema objects and return a list of differences.
+
+    Args:
+        local: Schema from local snapshot files.
+        live: Schema from the live database.
+
+    Returns:
+        List of SchemaDiff objects. Empty list means schemas are in sync.
+    """
+    results: list[SchemaDiff] = []
+
+    type_map: list[tuple[str, dict, dict]] = [
+        ("table", local.tables, live.tables),
+        ("view", local.views, live.views),
+        ("materialized_view", local.materialized_views, live.materialized_views),
+        ("dictionary", local.dictionaries, live.dictionaries),
+    ]
+
+    for obj_type, local_objs, live_objs in type_map:
+        all_names = dict.fromkeys(list(local_objs.keys()) + list(live_objs.keys()))
+
+        for name in all_names:
+            local_obj = local_objs.get(name)
+            live_obj = live_objs.get(name)
+
+            if local_obj and not live_obj:
+                results.append(SchemaDiff(
+                    name=name, obj_type=obj_type, status=DiffStatus.LOCAL_ONLY,
+                ))
+            elif live_obj and not local_obj:
+                results.append(SchemaDiff(
+                    name=name, obj_type=obj_type, status=DiffStatus.REMOTE_ONLY,
+                ))
+            else:
+                # Both exist — compare
+                field_diffs: list[FieldDiff] = []
+
+                if obj_type == "table" and isinstance(local_obj, TableDefinition) and isinstance(live_obj, TableDefinition):
+                    field_diffs = _compare_tables(local_obj, live_obj)
+                else:
+                    # Fallback to raw DDL comparison for views, MVs, dicts
+                    local_ddl = getattr(local_obj, "raw_ddl", "")
+                    live_ddl = getattr(live_obj, "raw_ddl", "")
+                    field_diffs = _compare_raw_ddl(local_ddl, live_ddl)
+
+                if field_diffs:
+                    results.append(SchemaDiff(
+                        name=name, obj_type=obj_type,
+                        status=DiffStatus.MODIFIED, field_diffs=field_diffs,
+                    ))
+                else:
+                    results.append(SchemaDiff(
+                        name=name, obj_type=obj_type, status=DiffStatus.IN_SYNC,
+                    ))
+
+    return results