ch-migrate-cli 0.5.0__py3-none-any.whl

ch_migrate/__init__.py

@@ -0,0 +1,68 @@
+"""
+ch-migrate-cli: Alembic-based migrations for ClickHouse Cloud.
+
+Usage:
+    from ch_migrate import read_sql, get_db, get_env_config, create_dictionary
+"""
+
+__version__ = "0.5.0"
+
+
+from typing import Any
+
+
+# Lazy imports to avoid import errors before dependencies are created
+def __getattr__(name: str) -> Any:
+    if name in ("read_sql", "get_db", "create_dictionary", "on_cluster", "get_cluster"):
+        from ch_migrate.helpers import (
+            create_dictionary,
+            get_cluster,
+            get_db,
+            on_cluster,
+            read_sql,
+        )
+
+        return {
+            "read_sql": read_sql,
+            "get_db": get_db,
+            "create_dictionary": create_dictionary,
+            "on_cluster": on_cluster,
+            "get_cluster": get_cluster,
+        }[name]
+    elif name == "run_sql":
+        from ch_migrate.sql import run_sql
+
+        return run_sql
+    elif name == "IrreversibleMigration":
+        from ch_migrate.downgrade import IrreversibleMigration
+
+        return IrreversibleMigration
+    elif name == "get_env_config":
+        from ch_migrate.config import get_env_config
+
+        return get_env_config
+    elif name in ("get_secret", "SSMSecretNotFoundError", "SSMJsonKeyError"):
+        from ch_migrate.secrets import SSMJsonKeyError, SSMSecretNotFoundError, get_secret
+
+        return {
+            "get_secret": get_secret,
+            "SSMSecretNotFoundError": SSMSecretNotFoundError,
+            "SSMJsonKeyError": SSMJsonKeyError,
+        }[name]
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+    "__version__",
+    "read_sql",
+    "run_sql",
+    "IrreversibleMigration",
+    "get_db",
+    "get_env_config",
+    "create_dictionary",
+    "on_cluster",
+    "get_cluster",
+    "get_secret",
+    "SSMSecretNotFoundError",
+    "SSMJsonKeyError",
+]

ch_migrate/authoring.py

@@ -0,0 +1,163 @@
+"""The files `ch-migrate new` writes for a SQL-first migration.
+
+Alembic generates the revision file; this module rewrites it to run an upgrade
+SQL file and a downgrade SQL file, and creates those files in the
+object-centric history layout under migrations/sql/history/.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+from ch_migrate.rebase import _MISSING, _literal_assignment
+
+OBJECT_DIRS = {"table": "tables", "view": "views", "dictionary": "dictionaries"}
+OTHER_DIR = "other"
+
+
+@dataclass(frozen=True)
+class NewOptions:
+    """The authoring choices for one `new` invocation."""
+
+    table_name: str | None = None
+    view_name: str | None = None
+    dict_name: str | None = None
+    exchange: bool = False
+    python_migration: bool = False
+    irreversible_reason: str | None = None
+
+    def named_objects(self) -> list[tuple[str, str]]:
+        return [
+            (kind, name)
+            for kind, name in (
+                ("table", self.table_name),
+                ("view", self.view_name),
+                ("dictionary", self.dict_name),
+            )
+            if name
+        ]
+
+
+@dataclass(frozen=True)
+class RevisionHeader:
+    """Identifiers Alembic wrote into a freshly generated revision file."""
+
+    message: str
+    revision: str
+    down_revision: Any
+    branch_labels: Any
+    depends_on: Any
+    docstring_source: str
+
+
+@dataclass(frozen=True)
+class SqlFiles:
+    """Paths of a migration's SQL files, relative to migrations/sql/."""
+
+    upgrade: str
+    downgrade: str | None
+
+
+def read_revision_header(path: Path) -> RevisionHeader:
+    """Read the identifiers out of an Alembic-generated revision file."""
+    content = path.read_text()
+    module = ast.parse(content)
+    docstring = ast.get_docstring(module) or ""
+
+    def literal(name: str) -> Any:
+        value = _literal_assignment(content, name)
+        return None if value is _MISSING else value
+
+    return RevisionHeader(
+        message=docstring.splitlines()[0].strip() if docstring else path.stem,
+        revision=str(literal("revision")),
+        down_revision=literal("down_revision"),
+        branch_labels=literal("branch_labels"),
+        depends_on=literal("depends_on"),
+        docstring_source=ast.get_source_segment(content, module.body[0]) if docstring else '""""""',
+    )
+
+
+def history_dir(object_type: str | None, object_name: str | None) -> PurePosixPath:
+    """Where a migration's SQL files live, relative to migrations/sql/."""
+    if object_type and object_name:
+        return PurePosixPath("history", OBJECT_DIRS[object_type], object_name)
+    return PurePosixPath("history", OTHER_DIR)
+
+
+def write_sql_files(sql_root: Path, header: RevisionHeader, options: NewOptions) -> SqlFiles:
+    """Create the upgrade (and, if reversible, downgrade) SQL files."""
+    named = options.named_objects()
+    rel_dir = history_dir(*named[0]) if named else history_dir(None, None)
+    stem = f"{datetime.now().strftime('%Y_%m_%d_%H%M')}_{header.revision}_"
+    stem += _slug(header.message)
+    (sql_root / rel_dir).mkdir(parents=True, exist_ok=True)
+
+    upgrade = rel_dir / f"{stem}.up.sql"
+    (sql_root / upgrade).write_text(_sql_file_text(header, "upgrade"))
+    if options.irreversible_reason is not None:
+        return SqlFiles(upgrade=str(upgrade), downgrade=None)
+
+    downgrade = rel_dir / f"{stem}.down.sql"
+    (sql_root / downgrade).write_text(_sql_file_text(header, "downgrade"))
+    return SqlFiles(upgrade=str(upgrade), downgrade=str(downgrade))
+
+
+def render_revision(header: RevisionHeader, files: SqlFiles, irreversible: str | None) -> str:
+    """The revision file for a SQL-first migration."""
+    imports = "IrreversibleMigration, run_sql" if irreversible else "run_sql"
+    marker = ""
+    if irreversible:
+        marker = (
+            "\n# `ch-migrate down` refuses to revert past this migration.\n"
+            f"irreversible = {irreversible!r}\n"
+        )
+        downgrade_body = "    raise IrreversibleMigration(revision, irreversible)"
+    else:
+        downgrade_body = f"    run_sql({files.downgrade!r})"
+
+    return (
+        f"{header.docstring_source}\n\n"
+        f"from ch_migrate import {imports}\n\n"
+        "# revision identifiers\n"
+        f"revision = {header.revision!r}\n"
+        f"down_revision = {header.down_revision!r}\n"
+        f"branch_labels = {header.branch_labels!r}\n"
+        f"depends_on = {header.depends_on!r}\n"
+        f"{marker}\n\n"
+        "def upgrade() -> None:\n"
+        f"    run_sql({files.upgrade!r})\n\n\n"
+        "def downgrade() -> None:\n"
+        f"{downgrade_body}\n"
+    )
+
+
+def _slug(message: str) -> str:
+    return re.sub(r"\W+", "_", message).strip("_").lower()[:40] or "migration"
+
+
+def _sql_file_text(header: RevisionHeader, direction: str) -> str:
+    guidance = {
+        "upgrade": (
+            "-- Write the statements for this change, each ending with a semicolon.\n"
+            "-- They run one at a time, in order. Make each one safe to run twice\n"
+            "-- (IF NOT EXISTS / IF EXISTS), so a failed run can simply be re-run.\n"
+        ),
+        "downgrade": (
+            "-- Statements end with a semicolon and run one at a time, in order.\n"
+            "-- If the change cannot be undone (dropped data does not come back),\n"
+            "-- mark the migration irreversible instead.\n"
+        ),
+    }[direction]
+    return (
+        f"-- {header.message}: {direction}\n"
+        f"-- Revision: {header.revision}\n"
+        "--\n"
+        f"{guidance}"
+        "-- {db} is replaced with the environment's database name.\n\n"
+    )

ch_migrate/bootstrap.py

@@ -0,0 +1,379 @@
+"""
+Bootstrap script for initializing ClickHouse database and users.
+
+Usage:
+    ch-migrate bootstrap <environment>
+    ch-migrate bootstrap <environment> --dry-run
+
+This script:
+1. Connects using admin credentials
+2. Creates the target database (if not exists)
+3. Creates roles for migration, read-only, and dict access
+4. Creates users and assigns roles
+5. Is idempotent - safe to run multiple times
+"""
+
+import re
+import sys
+from pathlib import Path
+from typing import Optional
+
+from ch_migrate import ui
+from ch_migrate.config import get_env_config
+from ch_migrate.secrets import get_secret
+
+# Pattern for valid SQL identifiers (database, user, role names)
+_IDENTIFIER_PATTERN = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
+
+
+def validate_identifier(name: str, label: str) -> str:
+    """
+    Validate a SQL identifier (database, user, role name).
+
+    Args:
+        name: The identifier to validate
+        label: Human-readable label for error messages (e.g., "database name")
+
+    Returns:
+        The validated identifier (unchanged if valid)
+
+    Raises:
+        ValueError: If the identifier contains invalid characters
+    """
+    if not _IDENTIFIER_PATTERN.match(name):
+        raise ValueError(
+            f"Invalid {label}: {name!r}. "
+            f"Must start with letter/underscore and contain only alphanumeric/underscore."
+        )
+    return name
+
+
+def escape_sql_string(s: str) -> str:
+    """
+    Escape a string for use in SQL single-quoted literals.
+
+    Note: This is for string literals (e.g., passwords) only, not identifiers.
+    Use validate_identifier() for database/user/role names.
+    """
+    return s.replace("\\", "\\\\").replace("'", "''")
+
+
+def build_bootstrap_sql(
+    *,
+    project: str,
+    db: str,
+    migration_user: str,
+    migration_password: str,
+    dict_reader_name: Optional[str] = None,
+    dict_reader_password: Optional[str] = None,
+    mcp_user_name: Optional[str] = None,
+    mcp_password: Optional[str] = None,
+) -> str:
+    """
+    Build the bootstrap SQL with optional users.
+
+    Args:
+        project: Project name (used for role prefixes)
+        db: Database name
+        migration_user: Migration user name
+        migration_password: Migration user password
+        dict_reader_name: Optional dict reader user name
+        dict_reader_password: Optional dict reader password
+        mcp_user_name: Optional MCP user name
+        mcp_password: Optional MCP user password
+
+    Returns:
+        Complete SQL string for bootstrap
+
+    Raises:
+        ValueError: If any identifier contains invalid characters
+    """
+    # Validate all identifiers to prevent SQL injection
+    project = validate_identifier(project, "project name")
+    db = validate_identifier(db, "database name")
+    migration_user = validate_identifier(migration_user, "migration user")
+    if dict_reader_name:
+        dict_reader_name = validate_identifier(dict_reader_name, "dict reader user")
+    if mcp_user_name:
+        mcp_user_name = validate_identifier(mcp_user_name, "MCP user")
+
+    lines = [
+        "-- Bootstrap script for ClickHouse database",
+        "-- Safe to run multiple times (idempotent)",
+        "",
+        "-- Create database",
+        f"CREATE DATABASE IF NOT EXISTS {db};",
+        "",
+        "-- =============================================================================",
+        "-- ROLES",
+        "-- =============================================================================",
+        "",
+        "-- Migration role: schema changes and data operations",
+        "-- (Explicit grants instead of ALL for ClickHouse Cloud compatibility)",
+        f"CREATE ROLE IF NOT EXISTS {project}_migration_role;",
+        "",
+        "-- Schema operations (RENAME is part of ALTER)",
+        f"GRANT CREATE TABLE, DROP TABLE, UNDROP TABLE, ALTER ON {db}.* TO {project}_migration_role WITH GRANT OPTION;",
+        f"GRANT CREATE VIEW, DROP VIEW ON {db}.* TO {project}_migration_role WITH GRANT OPTION;",
+        f"GRANT CREATE DICTIONARY, DROP DICTIONARY ON {db}.* TO {project}_migration_role WITH GRANT OPTION;",
+        f"GRANT CREATE FUNCTION, DROP FUNCTION ON *.* TO {project}_migration_role WITH GRANT OPTION;",
+        "",
+        "-- User/role management (for migrations that create RLS users/roles)",
+        "-- CURRENT GRANTS: ClickHouse Cloud's admin cannot grant ALTER/DROP USER on its managed sql-console user",
+        f"GRANT CURRENT GRANTS(CREATE USER, ALTER USER, DROP USER ON *.*) TO {project}_migration_role WITH GRANT OPTION;",
+        f"GRANT CREATE ROLE, ALTER ROLE, DROP ROLE ON *.* TO {project}_migration_role WITH GRANT OPTION;",
+        f"GRANT ROLE ADMIN ON *.* TO {project}_migration_role WITH GRANT OPTION;",
+        "",
+        "-- Row-level security management",
+        f"GRANT ACCESS MANAGEMENT ON {db}.* TO {project}_migration_role WITH GRANT OPTION;",
+        f"GRANT CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES ON {db}.* TO {project}_migration_role WITH GRANT OPTION;",
+        "",
+        "-- Data operations",
+        f"GRANT SELECT, INSERT, DELETE, TRUNCATE, OPTIMIZE ON {db}.* TO {project}_migration_role WITH GRANT OPTION;",
+        "",
+        "-- Introspection (needed by Alembic)",
+        f"GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES ON {db}.* TO {project}_migration_role WITH GRANT OPTION;",
+        "",
+        "-- Temp tables (for zero-downtime migrations with EXCHANGE TABLES)",
+        "-- Note: EXCHANGE TABLES requires DROP on both tables, already granted above",
+        f"GRANT CREATE TEMPORARY TABLE ON *.* TO {project}_migration_role WITH GRANT OPTION;",
+        "",
+        "-- System table access (monitoring migrations, RLS introspection, user/role management)",
+        "-- Use CURRENT GRANTS to avoid failure when admin lacks access to some system tables",
+        "-- (e.g., system.zookeeper on ClickHouse Cloud)",
+        f"GRANT CURRENT GRANTS(SELECT ON system.*) TO {project}_migration_role WITH GRANT OPTION;",
+        "-- ClickHouse Cloud requires explicit grants for individual system tables",
+        f"GRANT SELECT ON system.grants TO {project}_migration_role;",
+    ]
+
+    # MCP readonly role (optional)
+    if mcp_user_name and mcp_password:
+        lines.extend(
+            [
+                "",
+                "-- Read-only role: for MCP tools",
+                f"CREATE ROLE IF NOT EXISTS {project}_readonly_role;",
+                f"GRANT SELECT ON {db}.* TO {project}_readonly_role;",
+                f"GRANT SHOW TABLES ON {db}.* TO {project}_readonly_role;",
+            ]
+        )
+
+    # Dict reader role (optional)
+    if dict_reader_name and dict_reader_password:
+        lines.extend(
+            [
+                "",
+                "-- Dict reader role: for dictionary sources",
+                f"CREATE ROLE IF NOT EXISTS {project}_dict_role;",
+                "-- SELECT grants added per-table when dictionaries are created",
+            ]
+        )
+
+    lines.extend(
+        [
+            "",
+            "-- =============================================================================",
+            "-- USERS",
+            "-- =============================================================================",
+            "",
+            "-- Migration user (required)",
+            f"CREATE USER IF NOT EXISTS {migration_user}",
+            f"IDENTIFIED BY '{escape_sql_string(migration_password)}';",
+            f"GRANT {project}_migration_role TO {migration_user};",
+        ]
+    )
+
+    # MCP user (optional)
+    if mcp_user_name and mcp_password:
+        lines.extend(
+            [
+                "",
+                "-- MCP user: read-only access for AI tools",
+                f"CREATE USER IF NOT EXISTS {mcp_user_name}",
+                f"IDENTIFIED BY '{escape_sql_string(mcp_password)}';",
+                f"GRANT {project}_readonly_role TO {mcp_user_name};",
+            ]
+        )
+
+    # Dict reader user (optional)
+    if dict_reader_name and dict_reader_password:
+        lines.extend(
+            [
+                "",
+                "-- Dict reader user: for dictionary sources",
+                f"CREATE USER IF NOT EXISTS {dict_reader_name}",
+                f"IDENTIFIED BY '{escape_sql_string(dict_reader_password)}';",
+                f"GRANT {project}_dict_role TO {dict_reader_name};",
+            ]
+        )
+
+    return "\n".join(lines)
+
+
+def run_bootstrap(
+    env_name: str,
+    config_path: Optional[Path] = None,
+    dry_run: bool = False,
+    verbose: bool = False,
+) -> None:
+    """
+    Run bootstrap for the specified environment.
+
+    Args:
+        env_name: Environment name (dev, staging, production, etc.)
+        config_path: Path to config.yaml (defaults to ./config.yaml)
+        dry_run: If True, print SQL without executing
+        verbose: If True, print each SQL statement as it executes
+    """
+    if config_path is None:
+        config_path = Path.cwd() / "config.yaml"
+
+    env_config = get_env_config(env_name, config_path)
+
+    # Get SSM paths if configured
+    ssm_config = env_config.get("ssm", {})
+
+    # Get AWS region for SSM lookups (optional, uses AWS default if not set)
+    aws_region = env_config.get("aws_region")
+
+    # Get required credentials
+    admin_password = get_secret(
+        env_name,
+        "admin_password",
+        ssm_path=ssm_config.get("admin_password"),
+        aws_region=aws_region,
+        required=True,
+    )
+
+    migration_password = get_secret(
+        env_name,
+        "migration_password",
+        ssm_path=ssm_config.get("migration_password"),
+        aws_region=aws_region,
+        required=True,
+    )
+    assert admin_password is not None  # required=True ensures this
+    assert migration_password is not None  # required=True ensures this
+
+    # Get optional credentials
+    dict_reader_name = env_config.get("dict_reader_name")
+    dict_reader_password = None
+    if dict_reader_name:
+        dict_reader_password = get_secret(
+            env_name,
+            "dict_reader_password",
+            ssm_path=ssm_config.get("dict_reader_password"),
+            aws_region=aws_region,
+            required=True,
+        )
+
+    mcp_user_name = env_config.get("mcp_user_name")
+    mcp_password = None
+    if mcp_user_name:
+        mcp_password = get_secret(
+            env_name,
+            "mcp_password",
+            ssm_path=ssm_config.get("mcp_password"),
+            aws_region=aws_region,
+            required=True,
+        )
+
+    # Get migration user - support both new 'migration_user' and legacy 'user' field
+    migration_user = env_config.get("migration_user") or env_config.get("user")
+    if not migration_user:
+        raise ValueError(f"migration_user is required in config.yaml for environment '{env_name}'")
+
+    # Build SQL
+    project_config = env_config.get("project")
+    if project_config:
+        project = project_config
+    else:
+        # Derive from database name (e.g., myproject_dev -> myproject)
+        project = env_config["database"].split("_")[0]
+
+    sql = build_bootstrap_sql(
+        project=project,
+        db=env_config["database"],
+        migration_user=migration_user,
+        migration_password=migration_password,
+        dict_reader_name=dict_reader_name,
+        dict_reader_password=dict_reader_password,
+        mcp_user_name=mcp_user_name,
+        mcp_password=mcp_password,
+    )
+
+    if dry_run:
+        ui.step(f"Dry run for {env_name}: the SQL bootstrap would run")
+        # Mask passwords in output
+        masked_sql = sql
+        for pw in [migration_password, dict_reader_password, mcp_password]:
+            if pw:
+                masked_sql = masked_sql.replace(pw, "********")
+        print(masked_sql)
+        return
+
+    host = env_config["host"]
+    admin_user = env_config.get("admin_user", "default")
+    secure = env_config.get("secure", True)
+    port = env_config.get("port", 8443 if secure else 8123)
+
+    ui.step(f"Bootstrapping {env_name}: database {env_config['database']} on {host}, as {admin_user}")
+    ui.detail(f"Migration user: {migration_user}")
+    if dict_reader_name:
+        ui.detail(f"Dict reader: {dict_reader_name}")
+    if mcp_user_name:
+        ui.detail(f"MCP user: {mcp_user_name}")
+
+    import clickhouse_connect
+
+    client = clickhouse_connect.get_client(
+        host=host,
+        port=port,
+        username=admin_user,
+        password=admin_password,
+        secure=secure,
+        interface="https" if secure else "http",
+    )
+
+    # Collect passwords for masking in verbose output
+    passwords_to_mask = [p for p in [migration_password, dict_reader_password, mcp_password] if p]
+
+    # Execute each statement
+    for statement in sql.split(";"):
+        statement = statement.strip()
+        # Strip leading comment lines to get to the actual SQL
+        lines = statement.split("\n")
+        sql_lines = [line for line in lines if not line.strip().startswith("--")]
+        actual_sql = "\n".join(sql_lines).strip()
+        if actual_sql:
+            if verbose:
+                # Mask passwords in output
+                masked = statement
+                for pw in passwords_to_mask:
+                    masked = masked.replace(pw, "********")
+                ui.detail(f"{masked};")
+            client.command(actual_sql)
+
+    ui.success(f"Bootstrap complete for {env_name}.")
+    ui.hint(f"Next: `ch-migrate new {env_name} NAME --table TABLE`, or `ch-migrate up {env_name}`.")
+
+
+def main() -> None:
+    """CLI entrypoint for bootstrap."""
+    if len(sys.argv) < 2:
+        print("Usage: ch-migrate bootstrap <environment> [--dry-run]")
+        print("Example: ch-migrate bootstrap dev")
+        sys.exit(1)
+
+    dry_run = "--dry-run" in sys.argv
+    env_name = sys.argv[1]
+
+    try:
+        run_bootstrap(env_name, dry_run=dry_run)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()