ch-migrate-cli 0.5.0__py3-none-any.whl

ch_migrate/scaffold.py

@@ -0,0 +1,253 @@
+"""EXCHANGE TABLES migration scaffold generation."""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+
+def fetch_current_ddl(env_config: dict[str, Any], table_name: str) -> str | None:
+    """Fetch current CREATE TABLE statement from the live database.
+
+    Args:
+        env_config: Environment config dict from get_env_config().
+        table_name: Table name to inspect.
+
+    Returns:
+        DDL string or None if connection fails or table doesn't exist.
+    """
+    from ch_migrate.connection import get_client
+
+    try:
+        client = get_client(env_config)
+        db = env_config["database"]
+        result = client.query(f"SHOW CREATE TABLE {db}.{table_name}")
+        if result.result_rows:
+            return result.result_rows[0][0]
+    except Exception:
+        return None
+    return None
+
+
+def find_dependent_dictionaries(
+    env_config: dict[str, Any], table_name: str
+) -> list[str]:
+    """Find dictionaries that use this table as a source.
+
+    Queries system.dictionaries to find any dictionary whose source
+    references the given table.
+
+    Args:
+        env_config: Environment config dict from get_env_config().
+        table_name: Table name to check.
+
+    Returns:
+        List of dictionary names that depend on this table.
+    """
+    from ch_migrate.connection import get_client
+
+    try:
+        client = get_client(env_config)
+        db = env_config["database"]
+        result = client.query(
+            "SELECT name FROM system.dictionaries "
+            "WHERE database = {db:String} "
+            "AND (source LIKE {exact:String} OR source LIKE {dotted:String})",
+            parameters={
+                "db": db,
+                "exact": f"%'{table_name}'%",
+                "dotted": f"%.{table_name}%",
+            },
+        )
+        return [row[0] for row in result.result_rows]
+    except Exception:
+        return []
+
+
+def _make_shadow_ddl(ddl: str, table_name: str) -> str:
+    """Transform a CREATE TABLE statement into a shadow table version.
+
+    Replaces the table name with <table>_shadow and adds IF NOT EXISTS.
+    """
+    # Replace table name (handles db.table and just table patterns)
+    shadow = re.sub(
+        rf"(CREATE\s+TABLE\s+)(\S+\.)?{re.escape(table_name)}\b",
+        rf"\g<1>\g<2>{table_name}_shadow",
+        ddl,
+        count=1,
+        flags=re.IGNORECASE,
+    )
+    # Add IF NOT EXISTS if not present
+    if "IF NOT EXISTS" not in shadow.upper():
+        shadow = re.sub(
+            r"(CREATE\s+TABLE\s+)",
+            r"\1IF NOT EXISTS ",
+            shadow,
+            count=1,
+            flags=re.IGNORECASE,
+        )
+    return shadow
+
+
+def generate_exchange_sql(
+    table_name: str, current_ddl: str | None = None
+) -> str:
+    """Generate the SQL file content for an EXCHANGE TABLES migration.
+
+    This creates the shadow table DDL that the user should modify with
+    their desired schema changes before running the migration.
+
+    Args:
+        table_name: Name of the table being altered.
+        current_ddl: Current CREATE TABLE DDL from the database, if available.
+
+    Returns:
+        SQL file content for the shadow table creation.
+    """
+    if current_ddl:
+        # read_sql() runs str.format on this file; keep engine macros such as {uuid} and
+        # {replica} (Replicated/Shared engines, ClickHouse Cloud) literal.
+        shadow_ddl = _make_shadow_ddl(current_ddl, table_name).replace("{", "{{").replace("}", "}}")
+        return (
+            f"-- Shadow table for EXCHANGE TABLES migration\n"
+            f"-- Modify this schema with your desired changes.\n"
+            f"--\n"
+            f"-- Original DDL fetched from live database.\n"
+            f"-- The migration will:\n"
+            f"--   1. Create this shadow table\n"
+            f"--   2. Copy data from {table_name} into it\n"
+            f"--   3. Atomically swap via EXCHANGE TABLES\n"
+            f"--   4. Drop the old table\n\n"
+            f"{shadow_ddl}\n"
+        )
+
+    # Placeholder when no live DDL is available
+    return (
+        f"-- Shadow table for EXCHANGE TABLES migration\n"
+        f"-- Replace this placeholder with your desired schema.\n"
+        f"--\n"
+        f"-- TIP: Run `clickhouse-client --query 'SHOW CREATE TABLE {{db}}.{table_name}'`\n"
+        f"--      to get the current schema, then modify it here.\n\n"
+        f"CREATE TABLE IF NOT EXISTS {{db}}.{table_name}_shadow\n"
+        f"(\n"
+        f"    -- TODO: Define columns here\n"
+        f")\n"
+        f"ENGINE = MergeTree\n"
+        f"ORDER BY tuple()\n"
+    )
+
+
+def generate_exchange_migration(
+    revision: str,
+    down_revision: str | None,
+    message: str,
+    table_name: str,
+    sql_path: str,
+    dict_names: list[str] | None = None,
+) -> str:
+    """Generate migration .py content with the EXCHANGE TABLES pattern.
+
+    Args:
+        revision: Alembic revision ID.
+        down_revision: Previous revision ID.
+        message: Migration description.
+        table_name: Table being exchanged.
+        sql_path: Relative path to the SQL history file (from migrations/sql/).
+        dict_names: Dictionaries to reload after exchange, if any.
+
+    Returns:
+        Complete migration .py file content.
+    """
+    down_repr = repr(down_revision)
+    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    dict_lines = ""
+    if dict_names:
+        dict_lines = "\n    # Reload dependent dictionaries\n"
+        for d in dict_names:
+            dict_lines += f'    op.execute("SYSTEM RELOAD DICTIONARY {{db}}.{d}")\n'
+
+    return f'''"""{message}
+
+Revision ID: {revision}
+Revises: {down_revision or "None"}
+Create Date: {now}
+
+EXCHANGE TABLES migration for: {table_name}
+Steps: CREATE shadow -> INSERT SELECT -> EXCHANGE -> DROP
+"""
+
+from alembic import op
+
+from ch_migrate import IrreversibleMigration, get_db, read_sql
+
+# revision identifiers
+revision = {repr(revision)}
+down_revision = {down_repr}
+branch_labels = None
+depends_on = None
+irreversible = (
+    "EXCHANGE TABLES drops the old table; its data cannot be restored automatically."
+)
+
+
+def upgrade() -> None:
+    db = get_db()
+
+    # 1. Create shadow table with new schema
+    op.execute(read_sql("{sql_path}", db=db))
+
+    # 2. Copy data from original table into shadow
+    #    NOTE: Modify the SELECT if columns changed (added/removed/renamed)
+    op.execute(f"INSERT INTO {{db}}.{table_name}_shadow SELECT * FROM {{db}}.{table_name}")
+
+    # 3. Atomically swap tables
+    op.execute(f"EXCHANGE TABLES {{db}}.{table_name} AND {{db}}.{table_name}_shadow")
+
+    # 4. Drop old table (now named {table_name}_shadow)
+    op.execute(f"DROP TABLE IF EXISTS {{db}}.{table_name}_shadow")
+{dict_lines}
+
+def downgrade() -> None:
+    raise IrreversibleMigration(revision, irreversible)
+'''
+
+
+def rewrite_migration_file(
+    migration_path: Path,
+    table_name: str,
+    sql_path: str,
+    dict_names: list[str] | None = None,
+) -> None:
+    """Rewrite an alembic-generated migration file with EXCHANGE pattern.
+
+    Reads the revision and down_revision from the existing file, then
+    overwrites it with the EXCHANGE TABLES template.
+
+    Args:
+        migration_path: Path to the generated migration .py file.
+        table_name: Table being exchanged.
+        sql_path: Relative path to the SQL history file.
+        dict_names: Dictionaries to reload after exchange, if any.
+    """
+    content = migration_path.read_text()
+
+    rev_match = re.search(r'revision\s*=\s*["\'](\w+)["\']', content)
+    down_match = re.search(r'down_revision\s*=\s*["\'](\w+)["\']', content)
+    msg_match = re.search(r'^"""(.+?)$', content, re.MULTILINE)
+
+    revision = rev_match.group(1) if rev_match else "UNKNOWN"
+    down_revision = down_match.group(1) if down_match else None
+    message = msg_match.group(1) if msg_match else table_name
+
+    new_content = generate_exchange_migration(
+        revision=revision,
+        down_revision=down_revision,
+        message=message,
+        table_name=table_name,
+        sql_path=sql_path,
+        dict_names=dict_names,
+    )
+    migration_path.write_text(new_content)

ch_migrate/secrets.py

@@ -0,0 +1,162 @@
+"""Secrets management with environment variable and SSM support."""
+
+import json
+import os
+from typing import Optional, Union
+
+
+class SSMSecretNotFoundError(Exception):
+    """Raised when an SSM parameter cannot be found."""
+
+    pass
+
+
+class SSMJsonKeyError(Exception):
+    """Raised when a JSON key cannot be found in an SSM parameter value."""
+
+    pass
+
+
+def _get_ssm_client(region: Optional[str] = None):  # type: ignore[no-untyped-def]
+    """Get boto3 SSM client, raising helpful error if boto3 not installed.
+
+    Args:
+        region: Optional AWS region name (e.g., 'us-east-1'). If not provided,
+                uses AWS_REGION environment variable or default from AWS config.
+    """
+    try:
+        import boto3  # type: ignore[import-not-found,import-untyped]
+    except ImportError:
+        raise ImportError(
+            "boto3 is required for SSM support. "
+            "Install with: pip install ch-migrate-cli[ssm]"
+        )
+    if region:
+        return boto3.client("ssm", region_name=region)
+    return boto3.client("ssm")
+
+
+def _get_ssm_exceptions():  # type: ignore[no-untyped-def]
+    """Get boto3 SSM exception classes."""
+    try:
+        from botocore.exceptions import ClientError  # type: ignore[import-not-found,import-untyped]
+
+        return ClientError
+    except ImportError:
+        # Fallback if botocore not available (shouldn't happen if boto3 is installed)
+        return Exception
+
+
+def _parse_ssm_path(ssm_path: Union[str, dict]) -> tuple[str, Optional[str]]:
+    """
+    Parse SSM path into (path, json_key).
+
+    Supports two formats:
+    - String with hash suffix: "/path/to/param#json_key"
+    - Dict with explicit fields: {"path": "/path/to/param", "json_key": "password"}
+
+    Args:
+        ssm_path: SSM path as string or dict
+
+    Returns:
+        Tuple of (ssm_parameter_path, json_key_or_none)
+    """
+    if isinstance(ssm_path, dict):
+        return ssm_path["path"], ssm_path.get("json_key")
+    elif "#" in ssm_path:
+        path, json_key = ssm_path.rsplit("#", 1)
+        return path, json_key
+    else:
+        return ssm_path, None
+
+
+def get_secret(
+    env_name: str,
+    key: str,
+    *,
+    ssm_path: Optional[Union[str, dict]] = None,
+    aws_region: Optional[str] = None,
+    required: bool = True,
+) -> Optional[str]:
+    """
+    Get a secret value from SSM or environment variable.
+
+    Precedence:
+    1. SSM parameter at ssm_path (if provided) - use SSM directly
+    2. Environment variable CH_{ENV}_{KEY} (e.g., CH_DEV_MIGRATION_PASSWORD)
+    3. Legacy env var for migration_password: CH_{ENV}_PASSWORD
+    4. None (if not required) or raise ValueError
+
+    SSM path formats:
+    - Simple string: "/myproject/dev/password"
+    - With JSON key (hash suffix): "/myproject/credentials#password"
+    - With JSON key (object): {"path": "/myproject/credentials", "json_key": "password"}
+
+    Args:
+        env_name: Environment name (dev, staging, production)
+        key: Secret key (migration_password, admin_password, dict_reader_password)
+        ssm_path: Optional SSM parameter path (string or dict with path/json_key)
+        aws_region: Optional AWS region for SSM lookups (e.g., 'us-east-1')
+        required: Whether to raise if secret not found
+
+    Returns:
+        Secret value or None if not required and not found
+
+    Raises:
+        ValueError: If required and not found in env or SSM
+        SSMSecretNotFoundError: If SSM path provided but parameter not found
+        SSMJsonKeyError: If JSON key specified but not found in parameter value
+        ImportError: If SSM path provided but boto3 not installed
+    """
+    # If SSM path provided, use SSM directly (don't check env vars)
+    if ssm_path:
+        path, json_key = _parse_ssm_path(ssm_path)
+        client = _get_ssm_client(aws_region)
+        ClientError = _get_ssm_exceptions()
+        try:
+            response = client.get_parameter(Name=path, WithDecryption=True)
+            value = response["Parameter"]["Value"]
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "")
+            if error_code == "ParameterNotFound":
+                if required:
+                    raise SSMSecretNotFoundError(f"SSM parameter not found: {path}") from e
+                return None
+            # Re-raise other AWS errors (permission denied, etc.)
+            raise
+
+        # Extract JSON key if specified
+        if json_key:
+            try:
+                data = json.loads(value)
+            except json.JSONDecodeError as e:
+                raise SSMJsonKeyError(
+                    f"SSM parameter '{path}' is not valid JSON (needed for key '{json_key}')"
+                ) from e
+            if json_key not in data:
+                raise SSMJsonKeyError(f"JSON key '{json_key}' not found in SSM parameter '{path}'")
+            return str(data[json_key])
+
+        return value  # type: ignore[no-any-return]
+
+    # No SSM path - use environment variables
+    env_var = f"CH_{env_name.upper()}_{key.upper()}"
+    value = os.environ.get(env_var)
+    if value:
+        return value
+
+    # Legacy support: CH_{ENV}_PASSWORD for migration_password
+    if key == "migration_password":
+        legacy_var = f"CH_{env_name.upper()}_PASSWORD"
+        value = os.environ.get(legacy_var)
+        if value:
+            return value
+
+    # Not found anywhere
+    if required:
+        raise ValueError(
+            f"{env_var} is required. "
+            f"Set it in .env.local or provide an SSM path in config.yaml."
+        )
+
+    return None

ch_migrate/skills/ch-migrate/SKILL.md

@@ -0,0 +1,250 @@
+---
+name: ch-migrate
+description: Use when integrating ClickHouse migrations into a project, setting up ch-migrate, creating migration files, bootstrapping ClickHouse databases, or troubleshooting ClickHouse Alembic issues. Triggers on "ClickHouse migration", "ch-migrate", "Alembic ClickHouse", "bootstrap ClickHouse", "migration user", "EXCHANGE TABLES".
+---
+
+# ch-migrate: ClickHouse Migration Tool
+
+## Overview
+
+`ch-migrate` adds SQL-first authoring, environment configuration, bootstrap,
+inspection, and drift checks above Alembic. It complements ClickHouse's official
+Alembic integration; this development line still uses `clickhouse-sqlalchemy`
+for migration connections. Do not imply an endorsement or promise transactional
+DDL.
+
+**Install:** `uv tool install ch-migrate-cli` or `pip install ch-migrate-cli`.
+The command is `ch-migrate`; the import package is `ch_migrate`.
+
+## CLI Quick Reference
+
+| Command | Description |
+|---------|-------------|
+| `ch-migrate init [PATH] [--name NAME]` | Initialize project structure |
+| `ch-migrate bootstrap ENV [--dry-run]` | Create database, roles, users |
+| `ch-migrate new ENV NAME [--table X] [--irreversible REASON]` | Create upgrade/downgrade SQL and their revision |
+| `ch-migrate up ENV [-r REV]` | Apply migrations (default: head, or to REV) |
+| `ch-migrate down ENV [-r REV]` | Rollback (default: last, or to REV) |
+| `ch-migrate status ENV` | Show current migration state |
+| `ch-migrate history ENV` | Show migration history |
+| `ch-migrate lint [ENV]` | Check upgrade statements; ENV restricts to pending revisions and adds live checks |
+| `ch-migrate deps ENV [--validate PATH]` | Inspect live dependencies |
+| `ch-migrate snapshot ENV [--exclude GLOB] [--filter GLOB]` | Capture schema |
+| `ch-migrate diff ENV [--snapshot-dir PATH]` | Compare snapshot and live schema |
+| `ch-migrate rebase ENV [--onto REV] [--dry-run]` | Preview/rewrite dangling revision branches |
+| `ch-migrate upgrade-env` | Refresh env.py, keeping env.py.bak |
+| `ch-migrate skill [--user\|--project]` | Install Claude skill for ch-migrate |
+
+**Options for `new`:** choose one of `--table NAME`, `--view NAME`, and
+`--dict NAME`. SQL lives under `migrations/sql/history/{tables|views|dictionaries}/NAME/`;
+without an object it goes under `history/other/`. Filenames are
+`<YYYY_MM_DD_HHMM>_<revision>_<slug>.up.sql` and `.down.sql`.
+`--irreversible REASON` omits the down file and installs a static marker plus
+`IrreversibleMigration` backstop. `--python` retains the Python template and
+optional single SQL file. `--exchange --table NAME` retains the exchange
+scaffold. These three modes are mutually exclusive; invalid choices fail
+before Alembic writes a revision.
+
+## Project Structure
+
+```
+project/
+├── config.yaml           # ClickHouse hosts and settings
+├── .env.local            # Secrets (gitignored)
+├── alembic.ini           # Alembic configuration
+└── migrations/
+    ├── env.py            # Alembic environment
+    ├── versions/         # Generated revision adapters; no Python edits needed
+    └── sql/
+        ├── bootstrap/    # Custom bootstrap SQL (optional)
+        └── history/      # Object-centric SQL versions
+            ├── tables/
+            ├── views/
+            ├── dictionaries/
+            └── other/
+```
+
+## Configuration
+
+### config.yaml
+
+```yaml
+project:
+  name: my_project
+
+defaults:
+  port: 8443                    # 8123 for local HTTP
+  secure: true                  # false for local Docker
+  admin_user: default
+  # Optional users (uncomment to enable):
+  # mcp_user_name: mcp_reader   # Read-only for AI tools
+  # dict_reader_name: dict_reader
+
+environments:
+  dev:
+    host: dev.clickhouse.cloud  # or localhost for Docker
+    database: my_project_dev
+    migration_user: migration_dev
+    # aws_region: us-east-1  # Optional: for region-scoped SSM lookups
+    # Optional SSM paths (if set, fetches from SSM directly):
+    # Supports JSON key extraction: /path/to/param#json_key
+    # ssm:
+    #   admin_password: /my_project/dev/admin_password
+    #   migration_password: /my_project/credentials#password
+
+  staging:
+    host: staging.clickhouse.cloud
+    database: my_project_staging
+    migration_user: migration_staging
+
+  production:
+    host: prod.clickhouse.cloud
+    database: my_project
+    migration_user: migration_prod
+```
+
+### .env.local
+
+```bash
+# Required
+CH_DEV_MIGRATION_PASSWORD=your-migration-password
+CH_DEV_ADMIN_PASSWORD=your-admin-password    # For bootstrap only
+
+# Optional (if mcp_user_name configured)
+CH_DEV_MCP_PASSWORD=your-mcp-password
+
+# Repeat for staging/production with appropriate env name
+CH_PRODUCTION_MIGRATION_PASSWORD=prod-password
+CH_PRODUCTION_ADMIN_PASSWORD=prod-admin-password
+```
+
+## Integration Workflow
+
+### SQL-first workflow
+
+Use a dedicated, authorized server. Local HTTP normally uses port `8123` and
+`secure: false`; Cloud normally uses HTTPS `8443`. Do not start or modify shared
+infrastructure as part of trying the tool.
+
+```bash
+ch-migrate init ./schema --name my_project
+cd schema
+# Edit config.yaml and create .env.local here.
+ch-migrate bootstrap dev --dry-run
+ch-migrate bootstrap dev
+ch-migrate new dev add_status --table logs
+```
+
+Fill the generated `.up.sql` with:
+
+```sql
+CREATE TABLE IF NOT EXISTS {db}.logs (id UInt64)
+ENGINE = MergeTree ORDER BY id;
+ALTER TABLE {db}.logs ADD COLUMN IF NOT EXISTS status String;
+```
+
+For an empty test project only, fill `.down.sql` with:
+
+```sql
+DROP TABLE IF EXISTS {db}.logs;
+```
+
+Then run:
+
+```bash
+ch-migrate lint
+ch-migrate up dev
+ch-migrate status dev
+ch-migrate history dev
+ch-migrate down dev
+```
+
+Do not edit the generated Python adapter. Empty/comment-only SQL files fail.
+The example downgrade drops the table and its data; do not apply it to a live
+table that needs preserving.
+
+### SQL execution
+
+`run_sql` splits on semicolons outside strings, identifiers, comments, and
+heredocs, then sends one statement per request. It substitutes `{db}`,
+`{cluster}`, and `{on_cluster}`, plus explicit keyword overrides. Other braces
+remain literal, including JSON and `{id:UInt64}` parameters. Doubled braces
+are not escapes. A failure stops later statements but cannot undo earlier DDL.
+
+For migrations requiring logic:
+
+```bash
+ch-migrate new dev backfill --python
+```
+
+`read_sql` and `get_db` remain available to Python migrations. `read_sql` still
+uses `str.format`, so its literal-brace rules differ from `run_sql`.
+
+### Irreversible changes
+
+```bash
+ch-migrate new dev drop_legacy --table logs --irreversible "Drops legacy data"
+```
+
+`down` statically checks the whole known range before running Alembic and refuses
+if any revision carries an irreversible marker. It understands `-N` on a linear
+chain, full or unique-prefix IDs, and `base`. Unknown ranges fall back to each
+migration's exception; direct Alembic calls also rely on that backstop.
+
+There is no override flag. Implement a real downgrade and remove the marker
+through review to revert past it. Hand-written Python revisions must provide
+both `irreversible = "reason"` and a downgrade that raises
+`IrreversibleMigration(revision, irreversible)`.
+
+### Exchange and dictionary operations
+
+`new --exchange --table NAME` creates the existing copy-and-swap scaffold.
+Pause or coordinate writers: copying and exchanging alone does not preserve
+inserts arriving during the copy. Review the schema and column mapping.
+The generated revision is irreversible because it drops the old table.
+
+For dictionaries, `create_dictionary("history/dictionaries/NAME/file.sql")`
+retains the configured dictionary reader's automatic SELECT grant.
+
+For offline SQL, set `CH_ENVIRONMENT` and run `alembic upgrade head --sql`.
+Existing projects need `ch-migrate upgrade-env` for the offline version-table
+and literal-rendering fixes. Keep credentials out of SQL files and logs.
+
+
+## Roles Created by Bootstrap
+
+| Role | Purpose |
+|------|---------|
+| `{project}_migration_role` | Schema changes (CREATE/DROP/ALTER TABLE/VIEW/DICTIONARY), data ops (SELECT/INSERT/DELETE/TRUNCATE), and introspection including explicit `system.grants` access |
+| `{project}_readonly_role` | SELECT + SHOW (if mcp_user configured) |
+| `{project}_dict_role` | Dictionary source access (if dict_reader configured) |
+
+Note: Bootstrap uses explicit grants (not `GRANT ALL`) for ClickHouse Cloud compatibility.
+
+## Troubleshooting
+
+| Issue | Solution |
+|-------|----------|
+| "migration_user required" | Add `migration_user: username` to environment in config.yaml |
+| "Connection refused" | Check host/port. Local Docker: port 8123, secure: false |
+| Passwords not loading | Ensure .env.local exists in project root (not migrations/) |
+| Bootstrap hangs | Verify admin_password is correct for admin_user |
+| "Database does not exist" | Run `ch-migrate bootstrap ENV` first |
+
+### Verify Configuration
+
+```bash
+# Check what SQL would run
+ch-migrate bootstrap dev --dry-run
+
+# Should show masked passwords like:
+# CREATE USER IF NOT EXISTS migration_dev
+# IDENTIFIED BY '********';
+```
+
+## ClickHouse Cloud Notes
+
+- Uses standard engines (MergeTree, ReplacingMergeTree) - auto-upgraded to Shared* on Cloud
+- Default port 8443 (HTTPS), use 8123 for local HTTP
+- DDL is non-transactional - migrations can't be atomically rolled back
+- Each `op.execute()` runs one statement