ch-migrate-cli 0.5.0__py3-none-any.whl

ch_migrate/sql.py

@@ -0,0 +1,216 @@
+"""SQL files as migrations: placeholder rendering, statement splitting, execution.
+
+The ClickHouse HTTP interface accepts one statement per request, so a SQL file
+holding several statements is split here and run one statement at a time.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from itertools import chain
+from pathlib import Path
+from typing import Any, Iterator
+
+_PLACEHOLDER = re.compile(r"\{(\w+)\}")
+_HEREDOC_OPEN = re.compile(r"\$(\w*)\$")
+
+
+@dataclass(frozen=True)
+class Statement:
+    """One statement from a SQL file.
+
+    Attributes:
+        sql: The statement text, without its trailing semicolon or the comments
+            above it.
+        line: 1-based line in the file where the statement text begins.
+        comments: Comment lines directly above the statement, without their
+            comment markers. Waivers are written here.
+    """
+
+    sql: str
+    line: int
+    comments: tuple[str, ...] = ()
+
+
+class SqlStatementError(RuntimeError):
+    """A statement in a SQL migration file failed.
+
+    The message names the file, the statement and its line, then the first line
+    of the database's own error, so `ch-migrate up` can show it without a traceback.
+    """
+
+
+
+def run_sql(path: str, **values: Any) -> None:
+    """Run every statement in a SQL file under migrations/sql/, one at a time.
+
+    `{db}`, `{cluster}` and `{on_cluster}` are filled in from the environment;
+    keyword arguments add to or override them. Other braces are left alone.
+
+    Example:
+        run_sql("history/tables/logs/2026_10_02_1430_a1b2c3_add_status.up.sql")
+    """
+    from alembic import context, op
+
+    statements = load_statements(path, **values)
+    if not statements:
+        # An empty file is almost always a migration someone forgot to fill in.
+        # Applying it would record the revision as done with nothing changed.
+        raise ValueError(f"SQL file has no statements: migrations/sql/{path}")
+    if context.is_offline_mode():
+        for statement in statements:
+            op.execute(statement.sql.replace(":", r"\:"))
+    else:
+        connection = op.get_bind()
+        for index, statement in enumerate(statements, start=1):
+            try:
+                connection.exec_driver_sql(statement.sql.replace("%", "%%"))
+            except Exception as exc:
+                where = f"statement {index} of {len(statements)}, line {statement.line}"
+                raise SqlStatementError(
+                    f"migrations/sql/{path} ({where}): {_database_reason(exc)}"
+                ) from exc
+
+
+def load_statements(path: str, **values: Any) -> list[Statement]:
+    """Read, render and split a SQL file under migrations/sql/."""
+    sql_path = Path.cwd() / "migrations" / "sql" / path
+    if not sql_path.exists():
+        raise FileNotFoundError(f"SQL file not found: {sql_path}")
+    return split_statements(render_placeholders(sql_path.read_text(), default_values(values)))
+
+
+def default_values(overrides: dict[str, Any]) -> dict[str, Any]:
+    """Placeholder values from the environment, with explicit overrides applied."""
+    from ch_migrate.helpers import get_cluster, get_db, on_cluster
+
+    values: dict[str, Any] = {
+        "db": get_db(),
+        "cluster": get_cluster() or "",
+        "on_cluster": on_cluster(),
+    }
+    values.update(overrides)
+    return values
+
+
+def render_placeholders(sql: str, values: dict[str, Any]) -> str:
+    """Replace `{name}` for each known name; leave every other brace as written."""
+
+    def replace(match: re.Match[str]) -> str:
+        name = match.group(1)
+        return str(values[name]) if name in values else match.group(0)
+
+    return _PLACEHOLDER.sub(replace, sql)
+
+
+def split_statements(sql: str) -> list[Statement]:
+    """Split SQL text into statements on semicolons outside quotes and comments."""
+    statements: list[Statement] = []
+    start, line = 0, 1
+    for end in chain(_statement_ends(sql), (len(sql),)):
+        chunk = sql[start:end]
+        offset, comments = _leading_comments(chunk)
+        text = chunk[offset:].strip()
+        if text:
+            statements.append(Statement(text, line + chunk.count("\n", 0, offset), tuple(comments)))
+        line += sql.count("\n", start, end + 1)
+        start = end + 1
+    return statements
+
+
+def clean_database_error(message: str) -> str:
+    """Drop the driver's "Orig exception:" wrapper and the trailing server version."""
+    message = re.sub(r"^Orig exception:\s*", "", message.strip())
+    return re.sub(r"\s*\(version \d[^()]*(?:\([^()]*\))?\)\s*$", "", message)
+
+
+def _database_reason(exc: Exception) -> str:
+    """First line of a driver error, without the wrapper and server-version noise."""
+    lines = str(exc).strip().splitlines()
+    return clean_database_error(lines[0]) if lines else type(exc).__name__
+
+
+def _statement_ends(sql: str) -> Iterator[int]:
+    """Offsets of the semicolons that end statements."""
+    i = 0
+    while i < len(sql):
+        skip_to = _skip_quoted_or_comment(sql, i)
+        if skip_to is not None:
+            i = skip_to
+            continue
+        if sql[i] == ";":
+            yield i
+        i += 1
+
+
+def _skip_quoted_or_comment(sql: str, i: int) -> int | None:
+    """If a string, quoted identifier, comment or heredoc starts at i, return its end."""
+    char = sql[i]
+    if char in "'\"`":
+        return _end_of_quoted(sql, i, char)
+    if sql.startswith("--", i) or sql.startswith("#!", i) or sql.startswith("# ", i):
+        newline = sql.find("\n", i)
+        return len(sql) if newline == -1 else newline + 1
+    if sql.startswith("/*", i):
+        close = sql.find("*/", i + 2)
+        return len(sql) if close == -1 else close + 2
+    if char == "$":
+        match = _HEREDOC_OPEN.match(sql, i)
+        if match:
+            close = sql.find(match.group(0), match.end())
+            return len(sql) if close == -1 else close + len(match.group(0))
+    return None
+
+
+def _end_of_quoted(sql: str, i: int, quote: str) -> int:
+    """End offset (exclusive) of a quoted run, honouring backslash and doubled quotes."""
+    j = i + 1
+    while j < len(sql):
+        if sql[j] == "\\":
+            j += 2
+            continue
+        if sql[j] == quote:
+            if sql.startswith(quote * 2, j):
+                j += 2
+                continue
+            return j + 1
+        j += 1
+    return len(sql)
+
+
+def _leading_comments(chunk: str) -> tuple[int, list[str]]:
+    """Skip whitespace and comments at the start of chunk.
+
+    Returns the offset of the first SQL character and the comment texts seen,
+    keeping only the comments in the contiguous block directly above the SQL.
+    """
+    comments: list[str] = []
+    i = 0
+    at_line_start = True  # a line comment consumes its own newline
+    while i < len(chunk):
+        run_end = i
+        while run_end < len(chunk) and chunk[run_end].isspace():
+            run_end += 1
+        newlines = chunk.count("\n", i, run_end)
+        if newlines >= (1 if at_line_start else 2):
+            comments = []  # a blank line separates a comment block from the SQL
+        i = run_end
+        end = _skip_quoted_or_comment(chunk, i) if i < len(chunk) else None
+        if end is None or chunk[i] in "'\"`$":
+            break
+        comments.extend(_comment_text(chunk[i:end]).splitlines())
+        at_line_start = chunk[end - 1] == "\n"
+        i = end
+    return i, comments
+
+
+def _comment_text(comment: str) -> str:
+    """Strip comment markers from one comment."""
+    text = comment.strip()
+    for marker in ("--", "#!", "#"):
+        if text.startswith(marker):
+            return text[len(marker) :].strip()
+    if text.startswith("/*"):
+        return text[2:-2].strip() if text.endswith("*/") else text[2:].strip()
+    return text

ch_migrate/statements.py

@@ -0,0 +1,144 @@
+"""Extract migration SQL without importing or executing revision modules."""
+
+from __future__ import annotations
+
+import ast
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from ch_migrate.rebase import RevisionGraph
+from ch_migrate.sql import split_statements
+
+MigrationDirection = Literal["upgrade", "downgrade"]
+_FSTRING_POSITIONS = sys.version_info >= (3, 12)
+
+
+@dataclass(frozen=True)
+class MigrationStatement:
+    sql: str
+    source: str
+    line: int
+    comments: tuple[str, ...]
+    direction: MigrationDirection
+
+
+def migration_statements(path: Path) -> list[MigrationStatement]:
+    """Collect literal SQL in upgrade/downgrade bodies, preserving its source."""
+    content = path.read_text()
+    root = path.parent.parent
+    if root.name == "migrations":
+        root = root.parent
+    source = _Source(path, root, content, content.splitlines())
+    result: list[MigrationStatement] = []
+    for function in ast.parse(content).body:
+        if not isinstance(function, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            continue
+        if function.name not in ("upgrade", "downgrade"):
+            continue
+        direction: MigrationDirection = "upgrade" if function.name == "upgrade" else "downgrade"
+        calls = sorted(
+            (node for node in ast.walk(function) if isinstance(node, ast.Call)),
+            key=lambda node: (node.lineno, node.col_offset),
+        )
+        for call in calls:
+            if _call_name(call) in ("read_sql", "run_sql"):
+                result.extend(_file_statements(call, source, direction))
+            elif _call_name(call) == "op.execute":
+                result.extend(_inline_statements(call, source, direction))
+    return result
+
+
+def pending_revisions(graph: RevisionGraph, heads: set[str]) -> set[str]:
+    """Subtract applied heads and their ancestors, as status resolves them."""
+    applied: set[str] = set()
+    unknown = heads - graph.migrations.keys()
+    if unknown:
+        raise ValueError("Database revisions are missing locally: " + ", ".join(sorted(unknown)))
+    for head in heads:
+        applied.update(graph.walk_to_root(head))
+    return graph.migrations.keys() - applied
+
+
+@dataclass(frozen=True)
+class _Source:
+    path: Path
+    root: Path
+    content: str
+    lines: list[str]
+
+
+def _call_name(call: ast.Call) -> str:
+    if isinstance(call.func, ast.Name):
+        return call.func.id
+    if isinstance(call.func, ast.Attribute) and isinstance(call.func.value, ast.Name):
+        if call.func.attr in ("run_sql", "read_sql"):
+            return call.func.attr
+        return f"{call.func.value.id}.{call.func.attr}"
+    return ""
+
+
+def _file_statements(
+    call: ast.Call, source: _Source, direction: MigrationDirection
+) -> list[MigrationStatement]:
+    if not call.args or not isinstance(call.args[0], ast.Constant):
+        return []
+    path_value = call.args[0].value
+    if not isinstance(path_value, str):
+        return []
+    sql_path = source.path.parent.parent / "sql" / path_value
+    statements = split_statements(sql_path.read_text())
+    filename = sql_path.relative_to(source.root).as_posix()
+    return [
+        MigrationStatement(item.sql, filename, item.line, item.comments, direction)
+        for item in statements
+    ]
+
+
+def _inline_statements(
+    call: ast.Call, source: _Source, direction: MigrationDirection
+) -> list[MigrationStatement]:
+    if not call.args:
+        return []
+    text = _literal_sql(call.args[0], source.content)
+    if text is None:
+        return []
+    comments = _python_comments(source.lines, call.lineno)
+    filename = source.path.relative_to(source.root).as_posix()
+    result = []
+    for index, item in enumerate(split_statements(text)):
+        # Inline expressions point to their AST call site, including escaped-newline strings.
+        attached = comments + item.comments if index == 0 else item.comments
+        result.append(MigrationStatement(item.sql, filename, call.lineno, attached, direction))
+    return result
+
+
+def _literal_sql(node: ast.AST, content: str) -> str | None:
+    if isinstance(node, ast.Constant) and isinstance(node.value, str):
+        return node.value
+    if not isinstance(node, ast.JoinedStr):
+        return None
+    pieces = []
+    for value in node.values:
+        if isinstance(value, ast.Constant):
+            pieces.append(value.value)
+        elif isinstance(value, ast.FormattedValue):
+            # Before 3.12, f-string expression nodes carry wrong source positions.
+            segment = ast.get_source_segment(content, value.value) if _FSTRING_POSITIONS else None
+            expression = segment or ast.unparse(value.value)
+            conversion = "" if value.conversion == -1 else "!" + chr(value.conversion)
+            format_spec = ""
+            if value.format_spec is not None:
+                format_spec = ":" + (_literal_sql(value.format_spec, content) or "")
+            pieces.append("{" + expression + conversion + format_spec + "}")
+    return "".join(pieces)
+
+
+def _python_comments(lines: list[str], lineno: int) -> tuple[str, ...]:
+    comments = []
+    index = lineno - 2
+    while index >= 0 and lines[index].lstrip().startswith("#"):
+        comments.append(lines[index].lstrip()[1:].strip())
+        index -= 1
+    return tuple(reversed(comments))

ch_migrate/templates/bootstrap/init_users.sql

@@ -0,0 +1,56 @@
+-- Bootstrap SQL Reference
+-- This file documents the SQL structure created by `ch-migrate bootstrap`
+-- The actual SQL is generated dynamically by bootstrap.py
+--
+-- IMPORTANT: This file is NOT executed directly. It serves as documentation.
+-- To run bootstrap, use: ch-migrate bootstrap <environment>
+
+-- =============================================================================
+-- DATABASE
+-- =============================================================================
+
+CREATE DATABASE IF NOT EXISTS {db};
+
+-- =============================================================================
+-- ROLES (always created)
+-- =============================================================================
+
+-- Migration role: full access for schema changes and data operations
+CREATE ROLE IF NOT EXISTS {project}_migration_role;
+GRANT ALL ON {db}.* TO {project}_migration_role;
+GRANT CREATE TEMPORARY TABLE ON *.* TO {project}_migration_role;
+GRANT CURRENT GRANTS(SELECT ON system.*) TO {project}_migration_role WITH GRANT OPTION;
+-- ClickHouse Cloud requires explicit grants for individual system tables.
+GRANT SELECT ON system.grants TO {project}_migration_role;
+
+-- =============================================================================
+-- USERS (always created)
+-- =============================================================================
+
+-- Migration user
+CREATE USER IF NOT EXISTS {migration_user}
+IDENTIFIED BY '{migration_password}';
+GRANT {project}_migration_role TO {migration_user};
+
+-- =============================================================================
+-- OPTIONAL: MCP User (only if mcp_user_name configured in config.yaml)
+-- =============================================================================
+
+-- CREATE ROLE IF NOT EXISTS {project}_readonly_role;
+-- GRANT SELECT ON {db}.* TO {project}_readonly_role;
+-- GRANT SHOW TABLES ON {db}.* TO {project}_readonly_role;
+--
+-- CREATE USER IF NOT EXISTS {mcp_user_name}
+-- IDENTIFIED BY '{mcp_password}';
+-- GRANT {project}_readonly_role TO {mcp_user_name};
+
+-- =============================================================================
+-- OPTIONAL: Dict Reader (only if dict_reader_name configured in config.yaml)
+-- =============================================================================
+
+-- CREATE ROLE IF NOT EXISTS {project}_dict_role;
+-- SELECT grants added per-table when dictionaries are created
+--
+-- CREATE USER IF NOT EXISTS {dict_reader_name}
+-- IDENTIFIED BY '{dict_reader_password}';
+-- GRANT {project}_dict_role TO {dict_reader_name};

ch_migrate/templates/project/alembic.ini.template

@@ -0,0 +1,43 @@
+# ClickHouse Alembic Configuration
+# Generated by ch-migrate init
+
+[alembic]
+script_location = migrations
+prepend_sys_path = .
+version_path_separator = os
+file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d_%%(slug)s
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

ch_migrate/templates/project/config.yaml.template

@@ -0,0 +1,55 @@
+# ClickHouse Migration Configuration
+# Generated by ch-migrate init
+
+project:
+  name: {project_name}
+
+# Default settings inherited by all environments
+defaults:
+  port: 8443
+  secure: true
+  admin_user: default
+  # Optional: cluster name for ON CLUSTER DDL (self-hosted deployments)
+  # cluster: my_cluster
+  # Optional: dict_reader for dictionary sources
+  # dict_reader_name: dict_reader
+  # Optional: mcp_user for read-only MCP tool access
+  # mcp_user_name: mcp_reader
+
+# Environment-specific configuration
+# Passwords loaded from:
+#   - SSM directly (if ssm: paths configured below)
+#   - .env.local / environment variables (otherwise)
+environments:
+  dev:
+    host: your-dev-instance.clickhouse.cloud
+    database: {project_name}_dev
+    migration_user: migration_dev
+    # Optional: AWS region for SSM lookups (uses AWS default if not set)
+    # aws_region: us-east-1
+    # Optional SSM paths (if set, fetches from SSM directly)
+    # Supports JSON key extraction: /path/to/param#json_key
+    # ssm:
+    #   admin_password: /{project_name}/dev/admin_password
+    #   migration_password: /{project_name}/dev/credentials#password
+    #   dict_reader_password: /{project_name}/dev/dict_reader_password
+    #   mcp_password: /{project_name}/dev/mcp_password
+
+  staging:
+    host: your-staging-instance.clickhouse.cloud
+    database: {project_name}_staging
+    migration_user: migration_staging
+
+  production:
+    host: your-prod-instance.clickhouse.cloud
+    database: {project_name}
+    migration_user: migration_prod
+
+# Optional: Pre/post migration hooks
+# SQL statements executed before/after each migration run.
+# Use {db} placeholder for the database name.
+# hooks:
+#   pre_migrate:
+#     - "SELECT 1"  # validation query
+#   post_migrate:
+#     - "SYSTEM RELOAD DICTIONARY {db}.dict_regions ON CLUSTER default"

ch_migrate/templates/project/env.local.example.template

@@ -0,0 +1,25 @@
+# ClickHouse credentials - DO NOT COMMIT THIS FILE
+# Copy to .env.local and fill in your passwords
+#
+# Alternative: Use SSM paths in config.yaml (fetches directly from SSM)
+# See: https://github.com/DRYCodeWorks/ch-migrate#secrets
+
+# Dev environment
+CH_DEV_MIGRATION_PASSWORD=your-dev-migration-password
+CH_DEV_ADMIN_PASSWORD=your-dev-admin-password
+# Optional: only needed if dict_reader_name is configured
+# CH_DEV_DICT_READER_PASSWORD=your-dev-dict-reader-password
+# Optional: only needed if mcp_user_name is configured
+# CH_DEV_MCP_PASSWORD=your-dev-mcp-password
+
+# Staging environment
+CH_STAGING_MIGRATION_PASSWORD=your-staging-migration-password
+CH_STAGING_ADMIN_PASSWORD=your-staging-admin-password
+# CH_STAGING_DICT_READER_PASSWORD=your-staging-dict-reader-password
+# CH_STAGING_MCP_PASSWORD=your-staging-mcp-password
+
+# Production environment
+CH_PRODUCTION_MIGRATION_PASSWORD=your-prod-migration-password
+CH_PRODUCTION_ADMIN_PASSWORD=your-prod-admin-password
+# CH_PRODUCTION_DICT_READER_PASSWORD=your-prod-dict-reader-password
+# CH_PRODUCTION_MCP_PASSWORD=your-prod-mcp-password

ch_migrate/templates/project/script.py.mako.template

@@ -0,0 +1,30 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+
+from alembic import op
+
+from ch_migrate import get_db, read_sql
+
+# revision identifiers
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    db = get_db()
+    # TODO: Add your upgrade SQL here
+    # op.execute(read_sql("history/tables/my_table/001_{}.sql".format(revision), db=db))
+    pass
+
+
+def downgrade() -> None:
+    db = get_db()
+    # TODO: Add your downgrade SQL here
+    # op.execute(f"DROP TABLE IF EXISTS {db}.my_table")
+    pass

ch_migrate/ui.py

@@ -0,0 +1,79 @@
+"""One output style for every command: steps, results, warnings and errors.
+
+Each line starts with a fixed marker (→ step, ✓ done, ! warning, ✗ error) so the
+output reads the same in a terminal, a CI log or a pipe. Rich drops colour when
+output is not a terminal or NO_COLOR is set; the markers and words stay.
+
+Text between backticks in steps, results and hints is shown as a command,
+highlighted in a terminal, with the backticks removed. Errors, warnings and
+details print exactly as written, because they often carry database messages
+that use backticks for identifiers. Nothing is parsed as Rich markup.
+"""
+
+from __future__ import annotations
+
+import sys
+from typing import NoReturn
+
+from rich.console import Console
+from rich.text import Text
+
+# soft_wrap: never hard-wrap at the terminal width, so paths and SQL stay greppable.
+out = Console(highlight=False, soft_wrap=True)
+err = Console(stderr=True, highlight=False, soft_wrap=True)
+
+_COMMAND_STYLE = "bold cyan"
+
+
+def step(message: str) -> None:
+    """Something is starting or in progress."""
+    out.print(_marked("→ ", "cyan", message))
+
+
+def success(message: str) -> None:
+    """Something finished as intended."""
+    out.print(_marked("✓ ", "green", message))
+
+
+def detail(message: str, *, stderr: bool = False) -> None:
+    """An indented line under the step, result or error above it."""
+    (err if stderr else out).print(Text("  " + message, style="dim"))
+
+
+def hint(message: str, *, stderr: bool = False) -> None:
+    """What to do next; commands in backticks are highlighted."""
+    (err if stderr else out).print(_inline(message))
+
+
+def warn(message: str) -> None:
+    """Worth reading, but the command carries on."""
+    err.print(Text.assemble(("! ", "yellow"), (message, "yellow")))
+
+
+def error(message: str) -> None:
+    """The command could not do what was asked."""
+    err.print(Text.assemble(("✗ ", "bold red"), message))
+
+
+def fail(message: str, *hints: str) -> NoReturn:
+    """Print an error, then any hints, and exit with status 1."""
+    error(message)
+    for line in hints:
+        hint(line, stderr=True)
+    sys.exit(1)
+
+
+def _marked(marker: str, marker_style: str, message: str) -> Text:
+    # Text(marker, style=...) would make the style the base for the whole line.
+    text = Text()
+    text.append(marker, style=marker_style)
+    text.append_text(_inline(message))
+    return text
+
+
+def _inline(message: str) -> Text:
+    """Render `command` spans highlighted, everything else as plain text."""
+    text = Text()
+    for index, part in enumerate(message.split("`")):
+        text.append(part, style=_COMMAND_STYLE if index % 2 else "")
+    return text