chkit-py 0.1.0__py3-none-any.whl

chkit/__init__.py

@@ -0,0 +1,62 @@
+"""chkit — ClickHouse schema and migration toolkit."""
+
+from chkit.core import (
+    ChxResolvedClickHouseConfig,
+    ChxResolvedConfig,
+    ChxUserClickHouseConfig,
+    ChxUserConfig,
+    ChxValidationError,
+    ColumnDefinition,
+    MaterializedViewDefinition,
+    MaterializedViewRefresh,
+    MigrationOperation,
+    MigrationPlan,
+    ProjectionDefinition,
+    SchemaDefinition,
+    TableDefinition,
+    TableRef,
+    ValidationIssue,
+    ViewDefinition,
+    canonicalize_definitions,
+    define_config,
+    materialized_view,
+    plan_diff,
+    resolve_config,
+    schema,
+    table,
+    to_create_sql,
+    validate_definitions,
+    view,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ChxResolvedClickHouseConfig",
+    "ChxResolvedConfig",
+    "ChxUserClickHouseConfig",
+    "ChxUserConfig",
+    "ChxValidationError",
+    "ColumnDefinition",
+    "MaterializedViewDefinition",
+    "MaterializedViewRefresh",
+    "MigrationOperation",
+    "MigrationPlan",
+    "ProjectionDefinition",
+    "SchemaDefinition",
+    "TableDefinition",
+    "TableRef",
+    "ValidationIssue",
+    "ViewDefinition",
+    "__version__",
+    "canonicalize_definitions",
+    "define_config",
+    "materialized_view",
+    "plan_diff",
+    "resolve_config",
+    "schema",
+    "table",
+    "to_create_sql",
+    "validate_definitions",
+    "view",
+]

chkit/cli/__init__.py

@@ -0,0 +1 @@
+"""chkit CLI."""

chkit/cli/commands/__init__.py

@@ -0,0 +1,5 @@
+"""chkit CLI commands."""
+
+from chkit.cli.commands import check, drift, generate, init, migrate, status
+
+__all__ = ["check", "drift", "generate", "init", "migrate", "status"]

chkit/cli/commands/check.py

@@ -0,0 +1,59 @@
+"""`chkit check` — pre-flight gate for CI."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from chkit.cli.config_loader import load_config
+from chkit.cli.migration_store import list_migrations, read_snapshot
+from chkit.cli.schema_loader import load_schema
+from chkit.core.canonical import canonicalize_definitions
+from chkit.core.planner import plan_diff
+from chkit.core.validate import validate_definitions
+
+
+def run(
+    config_path: Annotated[
+        Path | None, typer.Option("--config", "-c", help="Path to chkit.config.py.")
+    ] = None,
+    output_json: Annotated[
+        bool, typer.Option("--json", help="Emit a JSON-formatted summary.")
+    ] = False,
+) -> None:
+    config = load_config(config_path)
+    schema_defs = canonicalize_definitions(load_schema(config.schema_))
+
+    issues = [issue.model_dump(mode="json") for issue in validate_definitions(schema_defs)]
+    pending = [m.stem for m in list_migrations(Path(config.migrations_dir))]
+    snapshot = read_snapshot(Path(config.meta_dir))
+    drift_ops: list[str] = []
+    if snapshot is not None:
+        plan = plan_diff(list(snapshot.definitions), schema_defs)
+        drift_ops = [op.key for op in plan.operations]
+
+    failed = False
+    if config.check.fail_on_pending and pending:
+        failed = True
+    if config.check.fail_on_drift and drift_ops:
+        failed = True
+    if issues:
+        failed = True
+
+    summary = {
+        "ok": not failed,
+        "issues": issues,
+        "pending_migrations": pending,
+        "drift_operations": drift_ops,
+    }
+    if output_json:
+        typer.echo(json.dumps(summary, indent=2))
+    else:
+        typer.echo(f"Validation issues: {len(issues)}")
+        typer.echo(f"Pending migrations: {len(pending)}")
+        typer.echo(f"Drift operations: {len(drift_ops)}")
+    if failed:
+        raise typer.Exit(code=1)

chkit/cli/commands/drift.py

@@ -0,0 +1,63 @@
+"""`chkit drift` — compare last snapshot vs. current schema."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from chkit.cli.config_loader import load_config
+from chkit.cli.migration_store import read_snapshot
+from chkit.cli.schema_loader import load_schema
+from chkit.core.canonical import canonicalize_definitions
+from chkit.core.planner import plan_diff
+
+
+def run(
+    config_path: Annotated[
+        Path | None, typer.Option("--config", "-c", help="Path to chkit.config.py.")
+    ] = None,
+    output_json: Annotated[
+        bool, typer.Option("--json", help="Emit a JSON-formatted summary.")
+    ] = False,
+) -> None:
+    config = load_config(config_path)
+    schema_defs = canonicalize_definitions(load_schema(config.schema_))
+    snapshot = read_snapshot(Path(config.meta_dir))
+    if snapshot is None:
+        typer.echo(
+            "No snapshot found — run `chkit generate` first."
+            if not output_json
+            else json.dumps({"status": "no-snapshot"})
+        )
+        return
+
+    plan = plan_diff(list(snapshot.definitions), schema_defs)
+    if not plan.operations:
+        typer.echo(
+            "No drift detected."
+            if not output_json
+            else json.dumps({"status": "no-drift"})
+        )
+        return
+
+    if output_json:
+        typer.echo(
+            json.dumps(
+                {
+                    "status": "drift",
+                    "operations": [op.model_dump(by_alias=True) for op in plan.operations],
+                    "rename_suggestions": [
+                        s.model_dump(by_alias=True) for s in plan.rename_suggestions
+                    ],
+                },
+                indent=2,
+            )
+        )
+        return
+
+    typer.secho(f"Drift detected: {len(plan.operations)} operation(s)", fg=typer.colors.YELLOW)
+    for op in plan.operations:
+        typer.echo(f"  [{op.risk}] {op.type} {op.key}")

chkit/cli/commands/generate.py

@@ -0,0 +1,79 @@
+"""`chkit generate` — diff current schema vs. last snapshot and emit a migration."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from chkit.cli.config_loader import load_config
+from chkit.cli.migration_store import read_snapshot, write_migration, write_snapshot
+from chkit.cli.schema_loader import load_schema
+from chkit.core.canonical import canonicalize_definitions
+from chkit.core.model import ChxValidationError
+from chkit.core.planner import plan_diff
+from chkit.core.snapshot import create_snapshot
+from chkit.core.validate import validate_definitions
+
+
+def run(
+    config_path: Annotated[
+        Path | None,
+        typer.Option("--config", "-c", help="Path to chkit.config.py."),
+    ] = None,
+    label: Annotated[
+        str,
+        typer.Option("--label", "-l", help="Label suffix appended to the migration id."),
+    ] = "auto",
+    output_json: Annotated[
+        bool,
+        typer.Option("--json", help="Emit a JSON-formatted summary."),
+    ] = False,
+) -> None:
+    config = load_config(config_path)
+    schema_globs = config.schema_
+    definitions = load_schema(schema_globs)
+    canonical = canonicalize_definitions(definitions)
+
+    issues = validate_definitions(canonical)
+    if issues:
+        raise ChxValidationError(issues)
+
+    meta_dir = Path(config.meta_dir)
+    migrations_dir = Path(config.migrations_dir)
+
+    previous = read_snapshot(meta_dir)
+    old_defs = list(previous.definitions) if previous is not None else []
+
+    plan = plan_diff(old_defs, canonical)
+    if not plan.operations:
+        message = {"status": "no-changes"}
+        typer.echo(json.dumps(message) if output_json else "No schema changes detected.")
+        return
+
+    artifact = write_migration(migrations_dir, plan, label=label)
+    snapshot = create_snapshot(canonical)
+    snapshot_path = write_snapshot(meta_dir, snapshot)
+
+    summary = {
+        "status": "generated",
+        "migration_id": artifact.id,
+        "sql_path": str(artifact.sql_path),
+        "snapshot_path": str(snapshot_path),
+        "operations": len(plan.operations),
+        "risk_summary": plan.risk_summary.model_dump(),
+    }
+    if output_json:
+        typer.echo(json.dumps(summary, indent=2))
+        return
+
+    typer.secho(f"Generated migration {artifact.id}", fg=typer.colors.GREEN)
+    typer.echo(f"  SQL:      {artifact.sql_path}")
+    typer.echo(f"  Snapshot: {snapshot_path}")
+    typer.echo(f"  Operations: {len(plan.operations)}")
+    risks = plan.risk_summary
+    typer.echo(
+        f"  Risk: safe={risks.safe} caution={risks.caution} danger={risks.danger}"
+    )

chkit/cli/commands/init.py

@@ -0,0 +1,68 @@
+"""`chkit init` — scaffold a starter project."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+_CONFIG_TEMPLATE = '''"""chkit configuration. Edit and re-run `chkit generate`."""
+
+from chkit import ChxUserConfig
+
+config = ChxUserConfig.model_validate(
+    {
+        "schema": ["./schema/**/*.py"],
+        "outDir": "./chkit",
+        "clickhouse": {
+            "url": "http://localhost:8123",
+            "username": "default",
+            "password": "",
+            "database": "default",
+        },
+    }
+)
+'''
+
+_EXAMPLE_TEMPLATE = '''"""Example chkit schema."""
+
+from chkit import ColumnDefinition, table
+
+events = table(
+    database="default",
+    name="events",
+    engine="MergeTree",
+    columns=[
+        ColumnDefinition(name="ts", type="DateTime"),
+        ColumnDefinition(name="user_id", type="UInt64"),
+        ColumnDefinition(name="event", type="String"),
+    ],
+    primary_key=["ts"],
+    order_by=["ts", "user_id"],
+)
+'''
+
+
+def run(
+    out_dir: Annotated[
+        Path,
+        typer.Option("--out", "-o", help="Project root to scaffold into."),
+    ] = Path("."),
+) -> None:
+    out_dir.mkdir(parents=True, exist_ok=True)
+    config_path = out_dir / "chkit.config.py"
+    example_path = out_dir / "schema" / "events.py"
+
+    if config_path.exists():
+        typer.secho(f"Skipping: {config_path} already exists", fg=typer.colors.YELLOW)
+    else:
+        config_path.write_text(_CONFIG_TEMPLATE, encoding="utf-8")
+        typer.secho(f"Created {config_path}", fg=typer.colors.GREEN)
+
+    example_path.parent.mkdir(parents=True, exist_ok=True)
+    if example_path.exists():
+        typer.secho(f"Skipping: {example_path} already exists", fg=typer.colors.YELLOW)
+    else:
+        example_path.write_text(_EXAMPLE_TEMPLATE, encoding="utf-8")
+        typer.secho(f"Created {example_path}", fg=typer.colors.GREEN)

chkit/cli/commands/migrate.py

@@ -0,0 +1,83 @@
+"""`chkit migrate` — apply pending migrations to the target database."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from chkit.cli.config_loader import load_config
+from chkit.cli.migration_store import list_migrations
+from chkit.clickhouse.client import ClickHouseClient
+from chkit.core.sql_splitter import split_sql_statements
+
+
+def _read_applied(meta_dir: Path) -> set[str]:
+    applied_file = meta_dir / "applied.json"
+    if not applied_file.exists():
+        return set()
+    payload = json.loads(applied_file.read_text(encoding="utf-8"))
+    return {str(item) for item in payload.get("ids", [])}
+
+
+def _write_applied(meta_dir: Path, ids: set[str]) -> None:
+    meta_dir.mkdir(parents=True, exist_ok=True)
+    applied_file = meta_dir / "applied.json"
+    applied_file.write_text(
+        json.dumps({"ids": sorted(ids)}, indent=2), encoding="utf-8"
+    )
+
+
+def run(
+    config_path: Annotated[
+        Path | None, typer.Option("--config", "-c", help="Path to chkit.config.py.")
+    ] = None,
+    dry_run: Annotated[
+        bool, typer.Option("--dry-run", help="Print statements without executing.")
+    ] = False,
+    output_json: Annotated[
+        bool, typer.Option("--json", help="Emit a JSON-formatted summary.")
+    ] = False,
+) -> None:
+    config = load_config(config_path)
+    if config.clickhouse is None:
+        msg = "chkit.config.py must include a `clickhouse` block to migrate."
+        raise typer.BadParameter(msg)
+
+    migrations_dir = Path(config.migrations_dir)
+    meta_dir = Path(config.meta_dir)
+
+    all_migrations = list_migrations(migrations_dir)
+    applied = _read_applied(meta_dir)
+    pending = [m for m in all_migrations if m.stem not in applied]
+
+    if not pending:
+        result = {"status": "up-to-date", "applied": sorted(applied)}
+        typer.echo(json.dumps(result) if output_json else "No pending migrations.")
+        return
+
+    executed: list[str] = []
+    if dry_run:
+        for migration in pending:
+            typer.echo(f"-- {migration.name}")
+            typer.echo(migration.read_text(encoding="utf-8"))
+        return
+
+    with ClickHouseClient.connect(config.clickhouse) as client:
+        for migration in pending:
+            sql_text = migration.read_text(encoding="utf-8")
+            for statement in split_sql_statements(sql_text):
+                client.execute(statement)
+            executed.append(migration.stem)
+            applied.add(migration.stem)
+            _write_applied(meta_dir, applied)
+
+    summary = {"status": "applied", "executed": executed}
+    if output_json:
+        typer.echo(json.dumps(summary, indent=2))
+        return
+    typer.secho(f"Applied {len(executed)} migration(s).", fg=typer.colors.GREEN)
+    for migration_id in executed:
+        typer.echo(f"  - {migration_id}")

chkit/cli/commands/status.py

@@ -0,0 +1,51 @@
+"""`chkit status` — show applied vs. pending migrations."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from chkit.cli.config_loader import load_config
+from chkit.cli.migration_store import list_migrations
+
+
+def _read_applied(meta_dir: Path) -> set[str]:
+    applied_file = meta_dir / "applied.json"
+    if not applied_file.exists():
+        return set()
+    payload = json.loads(applied_file.read_text(encoding="utf-8"))
+    return {str(item) for item in payload.get("ids", [])}
+
+
+def run(
+    config_path: Annotated[
+        Path | None, typer.Option("--config", "-c", help="Path to chkit.config.py.")
+    ] = None,
+    output_json: Annotated[
+        bool, typer.Option("--json", help="Emit a JSON-formatted summary.")
+    ] = False,
+) -> None:
+    config = load_config(config_path)
+    migrations_dir = Path(config.migrations_dir)
+    meta_dir = Path(config.meta_dir)
+    all_migrations = list_migrations(migrations_dir)
+    applied = _read_applied(meta_dir)
+    pending = [m.stem for m in all_migrations if m.stem not in applied]
+
+    summary = {
+        "applied": sorted(applied),
+        "pending": pending,
+        "total": len(all_migrations),
+    }
+    if output_json:
+        typer.echo(json.dumps(summary, indent=2))
+        return
+
+    typer.echo(f"Total migrations: {len(all_migrations)}")
+    typer.echo(f"Applied: {len(applied)}")
+    typer.echo(f"Pending: {len(pending)}")
+    for migration_id in pending:
+        typer.echo(f"  - {migration_id}")

chkit/cli/config_loader.py

@@ -0,0 +1,47 @@
+"""Load and validate a user's ``chkit.config.py`` config file."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+from typing import Any
+
+from chkit.core.model import ChxResolvedConfig, ChxUserConfig, resolve_config
+
+
+def _load_module(config_path: Path) -> Any:
+    spec = importlib.util.spec_from_file_location("chkit_user_config", config_path)
+    if spec is None or spec.loader is None:
+        msg = f"Unable to load config module from {config_path}"
+        raise RuntimeError(msg)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["chkit_user_config"] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def load_config(config_path: Path | None = None) -> ChxResolvedConfig:
+    """Resolve a ``ChxResolvedConfig`` from a config file.
+
+    The default path is ``./chkit.config.py``. The module must export a
+    ``config`` attribute of type ``ChxUserConfig`` (or a dict that validates
+    against ``ChxUserConfig``).
+    """
+    path = config_path if config_path is not None else Path("chkit.config.py")
+    if not path.exists():
+        msg = f"Config file not found: {path}"
+        raise FileNotFoundError(msg)
+
+    module = _load_module(path)
+    raw_config = getattr(module, "config", None)
+    if raw_config is None:
+        msg = f"Config file {path} must export a `config` attribute"
+        raise AttributeError(msg)
+
+    user_config = (
+        raw_config
+        if isinstance(raw_config, ChxUserConfig)
+        else ChxUserConfig.model_validate(raw_config)
+    )
+    return resolve_config(user_config)

chkit/cli/main.py

@@ -0,0 +1,47 @@
+"""Top-level Typer app and command wiring."""
+
+from __future__ import annotations
+
+import typer
+
+from chkit import __version__
+from chkit.cli.commands import check, drift, generate, init, migrate, status
+
+app = typer.Typer(
+    name="chkit",
+    help="ClickHouse schema and migration toolkit",
+    add_completion=False,
+)
+
+app.command("init", help="Create a starter chkit.config.py and example schema.")(init.run)
+app.command("generate", help="Generate a new migration from the current schema.")(generate.run)
+app.command("migrate", help="Apply pending migrations to the target database.")(migrate.run)
+app.command("status", help="Show migration status and pending operations.")(status.run)
+app.command("check", help="Run pre-flight checks (drift, checksums, pending).")(check.run)
+app.command("drift", help="Compare the live database against the schema snapshot.")(drift.run)
+
+
+@app.callback(invoke_without_command=True)
+def _root(
+    ctx: typer.Context,
+    version: bool = typer.Option(
+        False,  # noqa: FBT003 - typer positional default is part of its API
+        "--version",
+        "-V",
+        help="Show the chkit version and exit.",
+    ),
+) -> None:
+    if version:
+        typer.echo(__version__)
+        raise typer.Exit(code=0)
+    if ctx.invoked_subcommand is None:
+        typer.echo(ctx.get_help())
+        raise typer.Exit(code=0)
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    main()

chkit/cli/migration_store.py

@@ -0,0 +1,84 @@
+"""Filesystem layout for migrations and snapshots."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+from chkit.core.model import MigrationPlan, Snapshot
+
+
+class MigrationArtifact(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    id: str
+    sql_path: Path
+    meta_path: Path
+    plan: MigrationPlan
+    checksum: str
+
+
+def _timestamp_id() -> str:
+    return datetime.now(tz=UTC).strftime("%Y%m%d%H%M%S")
+
+
+def _hash_sql(sql_text: str) -> str:
+    return hashlib.sha256(sql_text.encode("utf-8")).hexdigest()
+
+
+def write_snapshot(meta_dir: Path, snapshot: Snapshot) -> Path:
+    meta_dir.mkdir(parents=True, exist_ok=True)
+    snapshot_path = meta_dir / "snapshot.json"
+    snapshot_path.write_text(
+        json.dumps(snapshot.model_dump(mode="json", by_alias=True), indent=2),
+        encoding="utf-8",
+    )
+    return snapshot_path
+
+
+def read_snapshot(meta_dir: Path) -> Snapshot | None:
+    snapshot_path = meta_dir / "snapshot.json"
+    if not snapshot_path.exists():
+        return None
+    raw = json.loads(snapshot_path.read_text(encoding="utf-8"))
+    return Snapshot.model_validate(raw)
+
+
+def write_migration(
+    migrations_dir: Path, plan: MigrationPlan, label: str = "auto"
+) -> MigrationArtifact:
+    migrations_dir.mkdir(parents=True, exist_ok=True)
+    migration_id = f"{_timestamp_id()}_{label}"
+    sql_text = "\n\n".join(op.sql for op in plan.operations) + "\n"
+    sql_path = migrations_dir / f"{migration_id}.sql"
+    meta_path = migrations_dir / f"{migration_id}.json"
+    sql_path.write_text(sql_text, encoding="utf-8")
+    checksum = _hash_sql(sql_text)
+    meta_path.write_text(
+        json.dumps(
+            {
+                "id": migration_id,
+                "checksum": checksum,
+                "plan": plan.model_dump(mode="json", by_alias=True),
+            },
+            indent=2,
+        ),
+        encoding="utf-8",
+    )
+    return MigrationArtifact(
+        id=migration_id,
+        sql_path=sql_path,
+        meta_path=meta_path,
+        plan=plan,
+        checksum=checksum,
+    )
+
+
+def list_migrations(migrations_dir: Path) -> list[Path]:
+    if not migrations_dir.exists():
+        return []
+    return sorted(migrations_dir.glob("*.sql"))