clickhouse-charon 1.0.2__py3-none-any.whl

charon/__init__.py

@@ -0,0 +1,3 @@
+"""CHaron — ClickHouse migration tool."""
+
+__version__ = "1.0.2"

charon/cli/app.py

@@ -0,0 +1,207 @@
+"""Main typer application — registers all sub-commands."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from charon import __version__
+from charon.config import AppConfig, CopyProfile, config_path_from_option, load_config
+
+from .config_cmd import config_app
+
+app = typer.Typer(
+    name="charon",
+    help="ClickHouse database copier — copy tables between instances.",
+    no_args_is_help=True,
+)
+app.add_typer(config_app, name="config")
+
+console = Console()
+
+
+# ---------------------------------------------------------------------------
+# State object passed via typer Context
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class State:
+    config_path: Path
+    profile_name: str | None
+    cfg: AppConfig
+    profile: CopyProfile | None = None
+
+
+def _get_state(ctx: typer.Context) -> State:
+    state: State = ctx.ensure_object(State)
+    return state
+
+
+# ---------------------------------------------------------------------------
+# Global callback (sets up state before any sub-command)
+# ---------------------------------------------------------------------------
+
+
+@app.callback()
+def main(
+    ctx: typer.Context,
+    config: str | None = typer.Option(
+        None, "--config", "-c", envvar="CHCOPY_CONFIG", help="Config file path"
+    ),
+    profile: str | None = typer.Option(
+        None, "--profile", "-p", envvar="CHCOPY_PROFILE", help="Profile name"
+    ),
+    version: bool = typer.Option(
+        False, "--version", "-v", help="Show version and exit", is_eager=True
+    ),
+) -> None:
+    if version:
+        console.print(f"charon {__version__}")
+        raise typer.Exit()
+
+    config_path = config_path_from_option(config)
+    cfg = load_config(config_path)
+
+    # Try to resolve the profile (may be None if config is empty)
+    copy_profile: CopyProfile | None = None
+    if cfg.profiles:
+        try:
+            copy_profile = cfg.get_profile(profile)
+        except KeyError:
+            pass
+
+    ctx.obj = State(
+        config_path=config_path,
+        profile_name=profile,
+        cfg=cfg,
+        profile=copy_profile,
+    )
+    ctx.ensure_object(State)
+
+
+def _require_profile(ctx: typer.Context) -> CopyProfile:
+    state = _get_state(ctx)
+    if state.profile is None:
+        console.print("[red]No profile configured. Run `charon config init` to create one.[/red]")
+        raise typer.Exit(1)
+    return state.profile
+
+
+# ---------------------------------------------------------------------------
+# status
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def status(ctx: typer.Context) -> None:
+    """Ping src and dst; show table count, rows, and bytes."""
+    from .status_cmd import run_status
+
+    profile = _require_profile(ctx)
+    run_status(profile)
+
+
+# ---------------------------------------------------------------------------
+# list
+# ---------------------------------------------------------------------------
+
+
+@app.command("list")
+def list_tables(
+    ctx: typer.Context,
+    dst: bool = typer.Option(False, "--dst", help="List destination tables instead of source"),
+    filter_re: str | None = typer.Option(None, "--filter", "-f", help="Regex filter on table name"),
+) -> None:
+    """List tables on src (or dst with --dst)."""
+    from .list_cmd import run_list
+
+    profile = _require_profile(ctx)
+    run_list(profile, use_dst=dst, filter_re=filter_re)
+
+
+# ---------------------------------------------------------------------------
+# diff
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def diff(
+    ctx: typer.Context,
+    table: str | None = typer.Option(
+        None, "--table", "-t", help="Focus on single table (partition-level)"
+    ),
+    output_json: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    """Show row-count diff between src and dst."""
+    from .diff_cmd import run_diff
+
+    profile = _require_profile(ctx)
+    run_diff(profile, table_name=table, output_json=output_json)
+
+
+# ---------------------------------------------------------------------------
+# copy
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def copy(
+    ctx: typer.Context,
+    table: str | None = typer.Option(None, "--table", "-t", help="Copy a single table"),
+    tables: str | None = typer.Option(
+        None, "--tables", help="Comma-separated list of tables to copy"
+    ),
+    dry_run: bool = typer.Option(
+        False, "--dry-run", help="Show what would be copied without executing"
+    ),
+    no_partitions: bool = typer.Option(
+        False, "--no-partitions", help="Copy whole tables without partition awareness"
+    ),
+) -> None:
+    """Copy tables from src to dst."""
+    from .copy_cmd import run_copy
+
+    profile = _require_profile(ctx)
+    table_list = [t.strip() for t in tables.split(",")] if tables else None
+    run_copy(
+        profile=profile,
+        table_name=table,
+        tables=table_list,
+        dry_run=dry_run,
+        no_partitions=no_partitions,
+    )
+
+
+# ---------------------------------------------------------------------------
+# web
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def web(
+    ctx: typer.Context,
+    host: str = typer.Option("127.0.0.1", "--host", help="Bind address"),
+    port: int = typer.Option(8765, "--port", help="Port"),
+    reload: bool = typer.Option(False, "--reload", help="Enable auto-reload (dev mode)"),
+) -> None:
+    """Start the web dashboard."""
+    import uvicorn
+
+    from charon.web.app import create_app
+
+    state = _get_state(ctx)
+
+    fastapi_app = create_app(config_path=state.config_path, profile_name=state.profile_name)
+
+    console.print(f"[bold green]CHaron[/bold green] → [link]http://{host}:{port}[/link]")
+
+    uvicorn.run(
+        fastapi_app,
+        host=host,
+        port=port,
+        reload=reload,
+    )

charon/cli/config_cmd.py

@@ -0,0 +1,184 @@
+"""CLI sub-app: manage connection profiles."""
+
+from __future__ import annotations
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from charon.config import (
+    CopyProfile,
+    InstanceConfig,
+    config_path_from_option,
+    load_config,
+    save_config,
+)
+
+config_app = typer.Typer(name="config", help="Manage connection profiles.")
+console = Console()
+
+
+@config_app.command("init")
+def init(
+    profile: str = typer.Option(
+        "default", "--profile", "-p", help="Profile name to create or update"
+    ),
+    config: str | None = typer.Option(None, "--config", "-c", help="Config file path"),
+) -> None:
+    """Interactively create or update a named profile."""
+    config_path = config_path_from_option(config)
+    cfg = load_config(config_path)
+
+    console.print(f"[bold]Creating/updating profile:[/bold] [cyan]{profile}[/cyan]")
+    console.print("(Press Enter to accept the default shown in brackets)\n")
+
+    existing = cfg.profiles.get(profile)
+
+    # Source
+    console.print("[bold yellow]Source ClickHouse[/bold yellow]")
+    src_host = typer.prompt(
+        "  Host", default=existing.src.host if existing else "http://localhost:8123"
+    )
+    src_user = typer.prompt("  User", default=existing.src.user if existing else "default")
+    src_password = typer.prompt(
+        "  Password",
+        default=existing.src.password if existing else "",
+        hide_input=True,
+    )
+    src_database = typer.prompt(
+        "  Database", default=existing.src.database if existing else "default"
+    )
+
+    # Destination
+    console.print("\n[bold yellow]Destination ClickHouse[/bold yellow]")
+    dst_host = typer.prompt(
+        "  Host", default=existing.dst.host if existing else "http://localhost:8123"
+    )
+    dst_user = typer.prompt("  User", default=existing.dst.user if existing else "default")
+    dst_password = typer.prompt(
+        "  Password",
+        default=existing.dst.password if existing else "",
+        hide_input=True,
+    )
+    dst_database = typer.prompt(
+        "  Database", default=existing.dst.database if existing else "default"
+    )
+    dst_tcp = typer.prompt(
+        "  TCP hostport (host:port, used for remote() INSERT)",
+        default=existing.dst.tcp_hostport if existing and existing.dst.tcp_hostport else "",
+    )
+
+    copy_profile = CopyProfile(
+        src=InstanceConfig(
+            host=src_host, user=src_user, password=src_password, database=src_database
+        ),
+        dst=InstanceConfig(
+            host=dst_host,
+            user=dst_user,
+            password=dst_password,
+            database=dst_database,
+            tcp_hostport=dst_tcp or None,
+        ),
+    )
+
+    cfg.profiles[profile] = copy_profile
+    if not cfg.default_profile or profile == "default":
+        cfg.default_profile = profile
+
+    save_config(cfg, config_path)
+    console.print(f"\n[green]Profile '{profile}' saved to {config_path}[/green]")
+
+
+@config_app.command("show")
+def show(
+    profile: str | None = typer.Option(None, "--profile", "-p", help="Profile name"),
+    config: str | None = typer.Option(None, "--config", "-c", help="Config file path"),
+) -> None:
+    """Print the current profile (password masked)."""
+    config_path = config_path_from_option(config)
+    cfg = load_config(config_path)
+
+    try:
+        prof = cfg.get_profile(profile)
+    except KeyError as exc:
+        console.print(f"[red]{exc}[/red]")
+        raise typer.Exit(1) from exc
+
+    name = profile or cfg.default_profile
+
+    table = Table(title=f"Profile: {name}", show_header=True)
+    table.add_column("Field", style="bold")
+    table.add_column("Source")
+    table.add_column("Destination")
+
+    table.add_row("Host", prof.src.host, prof.dst.host)
+    table.add_row("User", prof.src.user, prof.dst.user)
+    table.add_row("Password", "***", "***")
+    table.add_row("Database", prof.src.database, prof.dst.database)
+    table.add_row("Timeout", str(prof.src.timeout), str(prof.dst.timeout))
+    table.add_row("TCP hostport", "-", prof.dst.tcp_hostport or "-")
+
+    console.print(table)
+
+    console.print(f"\n[dim]copy_by_partitions:[/dim] {prof.copy_by_partitions}")
+    console.print(f"[dim]convert_replicated_to_merge:[/dim] {prof.convert_replicated_to_merge}")
+    console.print(f"[dim]retry_count:[/dim] {prof.retry_count}")
+    console.print(f"[dim]retry_sleep:[/dim] {prof.retry_sleep}s  max {prof.retry_max_sleep}s")
+
+
+@config_app.command("list")
+def list_profiles(
+    config: str | None = typer.Option(None, "--config", "-c", help="Config file path"),
+) -> None:
+    """List all available profiles."""
+    config_path = config_path_from_option(config)
+    cfg = load_config(config_path)
+
+    if not cfg.profiles:
+        console.print(
+            "[yellow]No profiles configured. Run `charon config init` to create one.[/yellow]"
+        )
+        return
+
+    table = Table(show_header=True, header_style="bold")
+    table.add_column("Profile")
+    table.add_column("Src host")
+    table.add_column("Src DB")
+    table.add_column("Dst host")
+    table.add_column("Dst DB")
+    table.add_column("Default")
+
+    for name, prof in cfg.profiles.items():
+        is_default = "[green]✓[/green]" if name == cfg.default_profile else ""
+        table.add_row(
+            name, prof.src.host, prof.src.database, prof.dst.host, prof.dst.database, is_default
+        )
+
+    console.print(table)
+
+
+@config_app.command("delete")
+def delete(
+    profile_name: str = typer.Argument(..., help="Profile name to delete"),
+    config: str | None = typer.Option(None, "--config", "-c", help="Config file path"),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation"),
+) -> None:
+    """Remove a profile."""
+    config_path = config_path_from_option(config)
+    cfg = load_config(config_path)
+
+    if profile_name not in cfg.profiles:
+        console.print(f"[red]Profile '{profile_name}' not found.[/red]")
+        raise typer.Exit(1)
+
+    if not yes:
+        confirmed = typer.confirm(f"Delete profile '{profile_name}'?")
+        if not confirmed:
+            raise typer.Abort()
+
+    del cfg.profiles[profile_name]
+    if cfg.default_profile == profile_name:
+        cfg.default_profile = next(iter(cfg.profiles), "default")
+
+    save_config(cfg, config_path)
+    console.print(f"[green]Profile '{profile_name}' deleted.[/green]")

charon/cli/copy_cmd.py

@@ -0,0 +1,173 @@
+"""CLI command: copy — copy tables from src to dst with rich progress."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from rich.console import Console
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID, TextColumn, TimeElapsedColumn
+from rich.table import Table
+
+from charon.client import CHClient, RetryConfig
+from charon.config import CopyProfile
+from charon.copy import copy_database, should_copy_table
+
+console = Console()
+
+
+@dataclass
+class CopySummary:
+    tables_copied: int = 0
+    tables_skipped: int = 0
+    tables_errored: int = 0
+    partitions_copied: int = 0
+    errors: list[tuple[str, str]] = field(default_factory=list)
+
+
+class RichCopyCallback:
+    """Implements CopyCallback using Rich progress bars."""
+
+    def __init__(self, progress: Progress, summary: CopySummary, dry_run: bool = False) -> None:
+        self._progress = progress
+        self._summary = summary
+        self._dry_run = dry_run
+        self._overall_task: TaskID | None = None
+        self._partition_task: TaskID | None = None
+
+    def set_overall_task(self, task_id: TaskID, total: int) -> None:
+        self._overall_task = task_id
+        self._progress.update(task_id, total=total)
+
+    def on_table_start(self, table: str, engine: str, partitions: int) -> None:
+        prefix = "[DRY RUN] " if self._dry_run else ""
+        if self._partition_task is not None:
+            self._progress.remove_task(self._partition_task)
+        self._partition_task = self._progress.add_task(
+            f"{prefix}[cyan]{table}[/cyan] ({engine})",
+            total=max(partitions, 1),
+        )
+
+    def on_partition_start(self, table: str, partition_id: str, rows: int) -> None:
+        if self._partition_task is not None:
+            self._progress.update(
+                self._partition_task,
+                description=f"  partition [yellow]{partition_id}[/yellow] ({rows:,} rows)",
+            )
+
+    def on_partition_done(self, table: str, partition_id: str) -> None:
+        self._summary.partitions_copied += 1
+        if self._partition_task is not None:
+            self._progress.advance(self._partition_task)
+
+    def on_table_done(self, table: str) -> None:
+        self._summary.tables_copied += 1
+        if self._overall_task is not None:
+            self._progress.advance(self._overall_task)
+        if self._partition_task is not None:
+            self._progress.update(self._partition_task, visible=False)
+
+    def on_table_skip(self, table: str, reason: str) -> None:
+        self._summary.tables_skipped += 1
+        console.print(f"  [dim]SKIP[/dim] {table}  [dim]{reason}[/dim]")
+        if self._overall_task is not None:
+            self._progress.advance(self._overall_task)
+
+    def on_error(self, table: str, error: Exception) -> None:
+        self._summary.tables_errored += 1
+        self._summary.errors.append((table, str(error)))
+        console.print(f"  [red]ERROR[/red] {table}: {error}")
+        if self._overall_task is not None:
+            self._progress.advance(self._overall_task)
+
+
+def run_copy(
+    profile: CopyProfile,
+    table_name: str | None = None,
+    tables: list[str] | None = None,
+    dry_run: bool = False,
+    no_partitions: bool = False,
+) -> None:
+    """Orchestrate the copy operation with live progress display."""
+    retry = RetryConfig(
+        retry_count=profile.retry_count,
+        retry_sleep=profile.retry_sleep,
+        retry_max_sleep=profile.retry_max_sleep,
+    )
+    src = CHClient(profile.src, retry=retry)
+    dst = CHClient(profile.dst, retry=retry)
+
+    # Resolve which tables to copy
+    if table_name:
+        table_filter: list[str] | None = [table_name]
+    elif tables:
+        table_filter = tables
+    else:
+        table_filter = None
+
+    # Fetch full table list to determine total work
+    src_tables = src.get_tables(profile.src.database)
+    if table_filter:
+        src_tables = [t for t in src_tables if str(t["name"]) in table_filter]
+
+    eligible = [
+        t
+        for t in src_tables
+        if str(t["name"]) not in profile.skip_tables
+        and should_copy_table(str(t.get("engine", "")), profile)[0]
+    ]
+
+    if dry_run:
+        console.print("[bold yellow]DRY RUN — no data will be copied[/bold yellow]\n")
+        tbl = Table(show_header=True, header_style="bold")
+        tbl.add_column("Table", style="cyan")
+        tbl.add_column("Engine")
+        tbl.add_column("Src rows", justify="right")
+        for t in eligible:
+            tbl.add_row(str(t["name"]), str(t.get("engine", "")), str(t.get("total_rows") or 0))
+        console.print(tbl)
+        console.print(f"\n[dim]{len(eligible)} table(s) would be copied.[/dim]")
+        return
+
+    summary = CopySummary()
+
+    eff_profile = profile
+    if no_partitions:
+        eff_profile = profile.model_copy(update={"copy_by_partitions": False})
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        TextColumn("{task.completed}/{task.total}"),
+        TimeElapsedColumn(),
+        console=console,
+        transient=False,
+    ) as progress:
+        cb = RichCopyCallback(progress, summary, dry_run=dry_run)
+        overall = progress.add_task("[bold]Overall progress[/bold]", total=len(eligible))
+        cb.set_overall_task(overall, len(eligible))
+
+        copy_database(
+            src=src,
+            dst=dst,
+            profile=eff_profile,
+            callback=cb,
+            table_filter=table_filter,
+        )
+
+    # Summary table
+    console.print("\n[bold]Copy summary[/bold]")
+    result_table = Table(show_header=False)
+    result_table.add_column("Label", style="dim")
+    result_table.add_column("Value", style="bold")
+    result_table.add_row("Tables copied", f"[green]{summary.tables_copied}[/green]")
+    result_table.add_row("Tables skipped", str(summary.tables_skipped))
+    errored_val = f"[red]{summary.tables_errored}[/red]" if summary.tables_errored else "0"
+    result_table.add_row("Tables errored", errored_val)
+    result_table.add_row("Partitions copied", str(summary.partitions_copied))
+    console.print(result_table)
+
+    if summary.errors:
+        console.print("\n[bold red]Errors:[/bold red]")
+        for tbl_name, err in summary.errors:
+            console.print(f"  [red]{tbl_name}[/red]: {err}")