synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/cli/turso.py

@@ -1,73 +1,52 @@
-"""Utility CLI command for managing Turso sqld binaries."""
+"""Backwards-compatible Turso CLI entry point.
 
-from __future__ import annotations
+This module re-exports the infra.turso helpers so existing imports
+(`synth_ai.cli.turso`) continue to work for tests and downstream users.
+It also proxies infra functions so monkeypatching works in tests.
+"""
 
-import subprocess
+from __future__ import annotations
 
-import click
+import synth_ai.cli.infra.turso as _infra_turso
+from synth_ai.cli.infra.turso import turso  # noqa: F401
+from synth_ai.cli.root import SQLD_VERSION, find_sqld_binary, install_sqld  # noqa: F401
 
-from synth_ai.cli.root import SQLD_VERSION, find_sqld_binary, install_sqld
+_get_sqld_version_inner = _infra_turso._get_sqld_version
 
 
-def register(cli: click.Group) -> None:
-    """Register the turso command on the main CLI group."""
+def _get_sqld_version(binary: str) -> str | None:  # noqa: F401
+    """Delegates to the original infra implementation (monkeypatchable)."""
+    return _get_sqld_version_inner(binary)
 
-    cli.add_command(turso)
 
+def _proxy_get_sqld_version(binary: str) -> str | None:
+    # Dynamic lookup so monkeypatching synth_ai.cli.turso._get_sqld_version is honored
+    import synth_ai.cli.turso as mod
 
-def _get_sqld_version(binary: str) -> str | None:
-    """Return the version string reported by the sqld binary."""
+    return mod._get_sqld_version(binary)
 
-    try:
-        result = subprocess.run(
-            [binary, "--version"],
-            capture_output=True,
-            text=True,
-            check=True,
-            timeout=5,
-        )
-    except (OSError, subprocess.CalledProcessError, ValueError):
-        return None
 
-    output = result.stdout.strip() or result.stderr.strip()
-    return output or None
+def _proxy_find_sqld_binary() -> str | None:
+    import synth_ai.cli.turso as mod
 
+    return mod.find_sqld_binary()
 
-@click.command()
-@click.option(
-    "--force",
-    is_flag=True,
-    help="Reinstall the pinned sqld build even if one is already available.",
-)
-def turso(force: bool) -> None:
-    """Ensure the Turso sqld binary required for tracing v3 is installed."""
 
-    existing_path = find_sqld_binary()
+def _proxy_install_sqld():
+    import synth_ai.cli.turso as mod
 
-    if existing_path and not force:
-        version_info = _get_sqld_version(existing_path)
-        click.echo(f"✅ Turso sqld detected at {existing_path}.")
-        if version_info:
-            click.echo(f"   Reported version: {version_info}")
-        if version_info and SQLD_VERSION not in version_info:
-            click.echo(
-                f"⚠️ Pinned version is {SQLD_VERSION}. Run with --force to install the supported build."
-            )
-        else:
-            click.echo("No action taken. Use --force to reinstall the pinned build.")
-        return
+    return mod.install_sqld()
 
-    if existing_path and force:
-        click.echo(f"♻️ Reinstalling Turso sqld {SQLD_VERSION} (previously at {existing_path}).")
-    else:
-        click.echo(f"📦 Installing Turso sqld {SQLD_VERSION}…")
 
-    try:
-        installed_path = install_sqld()
-    except subprocess.CalledProcessError as exc:  # pragma: no cover - surfaced as Click error
-        raise click.ClickException(
-            f"sqld installation failed (exit code {exc.returncode})."
-        ) from exc
+# Point infra at proxies so tests that monkeypatch this module influence the CLI command.
+_infra_turso._get_sqld_version = _proxy_get_sqld_version  # type: ignore[assignment]
+_infra_turso.find_sqld_binary = _proxy_find_sqld_binary  # type: ignore[assignment]
+_infra_turso.install_sqld = _proxy_install_sqld  # type: ignore[assignment]
 
-    click.echo(f"✅ sqld installed to {installed_path}")
-    click.echo("Ensure ~/.local/bin is on your PATH before running Synth AI services.")
+__all__ = [
+    "turso",
+    "_get_sqld_version",
+    "find_sqld_binary",
+    "install_sqld",
+    "SQLD_VERSION",
+]

synth_ai/cli/usage.py

@@ -0,0 +1,159 @@
+"""CLI command for displaying org usage and limits.
+
+Usage:
+    synth-ai usage          # Show usage summary
+    synth-ai usage --json   # Output as JSON
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from dataclasses import asdict
+
+import click
+
+
+def _format_number(n: int | float) -> str:
+    """Format a number for display (e.g., 1000 -> 1K)."""
+    if n >= 1_000_000:
+        return f"{n / 1_000_000:.1f}M"
+    if n >= 1_000:
+        return f"{n / 1_000:.1f}K"
+    if isinstance(n, float):
+        return f"{n:.1f}"
+    return str(n)
+
+
+def _progress_bar(percent: float, width: int = 20) -> str:
+    """Create a text-based progress bar."""
+    filled = int(width * min(percent, 100) / 100)
+    bar = "=" * filled + "-" * (width - filled)
+    return f"[{bar}]"
+
+
+def _format_metric(name: str, metric, indent: str = "  ") -> str:
+    """Format a single usage metric for display."""
+    percent = metric.percent_used
+    bar = _progress_bar(percent)
+    used_str = _format_number(metric.used)
+    limit_str = _format_number(metric.limit)
+    return f"{indent}{name}: {bar} {used_str} / {limit_str} ({percent:.1f}%)"
+
+
+@click.command()
+@click.option(
+    "--json",
+    "as_json",
+    is_flag=True,
+    default=False,
+    help="Output as JSON",
+)
+def usage_cmd(as_json: bool) -> None:
+    """Display org usage and rate limits.
+
+    Shows current usage across all Synth APIs with progress bars
+    indicating how much of each limit has been consumed.
+    """
+    try:
+        from synth_ai.sdk.usage import UsageClient
+    except ImportError as e:
+        click.echo(f"Error importing usage client: {e}", err=True)
+        sys.exit(1)
+
+    try:
+        client = UsageClient()
+        usage = client.get()
+    except Exception as e:
+        click.echo(f"Error fetching usage: {e}", err=True)
+        sys.exit(1)
+
+    if as_json:
+        # Convert to dict and output as JSON
+        output = {
+            "org_id": usage.org_id,
+            "tier": usage.tier,
+            "period": {
+                "daily_start": usage.period.daily_start.isoformat(),
+                "monthly_start": usage.period.monthly_start.isoformat(),
+            },
+            "apis": {
+                "inference": {
+                    "requests_per_min": asdict(usage.apis.inference.requests_per_min),
+                    "tokens_per_day": asdict(usage.apis.inference.tokens_per_day),
+                    "spend_cents_per_month": asdict(usage.apis.inference.spend_cents_per_month),
+                },
+                "judges": {
+                    "evaluations_per_day": asdict(usage.apis.judges.evaluations_per_day),
+                },
+                "prompt_opt": {
+                    "jobs_per_day": asdict(usage.apis.prompt_opt.jobs_per_day),
+                    "rollouts_per_day": asdict(usage.apis.prompt_opt.rollouts_per_day),
+                    "spend_cents_per_day": asdict(usage.apis.prompt_opt.spend_cents_per_day),
+                },
+                "rl": {
+                    "jobs_per_month": asdict(usage.apis.rl.jobs_per_month),
+                    "gpu_hours_per_month": asdict(usage.apis.rl.gpu_hours_per_month),
+                },
+                "sft": {
+                    "jobs_per_month": asdict(usage.apis.sft.jobs_per_month),
+                    "gpu_hours_per_month": asdict(usage.apis.sft.gpu_hours_per_month),
+                },
+                "research": {
+                    "jobs_per_month": asdict(usage.apis.research.jobs_per_month),
+                    "agent_spend_cents_per_month": asdict(usage.apis.research.agent_spend_cents_per_month),
+                },
+            },
+            "totals": {
+                "spend_cents_per_month": asdict(usage.totals.spend_cents_per_month),
+            },
+        }
+        click.echo(json.dumps(output, indent=2))
+        return
+
+    # Pretty print format
+    click.echo("")
+    click.echo(f"Org: {usage.org_id} ({usage.tier} tier)")
+    click.echo("")
+
+    # Inference
+    click.echo("Inference")
+    click.echo(_format_metric("requests/min", usage.apis.inference.requests_per_min))
+    click.echo(_format_metric("tokens/day", usage.apis.inference.tokens_per_day))
+    click.echo(_format_metric("spend/month", usage.apis.inference.spend_cents_per_month))
+    click.echo("")
+
+    # Judges
+    click.echo("Judges")
+    click.echo(_format_metric("evaluations/day", usage.apis.judges.evaluations_per_day))
+    click.echo("")
+
+    # Prompt Optimization
+    click.echo("Prompt Optimization")
+    click.echo(_format_metric("jobs/day", usage.apis.prompt_opt.jobs_per_day))
+    click.echo(_format_metric("rollouts/day", usage.apis.prompt_opt.rollouts_per_day))
+    click.echo(_format_metric("spend/day", usage.apis.prompt_opt.spend_cents_per_day))
+    click.echo("")
+
+    # RL Training
+    click.echo("RL Training")
+    click.echo(_format_metric("jobs/month", usage.apis.rl.jobs_per_month))
+    click.echo(_format_metric("gpu_hours/month", usage.apis.rl.gpu_hours_per_month))
+    click.echo("")
+
+    # SFT Training
+    click.echo("SFT Training")
+    click.echo(_format_metric("jobs/month", usage.apis.sft.jobs_per_month))
+    click.echo(_format_metric("gpu_hours/month", usage.apis.sft.gpu_hours_per_month))
+    click.echo("")
+
+    # Research Agents
+    click.echo("Research Agents")
+    click.echo(_format_metric("jobs/month", usage.apis.research.jobs_per_month))
+    click.echo(_format_metric("agent_spend/month", usage.apis.research.agent_spend_cents_per_month))
+    click.echo("")
+
+    # Totals
+    click.echo("Total Spend")
+    click.echo(_format_metric("spend/month", usage.totals.spend_cents_per_month))
+    click.echo("")

synth_ai/cli/utils/__init__.py

@@ -0,0 +1,8 @@
+"""CLI utility commands (recent, traces, queue, experiments).
+
+Utility commands for viewing and managing Synth data.
+"""
+
+__all__: list[str] = []
+
+

synth_ai/cli/utils/experiments.py

@@ -0,0 +1,235 @@
+"""CLI commands for experiment queue management."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+import click
+from rich.console import Console
+
+# Clear config cache if env vars are set (must happen before other imports)
+if os.getenv("EXPERIMENT_QUEUE_DB_PATH") or os.getenv("EXPERIMENT_QUEUE_TRAIN_CMD"):
+    from synth_ai.cli.local.experiment_queue import config as queue_config
+
+    queue_config.reset_config_cache()
+
+from synth_ai.cli.local.experiment_queue.models import ExperimentStatus
+from synth_ai.cli.local.experiment_queue.schemas import (
+    ExperimentJobSummary,
+    ExperimentSubmitRequest,
+    ExperimentSummary,
+    TrialSummary,
+)
+from synth_ai.cli.local.experiment_queue.service import (
+    cancel_experiment,
+    collect_dashboard_data,
+    create_experiment,
+    fetch_experiment,
+    list_experiments,
+)
+from synth_ai.cli.local.experiment_queue.status import (
+    experiment_jobs_table,  # type: ignore[attr-defined]
+    experiment_trials_table,  # type: ignore[attr-defined]
+    render_dashboard,  # type: ignore[attr-defined]
+)
+
+STATUS_CHOICES = [status.value for status in ExperimentStatus]
+
+
+def _load_request_payload(source: str, *, inline: bool = False) -> ExperimentSubmitRequest:
+    if inline:
+        data = json.loads(source)
+    elif source == "-":
+        data = json.load(sys.stdin)
+    else:
+        path = Path(source).expanduser()
+        if not path.exists():
+            raise click.ClickException(f"Request file not found: {path}")
+        with open(path, encoding="utf-8") as fh:
+            data = json.load(fh)
+    return ExperimentSubmitRequest.model_validate(data)
+
+
+def _parse_statuses(values: tuple[str, ...]) -> list[ExperimentStatus] | None:
+    if not values:
+        return None
+    return [ExperimentStatus(value) for value in values]
+
+
+@click.command("experiments")
+@click.option(
+    "--status",
+    "status_filter",
+    multiple=True,
+    type=click.Choice(STATUS_CHOICES),
+    help="Filter experiments by status.",
+)
+@click.option("--recent", default=5, show_default=True, help="Number of recent experiments to show.")
+@click.option("--watch", is_flag=True, help="Continuously refresh the dashboard.")
+@click.option("--json", "as_json", is_flag=True, help="Emit JSON instead of tables.")
+@click.option("--interval", default=2.0, show_default=True, help="Refresh interval for --watch.")
+def experiments_cmd(
+    status_filter: tuple[str, ...],
+    recent: int,
+    watch: bool,
+    as_json: bool,
+    interval: float,
+) -> None:
+    """Show live and recent experiment activity."""
+
+    console = Console()
+
+    def _snapshot() -> None:
+        live, recent_data = collect_dashboard_data(
+            status_filter=_parse_statuses(status_filter),
+            recent_limit=recent,
+        )
+        if as_json:
+            payload = {
+                "live": [ExperimentSummary.from_experiment(exp).model_dump(mode="json") for exp in live],
+                "recent": [ExperimentSummary.from_experiment(exp).model_dump(mode="json") for exp in recent_data],
+            }
+            click.echo(json.dumps(payload, indent=2, default=str))
+        else:
+            console.clear()
+            render_dashboard(console, live, recent_data)
+
+    if watch and not as_json:
+        while True:
+            _snapshot()
+            time.sleep(max(interval, 0.5))
+    else:
+        _snapshot()
+
+
+@click.group("experiment")
+def experiment_group() -> None:
+    """Manage experiment queue submissions."""
+
+
+@experiment_group.command("submit")
+@click.argument("request", type=str)
+@click.option("--inline", is_flag=True, help="Treat REQUEST argument as inline JSON payload.")
+def experiment_submit(request: str, inline: bool) -> None:
+    """Submit a new experiment from JSON request file."""
+
+    payload = _load_request_payload(request, inline=inline)
+    experiment = create_experiment(payload)
+    summary = ExperimentSummary.from_experiment(experiment)
+    click.echo(f"Enqueued experiment {summary.experiment_id} ({summary.name}) with {summary.job_count} jobs.")
+
+
+@experiment_group.command("list")
+@click.option(
+    "--status",
+    "status_filter",
+    multiple=True,
+    type=click.Choice(STATUS_CHOICES),
+    help="Status filter.",
+)
+@click.option("--limit", default=20, show_default=True, help="Max experiments to list.")
+@click.option("--json", "as_json", is_flag=True, help="Emit JSON output.")
+def experiment_list(
+    status_filter: tuple[str, ...],
+    limit: int,
+    as_json: bool,
+) -> None:
+    """List experiments."""
+    experiments = list_experiments(
+        status=_parse_statuses(status_filter),
+        limit=limit,
+        include_live=True,
+    )
+    if as_json:
+        payload = [ExperimentSummary.from_experiment(exp).model_dump(mode="json") for exp in experiments]
+        click.echo(json.dumps(payload, indent=2, default=str))
+        return
+
+    console = Console()
+    render_dashboard(console, experiments, [])
+
+
+def _experiment_detail_json(experiment_id: str) -> str:
+    experiment = fetch_experiment(experiment_id)
+    if not experiment:
+        raise click.ClickException(f"Experiment {experiment_id} not found.")
+    payload = ExperimentSummary.from_experiment(experiment).model_dump(mode="json")
+    payload["jobs"] = [ExperimentJobSummary.from_job(job).model_dump(mode="json") for job in experiment.jobs]
+    payload["trials"] = [TrialSummary.from_trial(trial).model_dump(mode="json") for trial in experiment.trials]
+    return json.dumps(payload, indent=2, default=str)
+
+
+def _experiment_detail_console(experiment_id: str, *, console: Console | None = None, clear: bool = False) -> None:
+    experiment = fetch_experiment(experiment_id)
+    if not experiment:
+        raise click.ClickException(f"Experiment {experiment_id} not found.")
+    console = console or Console()
+    if clear:
+        console.clear()
+    summary = ExperimentSummary.from_experiment(experiment)
+    console.rule(f"[bold]Experiment {summary.experiment_id} — {summary.name}")
+    console.print(f"Status: {summary.status.value}")
+    console.print(f"Description: {summary.description or '-'}")
+    metadata_blob = experiment.metadata_json if isinstance(experiment.metadata_json, dict) else {}
+    aggregate = metadata_blob.get("aggregate", {})
+    if aggregate:
+        console.print(f"Best Score: {aggregate.get('best_score')}")
+        console.print(f"Baseline: {aggregate.get('baseline_score')}")
+        console.print(f"Rollouts: {aggregate.get('total_rollouts')}")
+        console.print(f"Total Time: {aggregate.get('total_time')}")
+    console.print(experiment_jobs_table(experiment.jobs))
+    if experiment.trials:
+        console.print(experiment_trials_table(experiment.trials))
+    else:
+        console.print("No trials recorded yet.")
+
+
+@experiment_group.command("status")
+@click.argument("experiment_id")
+@click.option("--watch", is_flag=True, help="Continuously refresh status view.")
+@click.option("--json", "as_json", is_flag=True, help="Emit JSON payload.")
+@click.option("--interval", default=2.0, show_default=True, help="Refresh interval for --watch.")
+def experiment_status(experiment_id: str, watch: bool, as_json: bool, interval: float) -> None:
+    """Show detailed experiment status."""
+    if as_json:
+        click.echo(_experiment_detail_json(experiment_id))
+        return
+
+    if watch:
+        console = Console()
+        while True:
+            _experiment_detail_console(experiment_id, console=console, clear=True)
+            time.sleep(max(interval, 0.5))
+    else:
+        _experiment_detail_console(experiment_id)
+
+
+@experiment_group.command("results")
+@click.argument("experiment_id")
+@click.option("--json", "as_json", is_flag=True, help="Emit JSON payload.")
+def experiment_results(experiment_id: str, as_json: bool) -> None:
+    """Show aggregate results for an experiment."""
+    if as_json:
+        click.echo(_experiment_detail_json(experiment_id))
+    else:
+        _experiment_detail_console(experiment_id)
+
+
+@experiment_group.command("cancel")
+@click.argument("experiment_id")
+def experiment_cancel(experiment_id: str) -> None:
+    """Cancel an experiment and revoke in-flight jobs."""
+    experiment = cancel_experiment(experiment_id)
+    if not experiment:
+        raise click.ClickException(f"Experiment {experiment_id} not found.")
+    click.echo(f"Experiment {experiment_id} canceled.")
+
+
+def register(cli: click.Group) -> None:
+    """Register commands on the main CLI."""
+    cli.add_command(experiments_cmd)
+    cli.add_command(experiment_group)