bonito-cli 0.9.2__tar.gz → 0.9.4__tar.gz

bonito_cli-0.9.4/.gitignore

@@ -0,0 +1,32 @@
+node_modules/
+.next/
+__pycache__/
+*.pyc
+.env
+.venv/
+dist/
+*.egg-info/
+.env.secrets
+secrets/age-key.txt
+secrets/*.yaml
+!secrets/*.enc.yaml
+benchmarks/locomo-repo
+bonito-kb-sa-key.json
+.claude/worktrees/
+
+# Local dev overrides
+docker-compose.override.yml
+
+# Build artifacts
+frontend/tsconfig.tsbuildinfo
+
+# Test output
+frontend/test-results/
+
+# Business documents — never commit
+invoices/
+
+# Generated demo captures & audit reports
+studio-demo/
+studio-demo-haiku-clean/
+studio-*-audit.html

{bonito_cli-0.9.2 → bonito_cli-0.9.4}/CHANGELOG.md

@@ -2,6 +2,46 @@
 
 All notable changes to the Bonito CLI will be documented in this file.
 
+## 0.9.4
+
+- `bonito agents runs list <agent-id>` — recent runs with health rollups
+  (duration, llm/tool call counts, errors, cost).
+- `bonito agents runs show <agent-id> <run-id>` — the full trace tree for one
+  run, with gateway calls (model, tokens, cost) joined onto their llm spans.
+- New `bonito skills` group — org skill library (agent how-tos): list, add
+  from markdown (Poseidon-compatible frontmatter), show, enable/disable,
+  remove. Agents load skills on demand via the new `use_skill` tool.
+- New `bonito gateway sessions` group — inspect server-held /v1 conversations
+  (clients opt in with the X-Bonito-Session header): list, show (summary +
+  stored transcript), clear.
+- New `bonito gateway tokens` group — ephemeral browser tokens (`be-`):
+  mint (TTL, model allow-list, rpm, request cap), list, revoke. The safe
+  credential for browser apps calling /v1 directly; never ship a bn- key
+  to a browser.
+- New `bonito artifacts` group — org artifact store (files produced by
+  agents via the new `save_artifact` tool, or uploaded by harness runs):
+  list (filter by trace run/session), upload, download, show, remove.
+- New `bonito evals` group — quality checks over runs: add from a
+  json/yaml spec (deterministic trace/output checks + LLM-as-judge),
+  run against a trace run or output text, results history, remove.
+  Evals bound to an agent with `auto: true` fire after every turn.
+- `bonito providers add deepinfra` — connect DeepInfra (provider #8):
+  OpenAI-compatible OSS inference (Llama, Qwen, DeepSeek, Mixtral, gpt-oss).
+  Also declarable in `bonito.yaml` (`providers: [{name: deepinfra, api_key: ...}]`).
+
+## [0.9.3] - 2026-08-18
+
+### Fixed
+- **`bonito policies create` offered strategies the gateway does not implement.** The list contained `quality_optimized` and `round_robin`, neither of which exists server-side, so choosing one created a policy that silently fell through to the default and served whichever model was listed first. It also omitted `balanced`, `failover` and `ab_test`, which are real. The list now matches the gateway exactly, each strategy explains what it does, and a `--strategy` passed non-interactively is validated instead of posted blindly.
+- **Model roles and weights were written incorrectly.** Every model was sent as `{"role": "primary", "weight": 50}`, so a `failover` policy had no fallbacks for the chain to advance to, and an `ab_test` policy's weights summed to 50 x N instead of 100, meaning models past the first never received traffic. Roles are now assigned per strategy and `ab_test` weights sum to exactly 100.
+- The interactive strategy prompt was hardcoded to `Select (1-4)` regardless of how many strategies existed.
+
+### Changed
+- `bonito policies test` now reports the real blended cost and expected latency for the selected model. Both were previously hardcoded server-side as a "Demo value" (`$0.01`, `150ms`) and returned identically for every policy and every model.
+
+### Note
+Requires a backend running Bonito `959b5c0` or later, where `cost_optimized`, `latency_optimized` and `balanced` actually rank models. Before that commit those strategies returned the first configured model regardless of price or latency.
+
 ## [0.8.0] - 2026-05-27
 
 ### Added

{bonito_cli-0.9.2 → bonito_cli-0.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: bonito-cli
-Version: 0.9.2
+Version: 0.9.4
 Summary: Bonito CLI — Unified multi-cloud AI management from your terminal
 Author-email: Bonito <hello@getbonito.com>
 License: MIT

{bonito_cli-0.9.2 → bonito_cli-0.9.4}/bonito_cli/api.py

@@ -166,6 +166,42 @@ class BonitoAPI:
     def delete(self, endpoint: str) -> Any:
         return self._request("DELETE", endpoint)
 
+    def post_multipart(
+        self,
+        endpoint: str,
+        files: Dict[str, Any],
+        data: Optional[Dict[str, Any]] = None,
+    ) -> Any:
+        """POST a multipart form (file uploads). `files` is httpx's
+        {field: (filename, bytes, mime)} shape."""
+        url = endpoint if endpoint.startswith("http") else f"/api{endpoint}"
+        headers = {k: v for k, v in self._headers().items()
+                   if k.lower() != "content-type"}
+        try:
+            resp = self.client.request("POST", url, files=files, data=data,
+                                       headers=headers, timeout=120.0)
+        except httpx.RequestError as exc:
+            raise APIError(f"Connection failed: {exc}") from exc
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json().get("detail", f"HTTP {resp.status_code}")
+            except Exception:
+                detail = f"HTTP {resp.status_code}: {resp.text[:200]}"
+            raise APIError(str(detail), resp.status_code)
+        return resp.json()
+
+    def get_bytes(self, endpoint: str) -> bytes:
+        """GET raw bytes (artifact downloads)."""
+        url = endpoint if endpoint.startswith("http") else f"/api{endpoint}"
+        try:
+            resp = self.client.request("GET", url, headers=self._headers(),
+                                       timeout=120.0)
+        except httpx.RequestError as exc:
+            raise APIError(f"Connection failed: {exc}") from exc
+        if resp.status_code >= 400:
+            raise APIError(f"HTTP {resp.status_code}", resp.status_code)
+        return resp.content
+
     # ── streaming (SSE) ─────────────────────────────────────────
 
     def stream_post(

{bonito_cli-0.9.2 → bonito_cli-0.9.4}/bonito_cli/app.py

@@ -32,6 +32,9 @@ from .commands.approval import app as approval_app
 from .commands.scheduler import app as scheduler_app
 from .commands.audit import app as audit_app
 from .commands.memory import app as memory_app
+from .commands.skills import app as skills_app
+from .commands.artifacts import app as artifacts_app
+from .commands.evals import app as evals_app
 from .commands.mcp import app as mcp_app
 from .commands.routing import app as routing_app
 
@@ -102,6 +105,9 @@ app.add_typer(approval_app,    name="approval",    help="✅ Agent approval work
 app.add_typer(scheduler_app,   name="scheduler",   help="⏰ Agent scheduling")
 app.add_typer(audit_app,       name="audit",       help="📝 Audit logs")
 app.add_typer(memory_app,      name="memory",      help="🧠 Agent persistent memory")
+app.add_typer(skills_app,      name="skills",      help="📚 Org skill library (agent how-tos)")
+app.add_typer(artifacts_app,   name="artifacts",   help="📦 Artifact store (agent/harness outputs)")
+app.add_typer(evals_app,       name="evals",       help="✅ Evals (quality checks over runs)")
 app.add_typer(mcp_app,         name="mcp",         help="🔌 MCP server management")
 app.add_typer(routing_app,     name="routing",     help="🔀 Routing rule management")
 

{bonito_cli-0.9.2 → bonito_cli-0.9.4}/bonito_cli/commands/agents.py

@@ -556,6 +556,114 @@ def list_triggers(
 
 # ─── Scaling / HPA Commands ───
 
+# ── Trace tree: agent runs ───────────────────────────────────────────────────
+
+runs_app = typer.Typer(help="Per-run traces: what the agent did, called, and cost")
+app.add_typer(runs_app, name="runs")
+
+
+@runs_app.command("list")
+def runs_list(
+    agent_id: str = typer.Argument(..., help="Agent ID"),
+    limit: int = typer.Option(20, "--limit", "-n", help="How many runs"),
+    status: Optional[str] = typer.Option(None, "--status", help="Filter: ok | error | denied"),
+    json_output: bool = typer.Option(False, "--json", help="Output in JSON format"),
+):
+    """Recent runs with health rollups — duration, calls, errors, cost."""
+    ensure_authenticated()
+    try:
+        qs = f"?limit={limit}" + (f"&status={status}" if status else "")
+        data = api.get(f"/agents/{agent_id}/runs{qs}")
+    except APIError as e:
+        print_error(f"Failed to list runs: {e}")
+        raise typer.Exit(1)
+
+    runs = data.get("runs", [])
+    if json_output:
+        console.print_json(json.dumps(runs))
+        return
+    if not runs:
+        print_info("No traced runs yet — runs appear after the agent executes.")
+        return
+
+    table = Table(title=f"Runs for agent {agent_id}")
+    table.add_column("Run", style="cyan", no_wrap=True)
+    table.add_column("When")
+    table.add_column("Status")
+    table.add_column("Duration")
+    table.add_column("LLM")
+    table.add_column("Tools")
+    table.add_column("Errors")
+    table.add_column("Cost", justify="right")
+    for r in runs:
+        table.add_row(
+            r["run_id"][:8],
+            format_timestamp(r.get("started_at")),
+            format_status(r.get("status", "")),
+            f"{(r.get('duration_ms') or 0) / 1000:.1f}s",
+            str(r.get("llm_calls", 0)),
+            str(r.get("tool_calls", 0)),
+            str(r.get("errors", 0) + r.get("denied", 0)),
+            format_cost(r.get("cost", 0)),
+        )
+    console.print(table)
+    console.print("[dim]bonito agents runs show <agent-id> <run-id> for the full tree[/dim]")
+
+
+@runs_app.command("show")
+def runs_show(
+    agent_id: str = typer.Argument(..., help="Agent ID"),
+    run_id: str = typer.Argument(..., help="Run ID (full UUID, or copy from runs list)"),
+    json_output: bool = typer.Option(False, "--json", help="Output in JSON format"),
+):
+    """One run as a tree: spans nested, gateway calls joined onto llm spans."""
+    ensure_authenticated()
+    try:
+        data = api.get(f"/agents/{agent_id}/runs/{run_id}")
+    except APIError as e:
+        print_error(f"Failed to fetch run: {e}")
+        raise typer.Exit(1)
+
+    if json_output:
+        console.print_json(json.dumps(data))
+        return
+
+    from rich.tree import Tree as RichTree
+
+    def label(node):
+        status = node.get("status", "ok")
+        colour = {"ok": "green", "error": "red", "denied": "yellow",
+                  "timeout": "red"}.get(status, "white")
+        base = (f"[bold]{node['kind']}[/bold] {node['name']} "
+                f"[{colour}]{status}[/{colour}] "
+                f"[dim]{(node.get('duration_ms') or 0) / 1000:.2f}s[/dim]")
+        if node.get("error"):
+            base += f"\n[red dim]{node['error'][:120]}[/red dim]"
+        for g in node.get("gateway_calls", []):
+            base += (f"\n[dim]→ {g.get('model_used') or g.get('model_requested')} "
+                     f"({g.get('provider') or '?'}) "
+                     f"{g.get('input_tokens', 0)}in/{g.get('output_tokens', 0)}out "
+                     f"{format_cost(g.get('cost', 0))}[/dim]")
+        return base
+
+    def attach(rich_node, node):
+        for child in node.get("children", []):
+            attach(rich_node.add(label(child)), child)
+
+    root = data["tree"]
+    tree_view = RichTree(label(root))
+    attach(tree_view, root)
+    console.print(tree_view)
+
+    t = data.get("totals", {})
+    console.print(
+        f"\n[bold]Totals:[/bold] {t.get('spans', 0)} spans · "
+        f"{t.get('gateway_calls', 0)} model calls · "
+        f"{t.get('input_tokens', 0)}in/{t.get('output_tokens', 0)}out tokens · "
+        f"{format_cost(t.get('cost', 0))}"
+    )
+
+
 scaling_app = typer.Typer(help="Agent autoscaling (HPA) management")
 app.add_typer(scaling_app, name="scaling")
 

bonito_cli-0.9.4/bonito_cli/commands/artifacts.py

@@ -0,0 +1,145 @@
+"""Artifact store — durable outputs from agents and harness runs."""
+
+import json
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from ..api import api, APIError
+from ..utils.auth import ensure_authenticated
+from ..utils.display import print_error, print_info, print_success
+
+console = Console()
+app = typer.Typer(help="📦 Artifacts — files produced by agents and harness runs")
+
+
+@app.command("list")
+def list_artifacts(
+    run_id: Optional[str] = typer.Option(None, "--run", help="Filter by trace run id"),
+    session: Optional[str] = typer.Option(None, "--session", help="Filter by session key"),
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+):
+    """List the org's artifacts."""
+    ensure_authenticated()
+    params = {}
+    if run_id:
+        params["run_id"] = run_id
+    if session:
+        params["session_key"] = session
+    try:
+        data = api.get("/artifacts", params=params or None)
+    except APIError as e:
+        print_error(f"Failed to list artifacts: {e}")
+        raise typer.Exit(1)
+    artifacts = data.get("artifacts", [])
+    if json_output:
+        console.print_json(json.dumps(artifacts))
+        return
+    if not artifacts:
+        print_info("No artifacts yet.")
+        return
+    table = Table(title="Artifacts")
+    table.add_column("ID", style="cyan")
+    table.add_column("Name")
+    table.add_column("Type")
+    table.add_column("Size", justify="right")
+    table.add_column("By")
+    table.add_column("Created")
+    for a in artifacts:
+        table.add_row(a["id"][:8], a["name"], a["mime"],
+                      f"{a['size_bytes']:,}", a.get("created_by") or "-",
+                      (a.get("created_at") or "")[:19])
+    console.print(table)
+
+
+@app.command("upload")
+def upload_artifact(
+    file: Path = typer.Argument(..., exists=True, readable=True),
+    name: Optional[str] = typer.Option(None, "--name", help="Override the filename"),
+    run_id: Optional[str] = typer.Option(None, "--run", help="Link to a trace run id"),
+):
+    """Upload a file as an artifact."""
+    ensure_authenticated()
+    import mimetypes
+    mime = mimetypes.guess_type(str(file))[0] or "application/octet-stream"
+    data = {}
+    if name:
+        data["name"] = name
+    if run_id:
+        data["run_id"] = run_id
+    try:
+        result = api.post_multipart(
+            "/artifacts",
+            files={"file": (file.name, file.read_bytes(), mime)},
+            data=data or None)
+        print_success(f"Uploaded '{result['name']}' ({result['size_bytes']:,} bytes) "
+                      f"— id {result['id'][:8]}")
+    except APIError as e:
+        print_error(f"Upload failed: {e}")
+        raise typer.Exit(1)
+
+
+def _resolve_id(ref: str) -> str:
+    """Accept an 8-char prefix from `list`."""
+    if len(ref) >= 36:
+        return ref
+    data = api.get("/artifacts")
+    matches = [a["id"] for a in data.get("artifacts", []) if a["id"].startswith(ref)]
+    if len(matches) != 1:
+        print_error(f"Prefix '{ref}' matches {len(matches)} artifacts.")
+        raise typer.Exit(1)
+    return matches[0]
+
+
+@app.command("download")
+def download_artifact(
+    ref: str = typer.Argument(..., help="Artifact id (full or prefix)"),
+    output: Optional[Path] = typer.Option(None, "--output", "-o", help="Output path"),
+):
+    """Download an artifact's content."""
+    ensure_authenticated()
+    try:
+        art_id = _resolve_id(ref)
+        meta = api.get(f"/artifacts/{art_id}")
+        content = api.get_bytes(f"/artifacts/{art_id}/content")
+    except APIError as e:
+        print_error(f"Download failed: {e}")
+        raise typer.Exit(1)
+    dest = output or Path(meta["name"])
+    dest.write_bytes(content)
+    print_success(f"Saved {len(content):,} bytes to {dest}")
+
+
+@app.command("show")
+def show_artifact(
+    ref: str = typer.Argument(..., help="Artifact id (full or prefix)"),
+):
+    """Show an artifact's metadata."""
+    ensure_authenticated()
+    try:
+        meta = api.get(f"/artifacts/{_resolve_id(ref)}")
+    except APIError as e:
+        print_error(f"Failed: {e}")
+        raise typer.Exit(1)
+    console.print_json(json.dumps(meta))
+
+
+@app.command("remove")
+def remove_artifact(
+    ref: str = typer.Argument(..., help="Artifact id (full or prefix)"),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation"),
+):
+    """Delete an artifact."""
+    ensure_authenticated()
+    art_id = _resolve_id(ref)
+    if not yes and not typer.confirm(f"Delete artifact {art_id[:8]}?"):
+        raise typer.Exit(0)
+    try:
+        api.delete(f"/artifacts/{art_id}")
+        print_success("Artifact deleted.")
+    except APIError as e:
+        print_error(f"Failed to delete: {e}")
+        raise typer.Exit(1)

bonito_cli-0.9.4/bonito_cli/commands/evals.py

@@ -0,0 +1,183 @@
+"""Evals — quality checks over agent/harness runs (deterministic + LLM judge)."""
+
+import json
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from ..api import api, APIError
+from ..utils.auth import ensure_authenticated
+from ..utils.display import print_error, print_info, print_success
+
+console = Console()
+app = typer.Typer(help="✅ Evals — quality checks over agent runs and outputs")
+
+
+@app.command("list")
+def list_evals(
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+):
+    """List the org's evals."""
+    ensure_authenticated()
+    try:
+        data = api.get("/evals")
+    except APIError as e:
+        print_error(f"Failed to list evals: {e}")
+        raise typer.Exit(1)
+    evals = data.get("evals", [])
+    if json_output:
+        console.print_json(json.dumps(evals))
+        return
+    if not evals:
+        print_info("No evals yet — bonito evals add <spec.json|.yaml> to create one.")
+        return
+    table = Table(title="Evals")
+    table.add_column("Name", style="cyan")
+    table.add_column("Checks", justify="right")
+    table.add_column("Agent")
+    table.add_column("Auto")
+    table.add_column("Enabled")
+    for e in evals:
+        table.add_row(e["name"], str(len(e["checks"])),
+                      (e.get("agent_id") or "-")[:8],
+                      "yes" if e["auto"] else "no",
+                      "yes" if e["enabled"] else "no")
+    console.print(table)
+
+
+@app.command("add")
+def add_eval(
+    file: Path = typer.Argument(..., exists=True, readable=True,
+                                help="Eval spec (.json or .yaml): {name, description, checks, agent_id?, auto?}"),
+    update: bool = typer.Option(False, "--update", "-u",
+                                help="Overwrite if the eval already exists"),
+):
+    """Add an eval from a spec file."""
+    ensure_authenticated()
+    raw = file.read_text(encoding="utf-8")
+    if file.suffix in (".yaml", ".yml"):
+        import yaml
+        spec = yaml.safe_load(raw)
+    else:
+        spec = json.loads(raw)
+    try:
+        api.post("/evals", spec)
+        print_success(f"Eval '{spec.get('name')}' added.")
+    except APIError as e:
+        if update and "already exists" in str(e):
+            name = spec.pop("name")
+            api.patch(f"/evals/{name}", spec)
+            print_success(f"Eval '{name}' updated.")
+        else:
+            print_error(f"Failed to add eval: {e}")
+            raise typer.Exit(1)
+
+
+@app.command("show")
+def show_eval(name: str = typer.Argument(..., help="Eval name")):
+    """Show an eval's checks."""
+    ensure_authenticated()
+    try:
+        e = api.get(f"/evals/{name}")
+    except APIError as e2:
+        print_error(f"Failed: {e2}")
+        raise typer.Exit(1)
+    console.print_json(json.dumps(e))
+
+
+@app.command("run")
+def run_eval(
+    name: str = typer.Argument(..., help="Eval name"),
+    run_id: Optional[str] = typer.Option(None, "--run", help="Trace run id to evaluate"),
+    output: Optional[str] = typer.Option(None, "--output", help="Output text to evaluate"),
+    output_file: Optional[Path] = typer.Option(None, "--output-file",
+                                               help="Read output text from a file"),
+    judge_model: Optional[str] = typer.Option(None, "--judge-model",
+                                              help="Model for judge checks"),
+):
+    """Run an eval against a trace run and/or an output text."""
+    ensure_authenticated()
+    payload = {}
+    if run_id:
+        payload["run_id"] = run_id
+    if output_file:
+        payload["output"] = output_file.read_text(encoding="utf-8")
+    elif output:
+        payload["output"] = output
+    if judge_model:
+        payload["judge_model"] = judge_model
+    if not payload.get("run_id") and payload.get("output") is None:
+        print_error("Provide --run and/or --output/--output-file.")
+        raise typer.Exit(1)
+    try:
+        res = api.post(f"/evals/{name}/run", payload)
+    except APIError as e:
+        print_error(f"Eval run failed: {e}")
+        raise typer.Exit(1)
+    verdict = "[green]PASSED[/green]" if res["passed"] else "[red]FAILED[/red]"
+    score = f" (score {res['score']:.0f})" if res.get("score") is not None else ""
+    console.print(f"{verdict}{score}")
+    for c in res["checks"]:
+        mark = {"pass": "[green]✓[/green]", "fail": "[red]✗[/red]"}.get(
+            c["status"], "[dim]○[/dim]")
+        detail = c.get("reasoning") or c.get("detail") or ""
+        extra = ""
+        if "actual" in c:
+            extra = f" (actual {c['actual']}, limit {c.get('limit', '-')})"
+        elif "score" in c:
+            extra = f" (score {c['score']:.0f} vs min {c['min_score']})"
+        console.print(f"  {mark} {c['type']}{extra} {detail}")
+
+
+@app.command("results")
+def eval_results(
+    name: str = typer.Argument(..., help="Eval name"),
+    limit: int = typer.Option(20, "--limit"),
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+):
+    """Show recent results for an eval."""
+    ensure_authenticated()
+    try:
+        data = api.get(f"/evals/{name}/results", params={"limit": limit})
+    except APIError as e:
+        print_error(f"Failed: {e}")
+        raise typer.Exit(1)
+    results = data.get("results", [])
+    if json_output:
+        console.print_json(json.dumps(results))
+        return
+    if not results:
+        print_info("No results yet.")
+        return
+    table = Table(title=f"Results — {name}")
+    table.add_column("When")
+    table.add_column("Verdict")
+    table.add_column("Score", justify="right")
+    table.add_column("Trigger")
+    table.add_column("Run")
+    for r in results:
+        table.add_row((r.get("created_at") or "")[:19],
+                      "[green]pass[/green]" if r["passed"] else "[red]fail[/red]",
+                      f"{r['score']:.0f}" if r.get("score") is not None else "-",
+                      r["trigger"], (r.get("run_id") or "-")[:8])
+    console.print(table)
+
+
+@app.command("remove")
+def remove_eval(
+    name: str = typer.Argument(...),
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation"),
+):
+    """Delete an eval (results cascade)."""
+    ensure_authenticated()
+    if not yes and not typer.confirm(f"Delete eval '{name}' and its results?"):
+        raise typer.Exit(0)
+    try:
+        api.delete(f"/evals/{name}")
+        print_success(f"Eval '{name}' deleted.")
+    except APIError as e:
+        print_error(f"Failed to delete: {e}")
+        raise typer.Exit(1)