benchflow 0.6.7.dev1826__tar.gz → 0.6.7.dev1829__tar.gz

{benchflow-0.6.7.dev1826 → benchflow-0.6.7.dev1829}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1826
+Version: 0.6.7.dev1829
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1826 → benchflow-0.6.7.dev1829}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1826"
+version = "0.6.7.dev1829"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1826 → benchflow-0.6.7.dev1829}/src/benchflow/_utils/benchmark_repos.py

@@ -95,11 +95,21 @@ def _looks_like_commit_sha(ref: str) -> bool:
 
 def _checkout_fetched_ref(repo_root: Path, ref: str) -> None:
     subprocess.run(
-        ["git", "-C", str(repo_root), "fetch", "--depth", "1", "origin", ref],
+        [
+            "git",
+            "-C",
+            str(repo_root),
+            "fetch",
+            "--quiet",
+            "--depth",
+            "1",
+            "origin",
+            ref,
+        ],
         check=True,
     )
     subprocess.run(
-        ["git", "-C", str(repo_root), "checkout", "--detach", "FETCH_HEAD"],
+        ["git", "-C", str(repo_root), "checkout", "--quiet", "--detach", "FETCH_HEAD"],
         check=True,
     )
 
@@ -155,7 +165,10 @@ def _clone_repo_unlocked(org: str, repo: str, ref: str | None = None) -> Path:
     try:
         if clone_tmp.exists():
             shutil.rmtree(clone_tmp)
-        cmd = ["git", "clone", "--depth", "1"]
+        # --quiet: raw clone progress ("remote: Counting objects: …") on the
+        # console pre-dashboard is noise; the "Cloning …" log line above is
+        # the one-line summary. Errors still print (git keeps stderr on fail).
+        cmd = ["git", "clone", "--quiet", "--depth", "1"]
         if ref and not _looks_like_commit_sha(ref):
             cmd.extend(["--branch", ref])
         cmd.extend([url, str(clone_tmp)])

benchflow-0.6.7.dev1829/src/benchflow/_utils/live_activity.py

@@ -0,0 +1,82 @@
+"""Process-global registry of live rollouts for the eval dashboard.
+
+The Rich dashboard (``cli/_live_progress.py``) renders in the same process as
+the engine's rollouts, so surfacing per-task activity needs no new event
+plumbing: the engine registers each live Rollout under its task name and the
+dashboard polls the rollout's ACP session counters — the exact ones the
+console heartbeat (``ACPSession._maybe_log_progress``) logs — at render time.
+Best-effort by contract: every reader failure degrades to "no activity",
+never to a render error.
+"""
+
+from __future__ import annotations
+
+import threading
+from typing import Any, NamedTuple
+
+
+class SessionCounters(NamedTuple):
+    """The live ACP session counters the console heartbeat logs.
+
+    ``total_tokens`` is None until the session has a usage snapshot (i.e.
+    after a completed prompt).
+    """
+
+    tool_calls: int
+    last_tool: str
+    total_tokens: int | None
+
+
+class ActivitySnapshot(NamedTuple):
+    """One dashboard poll of a live rollout.
+
+    ``phase`` is the rollout's lifecycle phase (``Rollout._phase``) and is
+    always present, so the activity cell can label the long non-agent
+    stretches — sandbox create, agent install, verify — instead of blanking
+    in a way that is indistinguishable from a hang. ``counters`` is None
+    until the agent session exists (session-factory agents never grow one).
+    """
+
+    phase: str
+    counters: SessionCounters | None
+
+
+_lock = threading.Lock()
+# Task name -> live Rollout. Task basenames are unique keys by engine-wide
+# invariant: the whole run pipeline (resume filtering, result keying — see
+# Evaluation's `d.name not in completed` plan filter) is keyed by basename,
+# so each name is scheduled at most once at a time.
+_live: dict[str, Any] = {}
+
+
+def register(task_name: str, rollout: Any) -> None:
+    """Expose a live rollout to the dashboard under its task name."""
+    with _lock:
+        _live[task_name] = rollout
+
+
+def unregister(task_name: str) -> None:
+    with _lock:
+        _live.pop(task_name, None)
+
+
+def activity(task_name: str) -> ActivitySnapshot | None:
+    """The :class:`ActivitySnapshot` for a running task, or None when the
+    rollout isn't registered (pre-create, between retries, teardown).
+
+    ``counters`` inside the snapshot is None until the task's agent session
+    exists — including non-ACP (session-factory) agents, which never grow an
+    ACP client — but ``phase`` is always present. The dig into
+    client/session state lives on ``Rollout.activity_snapshot()`` (typed,
+    owner-side); the except here only guards renders racing rollout teardown.
+    """
+    with _lock:
+        rollout = _live.get(task_name)
+    if rollout is None:
+        return None
+    try:
+        return rollout.activity_snapshot()
+    except Exception:
+        # Dashboard reads race a live rollout; degrading to "no activity" is
+        # always preferable to perturbing the render.
+        return None

{benchflow-0.6.7.dev1826 → benchflow-0.6.7.dev1829}/src/benchflow/acp/session.py

@@ -221,6 +221,20 @@ class ACPSession:
             # which is otherwise easy to miss in a 64-concurrency log.
             logger.error(f"ACPSession on_change callback failed: {e}")
 
+    def progress_snapshot(self) -> tuple[int, str]:
+        """(tool-call count, last tool title) — the console heartbeat's
+        counters, also polled by the live eval dashboard's activity cell.
+
+        The title is the raw first line (untruncated — display width belongs
+        to each render site); ``split`` not ``splitlines`` so a whitespace-only
+        title strips to ``""`` instead of raising on an empty line list.
+        """
+        title = ""
+        if self.tool_calls:
+            last = self.tool_calls[-1]
+            title = (last.title or last.kind or "").strip().split("\n", 1)[0]
+        return len(self.tool_calls), title
+
     def _maybe_log_progress(self) -> None:
         if not self._progress_enabled or self._prompt_started_at is None:
             return
@@ -229,12 +243,10 @@ class ACPSession:
             return
         self._last_progress_at = now
         elapsed_min = (now - self._prompt_started_at) / 60.0
-        line = f"  … {elapsed_min:.1f}min, {len(self.tool_calls)} tool calls"
-        if self.tool_calls:
-            last = self.tool_calls[-1]
-            title = (last.title or last.kind or "").strip().splitlines()[0][:60]
-            if title:
-                line += f" (last: {title})"
+        calls, title = self.progress_snapshot()
+        line = f"  … {elapsed_min:.1f}min, {calls} tool calls"
+        if title:
+            line += f" (last: {title[:60]})"
         logger.info(line)
 
     def record_user_prompt(self, text: str) -> None:

{benchflow-0.6.7.dev1826 → benchflow-0.6.7.dev1829}/src/benchflow/cli/_live_progress.py

@@ -27,6 +27,7 @@ from rich.markup import escape
 from rich.table import Table
 from rich.text import Text
 
+from benchflow._utils import live_activity
 from benchflow.cli._shared import console
 from benchflow.usage_tracking import is_trusted_usage_source
 
@@ -127,6 +128,52 @@ def _fmt_tokens(n: int) -> str:
     return str(n)
 
 
+# Rollout lifecycle phase (Rollout._phase — the completed phase, except
+# "verifying" which verify() marks at entry) -> what is running now. Shown
+# while the session counters are unavailable — before the agent session
+# exists and after it is torn down — so sandbox create, agent install, and
+# the verifier read as work, not as a hang.
+_PHASE_LABELS = {
+    "created": "creating sandbox…",
+    "setup": "creating sandbox…",
+    "started": "installing agent…",
+    "installed": "running agent…",
+    "connected": "running agent…",
+    "executed": "verifying…",
+    "verifying": "verifying…",
+    "verified": "cleaning up…",
+    "cleaned": "cleaning up…",
+}
+# Unknown phases (e.g. "branched") and pre-register/teardown races: still
+# never blank — a blank cell on a live row reads as a hang.
+_PHASE_FALLBACK = "starting…"
+
+
+def _activity_cell(name: str) -> str:
+    """Per-task activity for the "running now" table, e.g.
+    ``38 calls · last: file_editor``.
+
+    Polls the live rollout's ACP session counters — the same ones the console
+    heartbeat logs, which the Live display otherwise mutes — via the
+    live-activity registry. Until the agent session exists (and after it is
+    closed for the verifier) the cell shows the rollout's lifecycle phase as a
+    label instead, so the row is never blank. Tokens appear only once the
+    session has a usage snapshot (i.e. after a completed prompt).
+    """
+    snap = live_activity.activity(name)
+    if snap is None:
+        return _PHASE_FALLBACK
+    counters = snap.counters
+    if counters is None:
+        return _PHASE_LABELS.get(snap.phase, _PHASE_FALLBACK)
+    cell = f"{counters.tool_calls} calls"
+    if counters.total_tokens:
+        cell += f" · {_fmt_tokens(counters.total_tokens)} tok"
+    if counters.last_tool:
+        cell += f" · last: {counters.last_tool[:30]}"
+    return cell
+
+
 class LiveEvalProgress:
     """A live ``Live``-rendered dashboard, driven by the engine's progress hooks.
 
@@ -295,17 +342,23 @@ class LiveEvalProgress:
             )
             tbl.add_column("running now", no_wrap=True)
             tbl.add_column("elapsed", justify="right")
+            tbl.add_column("activity", no_wrap=True)
             now = time.monotonic()
             for name in sorted(running, key=lambda n: running[n])[:_MAX_RUNNING_ROWS]:
                 # Text() so a task name containing Rich markup (`[` is legal in
                 # SkillsBench dir names) is rendered literally, not parsed as
                 # markup — a MarkupError here escapes __rich__ and aborts the
                 # CLI on live-context exit, violating this module's "a render
-                # bug can never perturb a run" contract.
-                tbl.add_row(Text(name), _fmt_dur(now - running[name]))
+                # bug can never perturb a run" contract. Same for the activity
+                # cell, which carries agent-authored tool titles.
+                tbl.add_row(
+                    Text(name),
+                    _fmt_dur(now - running[name]),
+                    Text(_activity_cell(name), style="dim"),
+                )
             extra = len(running) - _MAX_RUNNING_ROWS
             if extra > 0:
-                tbl.add_row(f"… {extra} more", "")
+                tbl.add_row(f"… {extra} more", "", "")
             parts.append(tbl)
 
         # Footer: pass-rate (excl errors) + token/cost economics. Show "—" (not

{benchflow-0.6.7.dev1826 → benchflow-0.6.7.dev1829}/src/benchflow/cli/_shared.py

@@ -18,10 +18,12 @@ import typer
 from rich.console import Console
 from rich.markup import escape
 
+from benchflow._utils.text import truncate_end
+
 if TYPE_CHECKING:
     from pathlib import Path
 
-    from benchflow.evaluation import EvaluationResult
+    from benchflow.evaluation import EvaluationResult, TaskFailure
 
 console = Console()
 
@@ -100,14 +102,52 @@ def _exit_if_evaluation_had_errors(result: object) -> None:
         raise typer.Exit(1)
 
 
+# Final-block failure lines: keep the block skimmable on big jobs and each
+# line inside a typical terminal width.
+_MAX_FAILURE_LINES = 5
+_FAILURE_LINE_LIMIT = 100
+_FAILURE_REASON_METRICS = 3
+
+
+def _failure_reason(failure: TaskFailure) -> str:
+    """One cheap line explaining why a FAILED (scored, reward != 1) task
+    failed, from evidence already on the result — no file reads.
+
+    Priority: the verifier's own error if set; else the reward plus a compact
+    breakdown of the named metrics in the reward dict (zero/failed metrics
+    first — they explain the miss); else just the reward.
+    """
+    if failure.verifier_error:
+        # Collapse whitespace: verifier errors are routinely multi-line.
+        return " ".join(failure.verifier_error.split())
+    rewards = failure.rewards or {}
+    reward = rewards.get("reward")
+    metrics = [
+        (name, value)
+        for name, value in rewards.items()
+        if name != "reward" and isinstance(value, (bool, int, float))
+    ]
+    if not metrics:
+        return f"reward {reward}"
+    # Zero/failed metrics first (stable within each group), capped so one
+    # metric-happy verifier can't flood the line.
+    metrics.sort(key=lambda kv: kv[1] != 0)
+    shown = ", ".join(
+        f"{name} {value}" for name, value in metrics[:_FAILURE_REASON_METRICS]
+    )
+    return f"reward {reward} — {shown}"
+
+
 def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -> None:
     """Print the Score/errors summary line, colored by outcome, plus artifacts.
 
     A clean pass and a total failure used to look identical (both bold white);
     now the line is green only on a full clean pass, red on a shutout, amber
-    otherwise, and ``errors=N`` is red when non-zero. When ``job_dir`` is given,
-    the result/summary paths are printed so testers know where to look (the
-    guide repeatedly says "read summary.json" but the CLI never said where).
+    otherwise, and ``errors=N`` is red when non-zero. Each FAILED task gets one
+    dim ``✗ task: reason`` line (capped at ``_MAX_FAILURE_LINES``) so the "why"
+    doesn't require opening summary.json. When ``job_dir`` is given, the
+    result/summary paths are printed so testers know where to look (the guide
+    repeatedly says "read summary.json" but the CLI never said where).
     """
     errors = int(getattr(result, "errored", 0) or 0)
     verifier_errors = int(getattr(result, "verifier_errored", 0) or 0)
@@ -132,6 +172,19 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
         f"\n[{style}]{mark} Score: {result.passed}/{result.total} "
         f"({result.score:.1%})[/{style}]{err_part}"
     )
+    # One dim reason line per FAILED task, so "0/1" doesn't force a dig into
+    # summary.json to learn why. getattr(): sharded aggregation and older
+    # SimpleNamespace-style callers don't carry task_failures.
+    failures = getattr(result, "task_failures", None) or []
+    for failure in failures[:_MAX_FAILURE_LINES]:
+        line = truncate_end(
+            f"  ✗ {failure.task_name}: {_failure_reason(failure)}",
+            _FAILURE_LINE_LIMIT,
+        )
+        console.print(f"[dim]{escape(line)}[/dim]")
+    extra = len(failures) - _MAX_FAILURE_LINES
+    if extra > 0:
+        console.print(f"[dim]  … and {extra} more[/dim]")
     if job_dir is not None:
         console.print(f"[dim]Artifacts:[/dim] {escape(str(job_dir))}")
         console.print(f"[dim]Summary:  [/dim] {escape(str(job_dir))}/summary.json")

{benchflow-0.6.7.dev1826 → benchflow-0.6.7.dev1829}/src/benchflow/cli/main.py

@@ -389,8 +389,9 @@ def eval_run(
         typer.Option(
             "--quiet",
             help=(
-                "Suppress the per-run console progress heartbeat (the "
-                "'… Nmin, K tool calls' lines during agent execution)."
+                "Suppress live progress output: the Rich dashboard on a TTY "
+                "and the per-run console progress heartbeat (the '… Nmin, K "
+                "tool calls' lines during agent execution)."
             ),
         ),
     ] = False,
@@ -597,6 +598,9 @@ def eval_run(
         # Explicit off beats the session layer's auto-gate; see
         # benchflow.acp.session._console_progress_enabled.
         os.environ["BENCHFLOW_PROGRESS"] = "off"
+        # --quiet silences ALL live progress: the heartbeat lines above and
+        # the TTY Rich dashboard (see cli._live_progress.progress_enabled).
+        os.environ["BENCHFLOW_NO_PROGRESS"] = "1"
 
     request = EvalCreateRequest(
         config_file=config_file,

{benchflow-0.6.7.dev1826 → benchflow-0.6.7.dev1829}/src/benchflow/evaluation.py

@@ -60,6 +60,7 @@ from benchflow._utils.scoring import (
     VERIFIER_TIMEOUT,
     api_error_is_transient,
     classify_error,
+    classify_score_outcome,
     classify_verifier_error,
     count_audit_outcomes,
     count_score_outcomes,
@@ -574,6 +575,21 @@ class EvaluationConfig:
             )
 
 
+@dataclass(frozen=True)
+class TaskFailure:
+    """Cheap failure evidence for one FAILED (scored, reward != 1) task.
+
+    Carried on :class:`EvaluationResult` so the CLI's final block can print a
+    one-line reason per failed task from data the engine already holds —
+    without re-reading result.json files. Errored tasks are excluded (they
+    already surface through the error counters and warning replay).
+    """
+
+    task_name: str
+    rewards: dict[str, Any] | None
+    verifier_error: str | None
+
+
 @dataclass
 class EvaluationResult:
     """Aggregated results for a job."""
@@ -588,6 +604,7 @@ class EvaluationResult:
     elapsed_sec: float = 0.0
     memory_score: float | None = None
     memory_scores: dict[str, float] = field(default_factory=dict)
+    task_failures: list[TaskFailure] = field(default_factory=list)
 
     @property
     def score(self) -> float:
@@ -1250,11 +1267,20 @@ class Evaluation:
 
             return await run_self_gen(rollout_config)
         rollout = await Rollout.create(rollout_config)
-        # Rollout.run() enforces its own host-side hard deadline against
-        # awaits wedged below the phase-level timeouts — see
-        # benchflow.rollout._deadline. A trip surfaces here as a normal
-        # infra-retryable error result.
-        return await rollout.run()
+        # Expose the live rollout to the eval dashboard's activity cell —
+        # a same-process poll of the session's heartbeat counters, see
+        # benchflow._utils.live_activity.
+        from benchflow._utils import live_activity
+
+        live_activity.register(task_dir.name, rollout)
+        try:
+            # Rollout.run() enforces its own host-side hard deadline against
+            # awaits wedged below the phase-level timeouts — see
+            # benchflow.rollout._deadline. A trip surfaces here as a normal
+            # infra-retryable error result.
+            return await rollout.run()
+        finally:
+            live_activity.unregister(task_dir.name)
 
     async def _run_single_task_legacy(
         self, task_dir: Path, cfg: EvaluationConfig
@@ -1767,6 +1793,18 @@ class Evaluation:
         score_counts = count_score_outcomes(all_results.values())
         audit_counts = count_audit_outcomes(all_results.values())
         memory, memory_scores = memory_summary(all_results)
+        # Per-task failure evidence for the CLI's final block — FAILED (scored,
+        # reward != 1) tasks only, from data already in memory. Sorted by name
+        # so the printed lines are deterministic across resume/concurrency.
+        task_failures = [
+            TaskFailure(
+                task_name=name,
+                rewards=r.get("rewards"),
+                verifier_error=r.get("verifier_error"),
+            )
+            for name, r in sorted(all_results.items())
+            if classify_score_outcome(r) == "failed"
+        ]
         job_result = EvaluationResult(
             job_name=self._job_name,
             config=cfg,
@@ -1782,6 +1820,7 @@ class Evaluation:
             elapsed_sec=elapsed,
             memory_score=memory["avg_score"],
             memory_scores=memory_scores,
+            task_failures=task_failures,
         )
 
         assert (

{benchflow-0.6.7.dev1826 → benchflow-0.6.7.dev1829}/src/benchflow/rollout/__init__.py

@@ -72,6 +72,7 @@ from benchflow._types import Role, Scene, Turn
 # / ``_capture_session_trajectory`` / ``default_rollout_planes`` keep affecting
 # the ``Rollout`` methods that call those names, because those methods stay
 # defined in this module.
+from benchflow._utils.live_activity import ActivitySnapshot, SessionCounters
 from benchflow._utils.scoring import classify_error as classify_error
 from benchflow.acp.types import McpServerSpec
 from benchflow.agents.credentials import upload_credential
@@ -755,6 +756,27 @@ class Rollout:
     def acp_client(self) -> Any:
         return self._acp_client
 
+    def activity_snapshot(self) -> ActivitySnapshot:
+        """The eval dashboard's per-task :class:`ActivitySnapshot`: the
+        current lifecycle phase plus the live :class:`SessionCounters` (tool
+        calls, last tool title, total tokens). ``counters`` is None before
+        the agent session exists — the phase then carries the cell
+        ("creating sandbox…", "verifying…"), so a 90s sandbox create is not
+        indistinguishable from a hang.
+
+        Rollout owns the client/session dig so a rename breaks here — in
+        typed, tested code — instead of silently blanking the dashboard cell
+        (see benchflow._utils.live_activity). Session-factory agents have no
+        ACP client and always report counter-less snapshots.
+        """
+        session = self._acp_client.session if self._acp_client else None
+        if session is None:
+            return ActivitySnapshot(self._phase, None)
+        calls, last_title = session.progress_snapshot()
+        usage = session.latest_usage_totals()
+        tokens = usage.get("total_tokens") if usage else None
+        return ActivitySnapshot(self._phase, SessionCounters(calls, last_title, tokens))
+
     @property
     def trajectory(self) -> list[dict]:
         return self._trajectory
@@ -1720,6 +1742,12 @@ class Rollout:
         """Run the verifier and return rewards."""
         cfg = self._config
 
+        # Mark the phase at entry (the other transitions mark completion):
+        # the verifier can run for minutes after disconnect() reset the phase
+        # to "installed", and the dashboard's activity cell reads _phase to
+        # label that stretch "verifying…" instead of going blank.
+        self._phase = "verifying"
+
         if not self._trajectory and cfg.primary_agent != "oracle":
             scraped = await _scrape_agent_trajectory(
                 self._env, cfg.primary_agent, cfg.sandbox_user
@@ -2401,7 +2429,11 @@ class Rollout:
         TrajectoryWriter(
             self._rollout_dir / "trajectory" / "acp_trajectory.jsonl"
         ).write_final(self._trajectory)
-        logger.warning(
+        # info, not warning: trajectory repair is evidence-mutation an auditor
+        # should find at default (non-TTY/CI) verbosity, but as a warning it
+        # survived the live dashboard's WARNING+ replay and printed between
+        # teardown and the score line (dogfood 2026-08-09).
+        logger.info(
             "Repaired %d lossy ACP tool event(s) from trusted provider capture",
             repaired,
         )