benchflow 0.6.7.dev1827__tar.gz → 0.6.7.dev1829__tar.gz

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1829}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1827
+Version: 0.6.7.dev1829
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1829}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1827"
+version = "0.6.7.dev1829"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1829}/src/benchflow/_utils/live_activity.py

@@ -12,7 +12,34 @@ never to a render error.
 from __future__ import annotations
 
 import threading
-from typing import Any
+from typing import Any, NamedTuple
+
+
+class SessionCounters(NamedTuple):
+    """The live ACP session counters the console heartbeat logs.
+
+    ``total_tokens`` is None until the session has a usage snapshot (i.e.
+    after a completed prompt).
+    """
+
+    tool_calls: int
+    last_tool: str
+    total_tokens: int | None
+
+
+class ActivitySnapshot(NamedTuple):
+    """One dashboard poll of a live rollout.
+
+    ``phase`` is the rollout's lifecycle phase (``Rollout._phase``) and is
+    always present, so the activity cell can label the long non-agent
+    stretches — sandbox create, agent install, verify — instead of blanking
+    in a way that is indistinguishable from a hang. ``counters`` is None
+    until the agent session exists (session-factory agents never grow one).
+    """
+
+    phase: str
+    counters: SessionCounters | None
+
 
 _lock = threading.Lock()
 # Task name -> live Rollout. Task basenames are unique keys by engine-wide
@@ -33,11 +60,13 @@ def unregister(task_name: str) -> None:
         _live.pop(task_name, None)
 
 
-def activity(task_name: str) -> tuple[int, str, int | None] | None:
-    """(tool calls, last tool title, total tokens) for a running task.
+def activity(task_name: str) -> ActivitySnapshot | None:
+    """The :class:`ActivitySnapshot` for a running task, or None when the
+    rollout isn't registered (pre-create, between retries, teardown).
 
-    None until the task's agent session exists — including non-ACP
-    (session-factory) agents, which never grow an ACP client. The dig into
+    ``counters`` inside the snapshot is None until the task's agent session
+    exists — including non-ACP (session-factory) agents, which never grow an
+    ACP client — but ``phase`` is always present. The dig into
     client/session state lives on ``Rollout.activity_snapshot()`` (typed,
     owner-side); the except here only guards renders racing rollout teardown.
     """

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1829}/src/benchflow/cli/_live_progress.py

@@ -128,24 +128,49 @@ def _fmt_tokens(n: int) -> str:
     return str(n)
 
 
+# Rollout lifecycle phase (Rollout._phase — the completed phase, except
+# "verifying" which verify() marks at entry) -> what is running now. Shown
+# while the session counters are unavailable — before the agent session
+# exists and after it is torn down — so sandbox create, agent install, and
+# the verifier read as work, not as a hang.
+_PHASE_LABELS = {
+    "created": "creating sandbox…",
+    "setup": "creating sandbox…",
+    "started": "installing agent…",
+    "installed": "running agent…",
+    "connected": "running agent…",
+    "executed": "verifying…",
+    "verifying": "verifying…",
+    "verified": "cleaning up…",
+    "cleaned": "cleaning up…",
+}
+# Unknown phases (e.g. "branched") and pre-register/teardown races: still
+# never blank — a blank cell on a live row reads as a hang.
+_PHASE_FALLBACK = "starting…"
+
+
 def _activity_cell(name: str) -> str:
     """Per-task activity for the "running now" table, e.g.
     ``38 calls · last: file_editor``.
 
     Polls the live rollout's ACP session counters — the same ones the console
     heartbeat logs, which the Live display otherwise mutes — via the
-    live-activity registry. Empty until the agent session exists. Tokens appear
-    only once the session has a usage snapshot (i.e. after a completed prompt).
+    live-activity registry. Until the agent session exists (and after it is
+    closed for the verifier) the cell shows the rollout's lifecycle phase as a
+    label instead, so the row is never blank. Tokens appear only once the
+    session has a usage snapshot (i.e. after a completed prompt).
     """
     snap = live_activity.activity(name)
     if snap is None:
-        return ""
-    calls, last_title, tokens = snap
-    cell = f"{calls} calls"
-    if tokens:
-        cell += f" · {_fmt_tokens(tokens)} tok"
-    if last_title:
-        cell += f" · last: {last_title[:30]}"
+        return _PHASE_FALLBACK
+    counters = snap.counters
+    if counters is None:
+        return _PHASE_LABELS.get(snap.phase, _PHASE_FALLBACK)
+    cell = f"{counters.tool_calls} calls"
+    if counters.total_tokens:
+        cell += f" · {_fmt_tokens(counters.total_tokens)} tok"
+    if counters.last_tool:
+        cell += f" · last: {counters.last_tool[:30]}"
     return cell
 
 

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1829}/src/benchflow/cli/_shared.py

@@ -18,10 +18,12 @@ import typer
 from rich.console import Console
 from rich.markup import escape
 
+from benchflow._utils.text import truncate_end
+
 if TYPE_CHECKING:
     from pathlib import Path
 
-    from benchflow.evaluation import EvaluationResult
+    from benchflow.evaluation import EvaluationResult, TaskFailure
 
 console = Console()
 
@@ -100,14 +102,52 @@ def _exit_if_evaluation_had_errors(result: object) -> None:
         raise typer.Exit(1)
 
 
+# Final-block failure lines: keep the block skimmable on big jobs and each
+# line inside a typical terminal width.
+_MAX_FAILURE_LINES = 5
+_FAILURE_LINE_LIMIT = 100
+_FAILURE_REASON_METRICS = 3
+
+
+def _failure_reason(failure: TaskFailure) -> str:
+    """One cheap line explaining why a FAILED (scored, reward != 1) task
+    failed, from evidence already on the result — no file reads.
+
+    Priority: the verifier's own error if set; else the reward plus a compact
+    breakdown of the named metrics in the reward dict (zero/failed metrics
+    first — they explain the miss); else just the reward.
+    """
+    if failure.verifier_error:
+        # Collapse whitespace: verifier errors are routinely multi-line.
+        return " ".join(failure.verifier_error.split())
+    rewards = failure.rewards or {}
+    reward = rewards.get("reward")
+    metrics = [
+        (name, value)
+        for name, value in rewards.items()
+        if name != "reward" and isinstance(value, (bool, int, float))
+    ]
+    if not metrics:
+        return f"reward {reward}"
+    # Zero/failed metrics first (stable within each group), capped so one
+    # metric-happy verifier can't flood the line.
+    metrics.sort(key=lambda kv: kv[1] != 0)
+    shown = ", ".join(
+        f"{name} {value}" for name, value in metrics[:_FAILURE_REASON_METRICS]
+    )
+    return f"reward {reward} — {shown}"
+
+
 def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -> None:
     """Print the Score/errors summary line, colored by outcome, plus artifacts.
 
     A clean pass and a total failure used to look identical (both bold white);
     now the line is green only on a full clean pass, red on a shutout, amber
-    otherwise, and ``errors=N`` is red when non-zero. When ``job_dir`` is given,
-    the result/summary paths are printed so testers know where to look (the
-    guide repeatedly says "read summary.json" but the CLI never said where).
+    otherwise, and ``errors=N`` is red when non-zero. Each FAILED task gets one
+    dim ``✗ task: reason`` line (capped at ``_MAX_FAILURE_LINES``) so the "why"
+    doesn't require opening summary.json. When ``job_dir`` is given, the
+    result/summary paths are printed so testers know where to look (the guide
+    repeatedly says "read summary.json" but the CLI never said where).
     """
     errors = int(getattr(result, "errored", 0) or 0)
     verifier_errors = int(getattr(result, "verifier_errored", 0) or 0)
@@ -132,6 +172,19 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
         f"\n[{style}]{mark} Score: {result.passed}/{result.total} "
         f"({result.score:.1%})[/{style}]{err_part}"
     )
+    # One dim reason line per FAILED task, so "0/1" doesn't force a dig into
+    # summary.json to learn why. getattr(): sharded aggregation and older
+    # SimpleNamespace-style callers don't carry task_failures.
+    failures = getattr(result, "task_failures", None) or []
+    for failure in failures[:_MAX_FAILURE_LINES]:
+        line = truncate_end(
+            f"  ✗ {failure.task_name}: {_failure_reason(failure)}",
+            _FAILURE_LINE_LIMIT,
+        )
+        console.print(f"[dim]{escape(line)}[/dim]")
+    extra = len(failures) - _MAX_FAILURE_LINES
+    if extra > 0:
+        console.print(f"[dim]  … and {extra} more[/dim]")
     if job_dir is not None:
         console.print(f"[dim]Artifacts:[/dim] {escape(str(job_dir))}")
         console.print(f"[dim]Summary:  [/dim] {escape(str(job_dir))}/summary.json")

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1829}/src/benchflow/cli/main.py

@@ -389,8 +389,9 @@ def eval_run(
         typer.Option(
             "--quiet",
             help=(
-                "Suppress the per-run console progress heartbeat (the "
-                "'… Nmin, K tool calls' lines during agent execution)."
+                "Suppress live progress output: the Rich dashboard on a TTY "
+                "and the per-run console progress heartbeat (the '… Nmin, K "
+                "tool calls' lines during agent execution)."
             ),
         ),
     ] = False,
@@ -597,6 +598,9 @@ def eval_run(
         # Explicit off beats the session layer's auto-gate; see
         # benchflow.acp.session._console_progress_enabled.
         os.environ["BENCHFLOW_PROGRESS"] = "off"
+        # --quiet silences ALL live progress: the heartbeat lines above and
+        # the TTY Rich dashboard (see cli._live_progress.progress_enabled).
+        os.environ["BENCHFLOW_NO_PROGRESS"] = "1"
 
     request = EvalCreateRequest(
         config_file=config_file,

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1829}/src/benchflow/evaluation.py

@@ -60,6 +60,7 @@ from benchflow._utils.scoring import (
     VERIFIER_TIMEOUT,
     api_error_is_transient,
     classify_error,
+    classify_score_outcome,
     classify_verifier_error,
     count_audit_outcomes,
     count_score_outcomes,
@@ -574,6 +575,21 @@ class EvaluationConfig:
             )
 
 
+@dataclass(frozen=True)
+class TaskFailure:
+    """Cheap failure evidence for one FAILED (scored, reward != 1) task.
+
+    Carried on :class:`EvaluationResult` so the CLI's final block can print a
+    one-line reason per failed task from data the engine already holds —
+    without re-reading result.json files. Errored tasks are excluded (they
+    already surface through the error counters and warning replay).
+    """
+
+    task_name: str
+    rewards: dict[str, Any] | None
+    verifier_error: str | None
+
+
 @dataclass
 class EvaluationResult:
     """Aggregated results for a job."""
@@ -588,6 +604,7 @@ class EvaluationResult:
     elapsed_sec: float = 0.0
     memory_score: float | None = None
     memory_scores: dict[str, float] = field(default_factory=dict)
+    task_failures: list[TaskFailure] = field(default_factory=list)
 
     @property
     def score(self) -> float:
@@ -1776,6 +1793,18 @@ class Evaluation:
         score_counts = count_score_outcomes(all_results.values())
         audit_counts = count_audit_outcomes(all_results.values())
         memory, memory_scores = memory_summary(all_results)
+        # Per-task failure evidence for the CLI's final block — FAILED (scored,
+        # reward != 1) tasks only, from data already in memory. Sorted by name
+        # so the printed lines are deterministic across resume/concurrency.
+        task_failures = [
+            TaskFailure(
+                task_name=name,
+                rewards=r.get("rewards"),
+                verifier_error=r.get("verifier_error"),
+            )
+            for name, r in sorted(all_results.items())
+            if classify_score_outcome(r) == "failed"
+        ]
         job_result = EvaluationResult(
             job_name=self._job_name,
             config=cfg,
@@ -1791,6 +1820,7 @@ class Evaluation:
             elapsed_sec=elapsed,
             memory_score=memory["avg_score"],
             memory_scores=memory_scores,
+            task_failures=task_failures,
         )
 
         assert (

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1829}/src/benchflow/rollout/__init__.py

@@ -72,6 +72,7 @@ from benchflow._types import Role, Scene, Turn
 # / ``_capture_session_trajectory`` / ``default_rollout_planes`` keep affecting
 # the ``Rollout`` methods that call those names, because those methods stay
 # defined in this module.
+from benchflow._utils.live_activity import ActivitySnapshot, SessionCounters
 from benchflow._utils.scoring import classify_error as classify_error
 from benchflow.acp.types import McpServerSpec
 from benchflow.agents.credentials import upload_credential
@@ -755,22 +756,26 @@ class Rollout:
     def acp_client(self) -> Any:
         return self._acp_client
 
-    def activity_snapshot(self) -> tuple[int, str, int | None] | None:
-        """(tool calls, last tool title, total tokens) for the eval
-        dashboard's activity cell, or None before the agent session exists.
+    def activity_snapshot(self) -> ActivitySnapshot:
+        """The eval dashboard's per-task :class:`ActivitySnapshot`: the
+        current lifecycle phase plus the live :class:`SessionCounters` (tool
+        calls, last tool title, total tokens). ``counters`` is None before
+        the agent session exists — the phase then carries the cell
+        ("creating sandbox…", "verifying…"), so a 90s sandbox create is not
+        indistinguishable from a hang.
 
         Rollout owns the client/session dig so a rename breaks here — in
         typed, tested code — instead of silently blanking the dashboard cell
         (see benchflow._utils.live_activity). Session-factory agents have no
-        ACP client and always return None.
+        ACP client and always report counter-less snapshots.
         """
         session = self._acp_client.session if self._acp_client else None
         if session is None:
-            return None
+            return ActivitySnapshot(self._phase, None)
         calls, last_title = session.progress_snapshot()
         usage = session.latest_usage_totals()
         tokens = usage.get("total_tokens") if usage else None
-        return calls, last_title, tokens
+        return ActivitySnapshot(self._phase, SessionCounters(calls, last_title, tokens))
 
     @property
     def trajectory(self) -> list[dict]:
@@ -1737,6 +1742,12 @@ class Rollout:
         """Run the verifier and return rewards."""
         cfg = self._config
 
+        # Mark the phase at entry (the other transitions mark completion):
+        # the verifier can run for minutes after disconnect() reset the phase
+        # to "installed", and the dashboard's activity cell reads _phase to
+        # label that stretch "verifying…" instead of going blank.
+        self._phase = "verifying"
+
         if not self._trajectory and cfg.primary_agent != "oracle":
             scraped = await _scrape_agent_trajectory(
                 self._env, cfg.primary_agent, cfg.sandbox_user

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1829}/tests/test_cli_live_progress.py

@@ -6,6 +6,7 @@ exercised for "doesn't raise" rather than pixel-asserted.
 
 from __future__ import annotations
 
+import contextlib
 import logging
 from types import SimpleNamespace
 
@@ -105,28 +106,43 @@ def test_activity_cell_polls_live_session_counters():
     # 2026-08-09): the heartbeat's counters must reach the dashboard, since
     # the Live mutes the logged heartbeat line itself.
     from benchflow._utils import live_activity
+    from benchflow._utils.live_activity import ActivitySnapshot, SessionCounters
     from benchflow.cli._live_progress import _activity_cell
 
     live_activity.register(
-        "edit-pdf", SimpleNamespace(activity_snapshot=lambda: (38, "file_editor", 1500))
+        "edit-pdf",
+        SimpleNamespace(
+            activity_snapshot=lambda: ActivitySnapshot(
+                "connected", SessionCounters(38, "file_editor", 1500)
+            )
+        ),
     )
     try:
         assert _activity_cell("edit-pdf") == "38 calls · 1.5k tok · last: file_editor"
     finally:
         live_activity.unregister("edit-pdf")
-    # Unregistered tasks (and pre-session rollouts) degrade to empty, not raise.
-    assert _activity_cell("edit-pdf") == ""
+    # Unregistered tasks (pre-register, teardown races) degrade to the
+    # fallback label — a live row's cell must never be blank, and never raise.
+    assert _activity_cell("edit-pdf") == "starting…"
 
 
 def test_rollout_activity_snapshot_reads_acp_session():
     # The client/session dig lives on Rollout (typed, owner-side) so a rename
     # of session counters breaks HERE instead of silently blanking the cell.
+    from benchflow._utils.live_activity import ActivitySnapshot, SessionCounters
     from benchflow.rollout import Rollout
 
-    connected = SimpleNamespace(_acp_client=SimpleNamespace(session=_FakeSession()))
-    assert Rollout.activity_snapshot(connected) == (38, "file_editor", 1500)
-    # Pre-connect (and session-factory) rollouts have no client -> None.
-    assert Rollout.activity_snapshot(SimpleNamespace(_acp_client=None)) is None
+    connected = SimpleNamespace(
+        _acp_client=SimpleNamespace(session=_FakeSession()), _phase="connected"
+    )
+    assert Rollout.activity_snapshot(connected) == ActivitySnapshot(
+        "connected", SessionCounters(38, "file_editor", 1500)
+    )
+    # Pre-connect (and session-factory) rollouts have no client: counters are
+    # None but the lifecycle phase still rides out so the cell can label it.
+    assert Rollout.activity_snapshot(
+        SimpleNamespace(_acp_client=None, _phase="setup")
+    ) == ActivitySnapshot("setup", None)
 
 
 def test_dashboard_renders_activity_for_registered_running_task():
@@ -135,9 +151,15 @@ def test_dashboard_renders_activity_for_registered_running_task():
     import io
 
     from benchflow._utils import live_activity
+    from benchflow._utils.live_activity import ActivitySnapshot, SessionCounters
 
     live_activity.register(
-        "edit-pdf", SimpleNamespace(activity_snapshot=lambda: (38, "file_editor", None))
+        "edit-pdf",
+        SimpleNamespace(
+            activity_snapshot=lambda: ActivitySnapshot(
+                "connected", SessionCounters(38, "file_editor", None)
+            )
+        ),
     )
     try:
         d = _dash()
@@ -152,17 +174,28 @@ def test_dashboard_renders_activity_for_registered_running_task():
         live_activity.unregister("edit-pdf")
 
 
-def test_activity_cell_empty_before_session_exists():
+def test_activity_cell_shows_phase_label_before_session_exists():
+    # Fresh-user dogfood follow-up: ~1.5min of sandbox create / agent install
+    # (and the whole verifier) used to render a blank cell — indistinguishable
+    # from a hang. Counter-less snapshots must surface the lifecycle phase.
     from benchflow._utils import live_activity
+    from benchflow._utils.live_activity import ActivitySnapshot
     from benchflow.cli._live_progress import _activity_cell
 
-    # Rollout registered but not yet connected (no ACP client / session):
-    # the cell stays empty and the render is unperturbed.
+    phase = "setup"
     live_activity.register(
-        "warming-up", SimpleNamespace(activity_snapshot=lambda: None)
+        "warming-up",
+        SimpleNamespace(activity_snapshot=lambda: ActivitySnapshot(phase, None)),
     )
     try:
-        assert _activity_cell("warming-up") == ""
+        assert _activity_cell("warming-up") == "creating sandbox…"
+        phase = "started"
+        assert _activity_cell("warming-up") == "installing agent…"
+        phase = "verifying"
+        assert _activity_cell("warming-up") == "verifying…"
+        # Unknown phases (e.g. "branched") still never blank the cell.
+        phase = "branched"
+        assert _activity_cell("warming-up") == "starting…"
         d = _dash()
         d.on_plan(total=1, done=0, remaining=1)
         d.on_task_start("warming-up")
@@ -171,6 +204,54 @@ def test_activity_cell_empty_before_session_exists():
         live_activity.unregister("warming-up")
 
 
+def test_dashboard_renders_phase_label_for_counterless_task():
+    # End-to-end through __rich__: the phase label must reach the rendered
+    # panel, not just the cell helper — reverting the table wiring fails this.
+    import io
+
+    from benchflow._utils import live_activity
+    from benchflow._utils.live_activity import ActivitySnapshot
+
+    live_activity.register(
+        "edit-pdf",
+        SimpleNamespace(activity_snapshot=lambda: ActivitySnapshot("started", None)),
+    )
+    try:
+        d = _dash()
+        d.on_plan(total=1, done=0, remaining=1)
+        d.on_task_start("edit-pdf")
+        out = Console(file=io.StringIO(), width=120)
+        out.print(d.__rich__())
+        assert "installing agent…" in out.file.getvalue()
+    finally:
+        live_activity.unregister("edit-pdf")
+
+
+def test_rollout_verify_marks_verifying_phase():
+    # verify() must flip _phase to "verifying" at ENTRY (other transitions
+    # mark completion): disconnect() has already reset the phase to
+    # "installed" by then, and the activity cell keys off this value for the
+    # minutes-long verifier stretch.
+    import asyncio
+
+    from benchflow.rollout import Rollout
+
+    rollout = SimpleNamespace(
+        _config=SimpleNamespace(primary_agent="x"),
+        _trajectory=[{"type": "tool_call"}],  # non-empty: skip the scrape path
+        _phase="installed",
+    )
+
+    async def run() -> None:
+        # The full verify flow needs a sandbox; stop at the first await and
+        # assert the phase already transitioned.
+        with contextlib.suppress(AttributeError):
+            await Rollout.verify(rollout)
+
+    asyncio.run(run())
+    assert rollout._phase == "verifying"
+
+
 def test_progress_enabled_respects_tty_and_optout(monkeypatch):
     tty = SimpleNamespace(is_terminal=True)
     notty = SimpleNamespace(is_terminal=False)
@@ -243,6 +324,110 @@ def test_report_eval_result_surfaces_verifier_errors(monkeypatch):
     assert "Score: 0/3" in out
 
 
+def _reported(result) -> str:
+    """Render _report_eval_result through a captured Console, return the text."""
+    import io
+
+    from rich.console import Console
+
+    import benchflow.cli._shared as shared
+
+    rec = Console(file=io.StringIO(), width=200)
+    original = shared.console
+    shared.console = rec
+    try:
+        shared._report_eval_result(result)
+    finally:
+        shared.console = original
+    return rec.file.getvalue()
+
+
+def _failed_result(task_failures):
+    return SimpleNamespace(
+        passed=0,
+        total=len(task_failures),
+        errored=0,
+        verifier_errored=0,
+        score=0.0,
+        job_name="j",
+        task_failures=task_failures,
+    )
+
+
+def test_report_eval_result_prints_failure_reason_lines():
+    # Dogfood follow-up: "✗ Score: 0/1" alone forces a dig into summary.json.
+    # Each FAILED task gets one dim reason line — verifier_error first, else a
+    # compact reward/metric breakdown (zero metrics first), else the reward.
+    from benchflow.evaluation import TaskFailure
+
+    out = _reported(
+        _failed_result(
+            [
+                TaskFailure(
+                    task_name="edit-pdf",
+                    rewards={"reward": 0.0},
+                    verifier_error="AssertionError:\n  output.pdf missing",
+                ),
+                TaskFailure(
+                    task_name="plan-meeting",
+                    rewards={
+                        "reward": 0.3,
+                        "decisions_found": 0.0,
+                        "deadlines_found": 0.0,
+                        "sections": 1.0,
+                        "extra_metric": 1.0,
+                    },
+                    verifier_error=None,
+                ),
+                TaskFailure(
+                    task_name="sum-csv", rewards={"reward": 0.0}, verifier_error=None
+                ),
+            ]
+        )
+    )
+    # verifier_error wins and is collapsed to one line.
+    assert "✗ edit-pdf: AssertionError: output.pdf missing" in out
+    # Metric breakdown: zero/failed metrics first, capped at 3.
+    assert (
+        "✗ plan-meeting: reward 0.3 — decisions_found 0.0, deadlines_found 0.0, "
+        "sections 1.0" in out
+    )
+    assert "extra_metric" not in out
+    # No named metrics: just the reward.
+    assert "✗ sum-csv: reward 0.0" in out
+
+
+def test_report_eval_result_caps_failure_lines_at_five():
+    from benchflow.evaluation import TaskFailure
+
+    failures = [
+        TaskFailure(task_name=f"task-{i}", rewards={"reward": 0.0}, verifier_error=None)
+        for i in range(7)
+    ]
+    out = _reported(_failed_result(failures))
+    assert out.count("✗ task-") == 5
+    assert "… and 2 more" in out
+
+
+def test_report_eval_result_truncates_failure_lines():
+    from benchflow.evaluation import TaskFailure
+
+    out = _reported(
+        _failed_result(
+            [
+                TaskFailure(
+                    task_name="edit-pdf",
+                    rewards=None,
+                    verifier_error="boom " * 60,
+                )
+            ]
+        )
+    )
+    (line,) = [ln for ln in out.splitlines() if "✗ edit-pdf" in ln]
+    assert len(line.rstrip()) <= 100
+    assert line.rstrip().endswith("…")
+
+
 def test_fire_progress_swallows_callback_errors():
     # The feature's core safety contract: a raising display hook must never
     # propagate out of the engine (a render bug can't abort a run).