benchflow 0.6.7.dev1839__tar.gz → 0.6.7.dev1841__tar.gz

{benchflow-0.6.7.dev1839 → benchflow-0.6.7.dev1841}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1839
+Version: 0.6.7.dev1841
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1839 → benchflow-0.6.7.dev1841}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1839"
+version = "0.6.7.dev1841"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1839 → benchflow-0.6.7.dev1841}/src/benchflow/_utils/live_activity.py

@@ -19,12 +19,15 @@ class SessionCounters(NamedTuple):
     """The live ACP session counters the console heartbeat logs.
 
     ``total_tokens`` is None until the session has a usage snapshot (i.e.
-    after a completed prompt).
+    after a completed prompt). ``distinct_tools`` counts distinct tool
+    display titles seen this session; None means unknown, which renderers
+    treat like varied (keep the ``last:`` cell).
     """
 
     tool_calls: int
     last_tool: str
     total_tokens: int | None
+    distinct_tools: int | None = None
 
 
 class ActivitySnapshot(NamedTuple):

{benchflow-0.6.7.dev1839 → benchflow-0.6.7.dev1841}/src/benchflow/acp/session.py

@@ -138,6 +138,17 @@ def _canonical_tool_kind(kind: object, title: object = "") -> str:
     return raw_kind
 
 
+def _tool_display_title(title: str, kind: str) -> str:
+    """A tool call's display title: first line of ``title``, else ``kind``.
+
+    ``split`` not ``splitlines`` so a whitespace-only title strips to ``""``
+    instead of raising on an empty line list. Shared by the heartbeat/dashboard
+    snapshot and the distinct-title tracker so "distinct" means exactly
+    "renders differently".
+    """
+    return (title or kind or "").strip().split("\n", 1)[0]
+
+
 class ToolCallRecord:
     """Record of a single tool call within a session.
 
@@ -191,6 +202,12 @@ class ACPSession:
         self.thought_chunks: list[str] = []
         self.tool_calls: list[ToolCallRecord] = []
         self._tool_call_map: dict[str, ToolCallRecord] = {}
+        # Distinct display titles across recorded tool calls, maintained
+        # incrementally at record creation so the eval dashboard can detect
+        # single-tool agents (prime-agent funnels everything through one
+        # IPython tool — a constant "last: IPython cell" carries no
+        # information) in O(1) at render time.
+        self._seen_tool_titles: set[str] = set()
         self.stop_reason: StopReason | None = None
         self.usage_snapshots: list[ACPUsageSnapshot] = []
         self.created_at = datetime.now()
@@ -226,15 +243,24 @@ class ACPSession:
         counters, also polled by the live eval dashboard's activity cell.
 
         The title is the raw first line (untruncated — display width belongs
-        to each render site); ``split`` not ``splitlines`` so a whitespace-only
-        title strips to ``""`` instead of raising on an empty line list.
+        to each render site; see :func:`_tool_display_title`).
         """
         title = ""
         if self.tool_calls:
             last = self.tool_calls[-1]
-            title = (last.title or last.kind or "").strip().split("\n", 1)[0]
+            title = _tool_display_title(last.title, last.kind)
         return len(self.tool_calls), title
 
+    @property
+    def distinct_tool_titles(self) -> int:
+        """Count of distinct display titles across recorded tool calls.
+
+        O(1) read for the eval dashboard: ``1`` with several calls means a
+        single-tool agent, whose activity cell drops the constant ``last:``
+        suffix in favour of the token count.
+        """
+        return len(self._seen_tool_titles)
+
     def _maybe_log_progress(self) -> None:
         if not self._progress_enabled or self._prompt_started_at is None:
             return
@@ -324,6 +350,22 @@ class ACPSession:
         self.events.append(current)
         self._pending_text.clear()
 
+    def _record_tool_call(self, record: ToolCallRecord) -> None:
+        """Register a newly created tool-call record in every live structure.
+
+        Single bookkeeping site for the two creation paths in
+        :meth:`handle_update` (``tool_call``, and the ``tool_call_update``
+        fallback for unseen ids), so the distinct-title tracker can never
+        drift from the record list. ``title`` itself is never rewritten after
+        creation, but an empty-title record's display title follows ``kind``,
+        which the legacy-skill upgrade can rewrite — that site re-registers
+        the new display title (over-counting fails safe).
+        """
+        self.tool_calls.append(record)
+        self._tool_call_map[record.tool_call_id] = record
+        self._seen_tool_titles.add(_tool_display_title(record.title, record.kind))
+        self.events.append({"type": "tool_call", "record": record})
+
     _RECOGNIZED_UPDATE_TYPES = frozenset(
         {
             "tool_call",
@@ -356,9 +398,7 @@ class ACPSession:
                     update.get("kind", "other"), update.get("title", "")
                 ),
             )
-            self.tool_calls.append(record)
-            self._tool_call_map[record.tool_call_id] = record
-            self.events.append({"type": "tool_call", "record": record})
+            self._record_tool_call(record)
 
         elif update_type == "tool_call_update":
             tc_id = update.get("toolCallId", "")
@@ -372,9 +412,7 @@ class ACPSession:
                         update.get("kind", "tool"), update.get("title", "")
                     ),
                 )
-                self.tool_calls.append(record)
-                self._tool_call_map[tc_id] = record
-                self.events.append({"type": "tool_call", "record": record})
+                self._record_tool_call(record)
             try:
                 status = ToolCallStatus(update.get("status", "in_progress"))
             except ValueError:
@@ -390,6 +428,14 @@ class ACPSession:
                 record.kind, record.title, record.content
             ):
                 record.kind = "skill"
+                # An empty-title record's DISPLAY title falls back to kind, so
+                # this upgrade can change what renders. Re-register the new
+                # display title: the set may now over-count (creation-time
+                # fallback + "skill"), which fails safe — the cell keeps the
+                # last: suffix instead of falsely claiming a single tool.
+                self._seen_tool_titles.add(
+                    _tool_display_title(record.title, record.kind)
+                )
 
         elif update_type == "agent_message_chunk":
             content = update.get("content", {})

benchflow-0.6.7.dev1841/src/benchflow/cli/_failure_evidence.py

@@ -0,0 +1,167 @@
+"""Verifier-artifact mining for the eval report's failure lines.
+
+The one file-touching corner of the CLI's final report block: when a failed
+task's reason would otherwise be the bare ``reward X`` fallback, mine the
+rollout's verifier artifacts for a one-line explanation. Lives in its own
+module so ``cli/_shared.py`` stays the side-effect-free display helpers it
+advertises.
+
+Contract (the engine stays file-free — these reads are CLI-side, report-time
+only):
+
+- The rollout dir is resolved exactly, from the ``rollout_name`` the engine
+  records on every result — never guessed from globs.
+- Evidence sources are tried in order (CTRF report, then test-stdout tail);
+  the first that yields a line wins.
+- Every read is bounded (``_ARTIFACT_READ_BYTES`` per file) and nothing here
+  raises — any surprise degrades to ``None``, i.e. the bare reward reason.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import TYPE_CHECKING, NamedTuple
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from pathlib import Path
+
+# Never read more than this many bytes per file, and evidence is only mined
+# for the <= _MAX_FAILURE_LINES tasks the report block displays.
+_ARTIFACT_READ_BYTES = 64 * 1024
+# Pytest final summary, decoration stripped: "1 failed, 2 passed in 40.86s".
+_PYTEST_SUMMARY_RE = re.compile(r"\d+ failed\b")
+
+
+class FailureLine(NamedTuple):
+    """A failure one-liner in two parts with distinct truncation contracts.
+
+    ``body`` is the free-text evidence (test name plus assertion) and competes
+    for the display line's char budget; ``suffix`` is the compact multi-failure
+    roll-up count (empty when the evidence covers everything) and must survive
+    display truncation. Renderers give ``body`` the full line budget and append
+    ``suffix`` whole past it — never truncating the concatenation, or a long
+    assertion would silently eat the "there is more broken than this" signal.
+    """
+
+    body: str
+    suffix: str = ""
+
+
+def _display_test_name(raw_name: str) -> str:
+    """Test name for display: node-id path segments dropped, param id kept.
+
+    Full pytest node ids ("tests/test_x.py::test_build[param]") waste the
+    100-char line budget; keep the function name plus any ``[param]`` id.
+    The split must not touch the bracket part — a param id may itself contain
+    ``::`` — so only the text before the first ``[`` is segment-split. (When
+    the report was written by pytest-json-ctrf — as of 0.5.x — the param id
+    is already gone — the plugin stores ``nodeid.split('[')[0]`` as ``name``
+    — but other CTRF producers keep the full name, and we must not re-trim
+    it.)
+    """
+    head, bracket, param = raw_name.partition("[")
+    return head.rsplit("::", 1)[-1] + bracket + param
+
+
+def _ctrf_failure_line(ctrf_path: Path) -> FailureLine | None:
+    """``<test> failed[: <assertion>]`` from the first failed CTRF test.
+
+    The verifier's pytest run writes a CTRF report (``pytest --ctrf``, see the
+    standard path in ``task_authoring/structural_checks.py``). Per test the
+    useful failure evidence is the ``E …`` assertion line inside ``trace``;
+    ``message`` is only a generic phase description, so it is the fallback.
+    When the report holds more than one failed test, the first failure's line
+    carries a count suffix (``(+N more failures; P/T checks passed)``) so the
+    console never under-reports how much is broken.
+    """
+    if ctrf_path.stat().st_size > _ARTIFACT_READ_BYTES:
+        # JSON only parses whole — an oversized report is skipped (not
+        # truncated) and the next evidence source gets its turn.
+        return None
+    data = json.loads(ctrf_path.read_text(encoding="utf-8", errors="replace"))
+    raw_tests = (data.get("results") or {}).get("tests") or []
+    tests = [test for test in raw_tests if isinstance(test, dict)]
+    failed = [test for test in tests if test.get("status") == "failed"]
+    if not failed:
+        return None
+    suffix = ""
+    if len(failed) > 1:
+        # Counts come from the same rolled-up test list the first-failure pick
+        # uses, so the suffix can never disagree with the shown evidence.
+        # T is all entries — skips count against the denominator, not as passes.
+        extra = len(failed) - 1
+        passed = sum(1 for test in tests if test.get("status") == "passed")
+        plural = "" if extra == 1 else "s"
+        suffix = (
+            f" (+{extra} more failure{plural}; {passed}/{len(tests)} checks passed)"
+        )
+    test = failed[0]
+    name = _display_test_name(str(test.get("name") or "test"))
+    trace = test.get("trace")
+    if isinstance(trace, str):
+        for line in trace.splitlines():
+            stripped = line.strip()
+            if stripped.startswith("E "):
+                assertion = stripped[2:].strip()
+                # "<test> failed:" already says it's an assertion — drop
+                # the prefix to reclaim line budget for the message.
+                assertion = assertion.removeprefix("AssertionError: ")
+                return FailureLine(f"{name} failed: {assertion}", suffix)
+    message = test.get("message")
+    if isinstance(message, str) and message.strip():
+        return FailureLine(f"{name} failed: {' '.join(message.split())}", suffix)
+    return FailureLine(f"{name} failed", suffix)
+
+
+def _stdout_tail_failure_line(stdout_path: Path) -> FailureLine | None:
+    """Last pytest summary ("N failed, M passed …") — else last ``FAILED …``
+    line — from a bounded tail of the verifier's test-stdout capture. No count
+    suffix: the summary line already carries the full counts itself."""
+    with stdout_path.open("rb") as fh:
+        fh.seek(0, 2)  # SEEK_END
+        fh.seek(max(0, fh.tell() - _ARTIFACT_READ_BYTES))
+        tail = fh.read(_ARTIFACT_READ_BYTES).decode("utf-8", errors="replace")
+    last_failed: str | None = None
+    for line in reversed(tail.splitlines()):
+        bare = line.strip().strip("=").strip()
+        if _PYTEST_SUMMARY_RE.match(bare):
+            return FailureLine(bare)
+        if last_failed is None and line.strip().startswith("FAILED "):
+            last_failed = line.strip()
+    return FailureLine(last_failed) if last_failed is not None else None
+
+
+def artifact_failure_evidence(
+    job_dir: Path, rollout_name: str
+) -> tuple[FailureLine, Path] | None:
+    """``FailureLine`` mined from the rollout's verifier artifacts, or ``None``.
+
+    Returns the structured one-liner plus the verifier dir it came from (for
+    the report block's single ``(details: …)`` pointer line). Never raises.
+    """
+    # Local import: RolloutPaths pulls in the task package, which the CLI
+    # shouldn't pay for until a failure actually needs artifact evidence.
+    from benchflow.task.paths import RolloutPaths
+
+    rollout_paths = RolloutPaths(rollout_dir=job_dir / rollout_name)
+    verifier_dir = rollout_paths.verifier_dir
+    attempts: list[tuple[Path, Callable[[Path], FailureLine | None]]] = [
+        # ctrf.json has no RolloutPaths property — the verifier recovers it by
+        # literal name (verifier_core.py, `_recover_main_verifier_outputs`).
+        (verifier_dir / "ctrf.json", _ctrf_failure_line),
+        (rollout_paths.test_stdout_path, _stdout_tail_failure_line),
+    ]
+    for path, extract in attempts:
+        try:
+            if not path.is_file():
+                continue
+            detail = extract(path)
+        except Exception:
+            # Unreadable file, bad JSON, permissions … try the next source;
+            # the report must never crash over evidence mining.
+            continue
+        if detail is not None:
+            return detail, verifier_dir
+    return None

{benchflow-0.6.7.dev1839 → benchflow-0.6.7.dev1841}/src/benchflow/cli/_live_progress.py

@@ -3,7 +3,9 @@
 Renders a single Rich :class:`~rich.live.Live` panel — a progress bar with ETA,
 queued/running/passed/failed/errored counts, a "running now" table, and running
 token / cost / pass-rate totals — fed by the :class:`~benchflow.evaluation.Evaluation`
-engine's ``on_plan`` / ``on_task_start`` / ``on_result`` hooks.
+engine's ``on_plan`` / ``on_task_start`` / ``on_result`` hooks, plus render-time
+polls of the live-activity registry for the running tasks' in-flight session
+counters (activity cells and the footer's live token sum).
 
 TTY-only by contract: the CLI keeps its plain ``logger.info`` lines when stdout
 isn't a terminal (CI, pipes, parity files), so machine-readable output is never
@@ -149,18 +151,21 @@ _PHASE_LABELS = {
 _PHASE_FALLBACK = "starting…"
 
 
-def _activity_cell(name: str) -> str:
+def _activity_cell(snap: live_activity.ActivitySnapshot | None) -> str:
     """Per-task activity for the "running now" table, e.g.
     ``38 calls · last: file_editor``.
 
-    Polls the live rollout's ACP session counters — the same ones the console
-    heartbeat logs, which the Live display otherwise mutes — via the
-    live-activity registry. Until the agent session exists (and after it is
-    closed for the verifier) the cell shows the rollout's lifecycle phase as a
-    label instead, so the row is never blank. Tokens appear only once the
-    session has a usage snapshot (i.e. after a completed prompt).
+    Pure formatter over one polled :class:`~benchflow._utils.live_activity.
+    ActivitySnapshot` — ``__rich__`` polls the registry exactly once per
+    running task per frame and shares the snapshots with the footer's live
+    token sum. The counters are the same ones the console heartbeat logs,
+    which the Live display otherwise mutes. Until the agent session exists
+    (and after it is closed for the verifier) the cell shows the rollout's
+    lifecycle phase as a label instead, and a registry miss (None) shows the
+    fallback label, so the row is never blank. Tokens appear only once the
+    session has a usage snapshot (i.e. after a completed prompt); once they
+    do, a single-tool agent's cell is ``38 calls · 412.0k tok`` (no ``last:``).
     """
-    snap = live_activity.activity(name)
     if snap is None:
         return _PHASE_FALLBACK
     counters = snap.counters
@@ -169,7 +174,12 @@ def _activity_cell(name: str) -> str:
     cell = f"{counters.tool_calls} calls"
     if counters.total_tokens:
         cell += f" · {_fmt_tokens(counters.total_tokens)} tok"
-    if counters.last_tool:
+    # A single-tool agent (prime-agent funnels every call through one IPython
+    # tool) makes "last: <name>" a constant: once tokens carry the signal,
+    # drop it. Varied tool names — and unknown distinct counts (0), and a
+    # first lone call, where the name is still news — keep the last: suffix.
+    constant_tool = counters.tool_calls > 1 and counters.distinct_tools == 1
+    if counters.last_tool and not (constant_tool and counters.total_tokens):
         cell += f" · last: {counters.last_tool[:30]}"
     return cell
 
@@ -335,6 +345,13 @@ class LiveEvalProgress:
 
         parts: list[RenderableType] = [header, bar, counts]
 
+        # ONE registry poll per running task per frame (each poll an O(1)
+        # counter read; see live_activity), shared by the running-now cells
+        # and the footer's live token sum — displayed rows are never polled
+        # twice, and cell vs footer can't read different snapshots within a
+        # frame.
+        snaps = {name: live_activity.activity(name) for name in running}
+
         # "Running now" — cap rows so short terminals don't overflow.
         if running:
             tbl = Table(
@@ -354,16 +371,34 @@ class LiveEvalProgress:
                 tbl.add_row(
                     Text(name),
                     _fmt_dur(now - running[name]),
-                    Text(_activity_cell(name), style="dim"),
+                    Text(_activity_cell(snaps.get(name)), style="dim"),
                 )
             extra = len(running) - _MAX_RUNNING_ROWS
             if extra > 0:
                 tbl.add_row(f"… {extra} more", "", "")
             parts.append(tbl)
 
-        # Footer: pass-rate (excl errors) + token/cost economics. Show "—" (not
-        # 0 / $0.00) when no trusted telemetry, so a coverage-0 run reads broken,
-        # not free — matching the summary.json contract.
+        # Footer: pass-rate (excl errors) + token/cost economics. Tokens sum
+        # the completed tasks' trusted telemetry PLUS the running sessions'
+        # live ACP usage (stepping forward as each prompt completes), so a
+        # 51-minute agent phase shows its spend while it runs, not only at
+        # scoring. The total is NOT monotonic across a task's completion
+        # boundary: scoring swaps the ACP self-report for the gateway
+        # measurement (which can be lower), and a task completing with
+        # untrusted/absent telemetry drops its live contribution entirely —
+        # a sole-task footer can collapse back to "— tokens". Clamping would
+        # falsify the trusted sum, so the dip is intended. Show "—" (not 0 /
+        # $0.00) when neither signal exists, so a coverage-0 run reads
+        # broken, not free — matching the summary.json contract. Cost stays
+        # completed-tasks-only: $ comes from the sandbox LiteLLM gateway log
+        # imported at scoring time (BenchFlow computes no prices of its own),
+        # so there is no live $ to show.
+        live_tokens = sum(
+            snap.counters.total_tokens or 0
+            for snap in snaps.values()
+            if snap is not None and snap.counters is not None
+        )
+
         footer = Text()
         scored = passed + failed
         if scored:
@@ -371,7 +406,12 @@ class LiveEvalProgress:
         else:
             footer.append("pass-rate —", style="dim")
         footer.append(" · ")
-        footer.append(f"{_fmt_tokens(tokens)} tokens" if covered else "— tokens", "dim")
+        footer.append(
+            f"{_fmt_tokens(tokens + live_tokens)} tokens"
+            if covered or live_tokens
+            else "— tokens",
+            "dim",
+        )
         footer.append(" · ")
         footer.append(f"${cost:.2f}" if covered else "$—", style="dim")
         if completed:

{benchflow-0.6.7.dev1839 → benchflow-0.6.7.dev1841}/src/benchflow/cli/_shared.py

@@ -21,7 +21,7 @@ from rich.console import Console
 from rich.markup import escape
 
 from benchflow._utils.text import truncate_end
-from benchflow.cli._failure_evidence import artifact_failure_evidence
+from benchflow.cli._failure_evidence import FailureLine, artifact_failure_evidence
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -106,7 +106,8 @@ def _exit_if_evaluation_had_errors(result: object) -> None:
 
 
 # Final-block failure lines: keep the block skimmable on big jobs and each
-# line inside a typical terminal width.
+# line inside a typical terminal width — except lines carrying a multi-failure
+# count suffix, which deliberately run ~40 chars past the budget.
 _MAX_FAILURE_LINES = 5
 _FAILURE_LINE_LIMIT = 100
 _FAILURE_REASON_METRICS = 3
@@ -114,7 +115,7 @@ _FAILURE_REASON_METRICS = 3
 
 def _failure_reason(
     failure: TaskFailure, job_dir: Path | None = None
-) -> tuple[str, Path | None]:
+) -> tuple[FailureLine, Path | None]:
     """One cheap line explaining why a FAILED (scored, reward != 1) task
     failed, plus the verifier dir when artifacts supplied the evidence.
 
@@ -123,13 +124,15 @@ def _failure_reason(
     first — they explain the miss); else the reward plus a one-liner mined
     from the rollout's verifier artifacts (bounded CLI-side reads resolved via
     the ``rollout_name`` the engine records — the engine stays file-free);
-    else just the reward. The returned path is non-``None`` only for the
-    artifact tier, so the caller can print one ``(details: …)`` pointer per
-    report block.
+    else just the reward. The reason is a ``FailureLine`` so the artifact
+    tier's multi-failure count suffix stays separable from the truncatable
+    body (only the artifact tier ever sets it). The returned path is
+    non-``None`` only for the artifact tier, so the caller can print one
+    ``(details: …)`` pointer per report block.
     """
     if failure.verifier_error:
         # Collapse whitespace: verifier errors are routinely multi-line.
-        return " ".join(failure.verifier_error.split()), None
+        return FailureLine(" ".join(failure.verifier_error.split())), None
     rewards = failure.rewards or {}
     reward = rewards.get("reward")
     metrics = [
@@ -144,15 +147,18 @@ def _failure_reason(
         shown = ", ".join(
             f"{name} {value}" for name, value in metrics[:_FAILURE_REASON_METRICS]
         )
-        return f"reward {reward} — {shown}", None
+        return FailureLine(f"reward {reward} — {shown}"), None
     # No rollout_name (pre-#957-follow-up result.json, sharded aggregation):
     # nothing to resolve — the bare reward is as honest as it gets.
     if job_dir is not None and failure.rollout_name:
         evidence = artifact_failure_evidence(job_dir, failure.rollout_name)
         if evidence is not None:
             detail, verifier_dir = evidence
-            return f"reward {reward} — {detail}", verifier_dir
-    return f"reward {reward}", None
+            return (
+                FailureLine(f"reward {reward} — {detail.body}", detail.suffix),
+                verifier_dir,
+            )
+    return FailureLine(f"reward {reward}"), None
 
 
 def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -> None:
@@ -207,11 +213,16 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
         reason, verifier_dir = _failure_reason(failure, job_dir)
         if artifact_pointer is None and verifier_dir is not None:
             artifact_pointer = verifier_dir
-        line = truncate_end(
-            f"  ✗ {failure.task_name}: {reason}",
+        # The char budget governs only the free-text body; the multi-failure
+        # count suffix is appended AFTER truncation so a long assertion can
+        # never swallow the "more than this is broken" signal. Worst case the
+        # line runs ~40 chars past the budget — completeness beats strict
+        # width for this one signal.
+        body = truncate_end(
+            f"  ✗ {failure.task_name}: {reason.body}",
             _FAILURE_LINE_LIMIT,
         )
-        console.print(f"[dim]{escape(line)}[/dim]")
+        console.print(f"[dim]{escape(body + reason.suffix)}[/dim]")
     extra = len(failures) - _MAX_FAILURE_LINES
     if extra > 0:
         console.print(f"[dim]  … and {extra} more[/dim]")

{benchflow-0.6.7.dev1839 → benchflow-0.6.7.dev1841}/src/benchflow/rollout/__init__.py

@@ -759,7 +759,8 @@ class Rollout:
     def activity_snapshot(self) -> ActivitySnapshot:
         """The eval dashboard's per-task :class:`ActivitySnapshot`: the
         current lifecycle phase plus the live :class:`SessionCounters` (tool
-        calls, last tool title, total tokens). ``counters`` is None before
+        calls, last tool title, total tokens, distinct tool titles).
+        ``counters`` is None before
         the agent session exists — the phase then carries the cell
         ("creating sandbox…", "verifying…"), so a 90s sandbox create is not
         indistinguishable from a hang.
@@ -775,7 +776,10 @@ class Rollout:
         calls, last_title = session.progress_snapshot()
         usage = session.latest_usage_totals()
         tokens = usage.get("total_tokens") if usage else None
-        return ActivitySnapshot(self._phase, SessionCounters(calls, last_title, tokens))
+        return ActivitySnapshot(
+            self._phase,
+            SessionCounters(calls, last_title, tokens, session.distinct_tool_titles),
+        )
 
     @property
     def trajectory(self) -> list[dict]: