benchflow 0.6.7.dev1840__tar.gz → 0.6.7.dev1841__tar.gz

{benchflow-0.6.7.dev1840 → benchflow-0.6.7.dev1841}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1840
+Version: 0.6.7.dev1841
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1840 → benchflow-0.6.7.dev1841}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1840"
+version = "0.6.7.dev1841"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1840 → benchflow-0.6.7.dev1841}/src/benchflow/_utils/live_activity.py

@@ -19,12 +19,15 @@ class SessionCounters(NamedTuple):
     """The live ACP session counters the console heartbeat logs.
 
     ``total_tokens`` is None until the session has a usage snapshot (i.e.
-    after a completed prompt).
+    after a completed prompt). ``distinct_tools`` counts distinct tool
+    display titles seen this session; None means unknown, which renderers
+    treat like varied (keep the ``last:`` cell).
     """
 
     tool_calls: int
     last_tool: str
     total_tokens: int | None
+    distinct_tools: int | None = None
 
 
 class ActivitySnapshot(NamedTuple):

{benchflow-0.6.7.dev1840 → benchflow-0.6.7.dev1841}/src/benchflow/acp/session.py

@@ -138,6 +138,17 @@ def _canonical_tool_kind(kind: object, title: object = "") -> str:
     return raw_kind
 
 
+def _tool_display_title(title: str, kind: str) -> str:
+    """A tool call's display title: first line of ``title``, else ``kind``.
+
+    ``split`` not ``splitlines`` so a whitespace-only title strips to ``""``
+    instead of raising on an empty line list. Shared by the heartbeat/dashboard
+    snapshot and the distinct-title tracker so "distinct" means exactly
+    "renders differently".
+    """
+    return (title or kind or "").strip().split("\n", 1)[0]
+
+
 class ToolCallRecord:
     """Record of a single tool call within a session.
 
@@ -191,6 +202,12 @@ class ACPSession:
         self.thought_chunks: list[str] = []
         self.tool_calls: list[ToolCallRecord] = []
         self._tool_call_map: dict[str, ToolCallRecord] = {}
+        # Distinct display titles across recorded tool calls, maintained
+        # incrementally at record creation so the eval dashboard can detect
+        # single-tool agents (prime-agent funnels everything through one
+        # IPython tool — a constant "last: IPython cell" carries no
+        # information) in O(1) at render time.
+        self._seen_tool_titles: set[str] = set()
         self.stop_reason: StopReason | None = None
         self.usage_snapshots: list[ACPUsageSnapshot] = []
         self.created_at = datetime.now()
@@ -226,15 +243,24 @@ class ACPSession:
         counters, also polled by the live eval dashboard's activity cell.
 
         The title is the raw first line (untruncated — display width belongs
-        to each render site); ``split`` not ``splitlines`` so a whitespace-only
-        title strips to ``""`` instead of raising on an empty line list.
+        to each render site; see :func:`_tool_display_title`).
         """
         title = ""
         if self.tool_calls:
             last = self.tool_calls[-1]
-            title = (last.title or last.kind or "").strip().split("\n", 1)[0]
+            title = _tool_display_title(last.title, last.kind)
         return len(self.tool_calls), title
 
+    @property
+    def distinct_tool_titles(self) -> int:
+        """Count of distinct display titles across recorded tool calls.
+
+        O(1) read for the eval dashboard: ``1`` with several calls means a
+        single-tool agent, whose activity cell drops the constant ``last:``
+        suffix in favour of the token count.
+        """
+        return len(self._seen_tool_titles)
+
     def _maybe_log_progress(self) -> None:
         if not self._progress_enabled or self._prompt_started_at is None:
             return
@@ -324,6 +350,22 @@ class ACPSession:
         self.events.append(current)
         self._pending_text.clear()
 
+    def _record_tool_call(self, record: ToolCallRecord) -> None:
+        """Register a newly created tool-call record in every live structure.
+
+        Single bookkeeping site for the two creation paths in
+        :meth:`handle_update` (``tool_call``, and the ``tool_call_update``
+        fallback for unseen ids), so the distinct-title tracker can never
+        drift from the record list. ``title`` itself is never rewritten after
+        creation, but an empty-title record's display title follows ``kind``,
+        which the legacy-skill upgrade can rewrite — that site re-registers
+        the new display title (over-counting fails safe).
+        """
+        self.tool_calls.append(record)
+        self._tool_call_map[record.tool_call_id] = record
+        self._seen_tool_titles.add(_tool_display_title(record.title, record.kind))
+        self.events.append({"type": "tool_call", "record": record})
+
     _RECOGNIZED_UPDATE_TYPES = frozenset(
         {
             "tool_call",
@@ -356,9 +398,7 @@ class ACPSession:
                     update.get("kind", "other"), update.get("title", "")
                 ),
             )
-            self.tool_calls.append(record)
-            self._tool_call_map[record.tool_call_id] = record
-            self.events.append({"type": "tool_call", "record": record})
+            self._record_tool_call(record)
 
         elif update_type == "tool_call_update":
             tc_id = update.get("toolCallId", "")
@@ -372,9 +412,7 @@ class ACPSession:
                         update.get("kind", "tool"), update.get("title", "")
                     ),
                 )
-                self.tool_calls.append(record)
-                self._tool_call_map[tc_id] = record
-                self.events.append({"type": "tool_call", "record": record})
+                self._record_tool_call(record)
             try:
                 status = ToolCallStatus(update.get("status", "in_progress"))
             except ValueError:
@@ -390,6 +428,14 @@ class ACPSession:
                 record.kind, record.title, record.content
             ):
                 record.kind = "skill"
+                # An empty-title record's DISPLAY title falls back to kind, so
+                # this upgrade can change what renders. Re-register the new
+                # display title: the set may now over-count (creation-time
+                # fallback + "skill"), which fails safe — the cell keeps the
+                # last: suffix instead of falsely claiming a single tool.
+                self._seen_tool_titles.add(
+                    _tool_display_title(record.title, record.kind)
+                )
 
         elif update_type == "agent_message_chunk":
             content = update.get("content", {})

{benchflow-0.6.7.dev1840 → benchflow-0.6.7.dev1841}/src/benchflow/cli/_live_progress.py

@@ -3,7 +3,9 @@
 Renders a single Rich :class:`~rich.live.Live` panel — a progress bar with ETA,
 queued/running/passed/failed/errored counts, a "running now" table, and running
 token / cost / pass-rate totals — fed by the :class:`~benchflow.evaluation.Evaluation`
-engine's ``on_plan`` / ``on_task_start`` / ``on_result`` hooks.
+engine's ``on_plan`` / ``on_task_start`` / ``on_result`` hooks, plus render-time
+polls of the live-activity registry for the running tasks' in-flight session
+counters (activity cells and the footer's live token sum).
 
 TTY-only by contract: the CLI keeps its plain ``logger.info`` lines when stdout
 isn't a terminal (CI, pipes, parity files), so machine-readable output is never
@@ -149,18 +151,21 @@ _PHASE_LABELS = {
 _PHASE_FALLBACK = "starting…"
 
 
-def _activity_cell(name: str) -> str:
+def _activity_cell(snap: live_activity.ActivitySnapshot | None) -> str:
     """Per-task activity for the "running now" table, e.g.
     ``38 calls · last: file_editor``.
 
-    Polls the live rollout's ACP session counters — the same ones the console
-    heartbeat logs, which the Live display otherwise mutes — via the
-    live-activity registry. Until the agent session exists (and after it is
-    closed for the verifier) the cell shows the rollout's lifecycle phase as a
-    label instead, so the row is never blank. Tokens appear only once the
-    session has a usage snapshot (i.e. after a completed prompt).
+    Pure formatter over one polled :class:`~benchflow._utils.live_activity.
+    ActivitySnapshot` — ``__rich__`` polls the registry exactly once per
+    running task per frame and shares the snapshots with the footer's live
+    token sum. The counters are the same ones the console heartbeat logs,
+    which the Live display otherwise mutes. Until the agent session exists
+    (and after it is closed for the verifier) the cell shows the rollout's
+    lifecycle phase as a label instead, and a registry miss (None) shows the
+    fallback label, so the row is never blank. Tokens appear only once the
+    session has a usage snapshot (i.e. after a completed prompt); once they
+    do, a single-tool agent's cell is ``38 calls · 412.0k tok`` (no ``last:``).
     """
-    snap = live_activity.activity(name)
     if snap is None:
         return _PHASE_FALLBACK
     counters = snap.counters
@@ -169,7 +174,12 @@ def _activity_cell(name: str) -> str:
     cell = f"{counters.tool_calls} calls"
     if counters.total_tokens:
         cell += f" · {_fmt_tokens(counters.total_tokens)} tok"
-    if counters.last_tool:
+    # A single-tool agent (prime-agent funnels every call through one IPython
+    # tool) makes "last: <name>" a constant: once tokens carry the signal,
+    # drop it. Varied tool names — and unknown distinct counts (0), and a
+    # first lone call, where the name is still news — keep the last: suffix.
+    constant_tool = counters.tool_calls > 1 and counters.distinct_tools == 1
+    if counters.last_tool and not (constant_tool and counters.total_tokens):
         cell += f" · last: {counters.last_tool[:30]}"
     return cell
 
@@ -335,6 +345,13 @@ class LiveEvalProgress:
 
         parts: list[RenderableType] = [header, bar, counts]
 
+        # ONE registry poll per running task per frame (each poll an O(1)
+        # counter read; see live_activity), shared by the running-now cells
+        # and the footer's live token sum — displayed rows are never polled
+        # twice, and cell vs footer can't read different snapshots within a
+        # frame.
+        snaps = {name: live_activity.activity(name) for name in running}
+
         # "Running now" — cap rows so short terminals don't overflow.
         if running:
             tbl = Table(
@@ -354,16 +371,34 @@ class LiveEvalProgress:
                 tbl.add_row(
                     Text(name),
                     _fmt_dur(now - running[name]),
-                    Text(_activity_cell(name), style="dim"),
+                    Text(_activity_cell(snaps.get(name)), style="dim"),
                 )
             extra = len(running) - _MAX_RUNNING_ROWS
             if extra > 0:
                 tbl.add_row(f"… {extra} more", "", "")
             parts.append(tbl)
 
-        # Footer: pass-rate (excl errors) + token/cost economics. Show "—" (not
-        # 0 / $0.00) when no trusted telemetry, so a coverage-0 run reads broken,
-        # not free — matching the summary.json contract.
+        # Footer: pass-rate (excl errors) + token/cost economics. Tokens sum
+        # the completed tasks' trusted telemetry PLUS the running sessions'
+        # live ACP usage (stepping forward as each prompt completes), so a
+        # 51-minute agent phase shows its spend while it runs, not only at
+        # scoring. The total is NOT monotonic across a task's completion
+        # boundary: scoring swaps the ACP self-report for the gateway
+        # measurement (which can be lower), and a task completing with
+        # untrusted/absent telemetry drops its live contribution entirely —
+        # a sole-task footer can collapse back to "— tokens". Clamping would
+        # falsify the trusted sum, so the dip is intended. Show "—" (not 0 /
+        # $0.00) when neither signal exists, so a coverage-0 run reads
+        # broken, not free — matching the summary.json contract. Cost stays
+        # completed-tasks-only: $ comes from the sandbox LiteLLM gateway log
+        # imported at scoring time (BenchFlow computes no prices of its own),
+        # so there is no live $ to show.
+        live_tokens = sum(
+            snap.counters.total_tokens or 0
+            for snap in snaps.values()
+            if snap is not None and snap.counters is not None
+        )
+
         footer = Text()
         scored = passed + failed
         if scored:
@@ -371,7 +406,12 @@ class LiveEvalProgress:
         else:
             footer.append("pass-rate —", style="dim")
         footer.append(" · ")
-        footer.append(f"{_fmt_tokens(tokens)} tokens" if covered else "— tokens", "dim")
+        footer.append(
+            f"{_fmt_tokens(tokens + live_tokens)} tokens"
+            if covered or live_tokens
+            else "— tokens",
+            "dim",
+        )
         footer.append(" · ")
         footer.append(f"${cost:.2f}" if covered else "$—", style="dim")
         if completed:

{benchflow-0.6.7.dev1840 → benchflow-0.6.7.dev1841}/src/benchflow/rollout/__init__.py

@@ -759,7 +759,8 @@ class Rollout:
     def activity_snapshot(self) -> ActivitySnapshot:
         """The eval dashboard's per-task :class:`ActivitySnapshot`: the
         current lifecycle phase plus the live :class:`SessionCounters` (tool
-        calls, last tool title, total tokens). ``counters`` is None before
+        calls, last tool title, total tokens, distinct tool titles).
+        ``counters`` is None before
         the agent session exists — the phase then carries the cell
         ("creating sandbox…", "verifying…"), so a 90s sandbox create is not
         indistinguishable from a hang.
@@ -775,7 +776,10 @@ class Rollout:
         calls, last_title = session.progress_snapshot()
         usage = session.latest_usage_totals()
         tokens = usage.get("total_tokens") if usage else None
-        return ActivitySnapshot(self._phase, SessionCounters(calls, last_title, tokens))
+        return ActivitySnapshot(
+            self._phase,
+            SessionCounters(calls, last_title, tokens, session.distinct_tool_titles),
+        )
 
     @property
     def trajectory(self) -> list[dict]:

{benchflow-0.6.7.dev1840 → benchflow-0.6.7.dev1841}/tests/test_cli_live_progress.py

@@ -10,10 +10,14 @@ import contextlib
 import logging
 from types import SimpleNamespace
 
+import pytest
 from rich.console import Console
 
+from benchflow._utils import live_activity
+from benchflow._utils.live_activity import ActivitySnapshot, SessionCounters
 from benchflow.cli._live_progress import (
     LiveEvalProgress,
+    _activity_cell,
     progress_enabled,
     quiet_root_logging,
 )
@@ -34,6 +38,36 @@ def _dash() -> LiveEvalProgress:
     )
 
 
+@contextlib.contextmanager
+def _registered_rollouts(rollouts: dict):
+    """Register fake rollouts and guarantee unregistration.
+
+    Values are the ActivitySnapshot to serve — or an Exception instance to
+    raise from ``activity_snapshot()`` (a teardown-racing rollout).
+    """
+
+    def _fake(snap):
+        if isinstance(snap, Exception):
+
+            def _boom():
+                raise snap
+
+            return SimpleNamespace(activity_snapshot=_boom)
+        return SimpleNamespace(activity_snapshot=lambda: snap)
+
+    for name, snap in rollouts.items():
+        live_activity.register(name, _fake(snap))
+    try:
+        yield
+    finally:
+        for name in rollouts:
+            live_activity.unregister(name)
+
+
+def _counters_snap(tokens, *, calls=3, last="file_editor", distinct=2):
+    return ActivitySnapshot("connected", SessionCounters(calls, last, tokens, distinct))
+
+
 def test_counts_classify_like_the_engine():
     d = _dash()
     d.on_plan(total=4, done=0, remaining=4)
@@ -93,7 +127,85 @@ def test_trusted_telemetry_accumulates():
     assert d._covered == 2
 
 
+def _rendered(d: LiveEvalProgress) -> str:
+    import io
+
+    out = Console(file=io.StringIO(), width=200)
+    out.print(d.__rich__())
+    return out.file.getvalue()
+
+
+def test_footer_sums_live_running_usage_with_completed():
+    # Mid-run spend visibility (51-min single-task dogfood: footer read
+    # "— tokens · $—" for the whole agent phase): the footer must sum the
+    # completed tasks' trusted telemetry PLUS every running session's live
+    # usage. Cost stays completed-only — $ exists only in the scoring-time
+    # LiteLLM gateway import.
+    with _registered_rollouts(
+        {"task-a": _counters_snap(1_000_000), "task-b": _counters_snap(500_000)}
+    ):
+        d = _dash()
+        d.on_plan(total=3, done=0, remaining=3)
+        for name in ("task-a", "task-b", "task-c"):
+            d.on_task_start(name)
+        d.on_result(
+            "task-c", _result(1.0, tokens=2_500_000, cost=0.02, src="provider_response")
+        )
+        text = _rendered(d)
+        assert "4.00M tokens" in text
+        assert "$0.02" in text
+
+
+def test_footer_shows_live_usage_before_any_completion():
+    # The single-task common case: no task has finished, but the running
+    # session has per-completed-prompt usage — real tokens render, and $
+    # stays "—" (no live price signal exists).
+    with _registered_rollouts({"task-a": _counters_snap(750_000)}):
+        d = _dash()
+        d.on_plan(total=1, done=0, remaining=1)
+        d.on_task_start("task-a")
+        text = _rendered(d)
+        assert "750.0k tokens" in text
+        assert "$—" in text
+        assert "— tokens" not in text
+
+
+def test_footer_degrades_to_dash_without_any_usage_signal():
+    # No completed telemetry and no live counters (counter-less snapshot, a
+    # teardown-racing rollout whose snapshot raises, an unregistered task):
+    # the footer keeps the "—" contract — a coverage-0 run reads broken, not
+    # free — and never raises.
+    with _registered_rollouts(
+        {
+            "warming": ActivitySnapshot("setup", None),
+            "racing": RuntimeError("teardown race"),
+        }
+    ):
+        d = _dash()
+        d.on_plan(total=3, done=0, remaining=3)
+        for name in ("warming", "racing", "unregistered"):
+            d.on_task_start(name)
+        text = _rendered(d)
+        assert "— tokens" in text
+        assert "$—" in text
+
+
+def test_footer_live_usage_ignores_sessions_without_usage():
+    # A running session before its first completed prompt reports
+    # total_tokens=None — it must contribute 0, not poison the sum.
+    with _registered_rollouts(
+        {"task-a": _counters_snap(None), "task-b": _counters_snap(250_000)}
+    ):
+        d = _dash()
+        d.on_plan(total=2, done=0, remaining=2)
+        d.on_task_start("task-a")
+        d.on_task_start("task-b")
+        assert "250.0k tokens" in _rendered(d)
+
+
 class _FakeSession:
+    distinct_tool_titles = 4
+
     def progress_snapshot(self):
         return 38, "file_editor"
 
@@ -101,29 +213,49 @@ class _FakeSession:
         return {"total_tokens": 1500}
 
 
-def test_activity_cell_polls_live_session_counters():
-    # Per-task activity in the running-now table (fresh-user dogfood
-    # 2026-08-09): the heartbeat's counters must reach the dashboard, since
-    # the Live mutes the logged heartbeat line itself.
-    from benchflow._utils import live_activity
-    from benchflow._utils.live_activity import ActivitySnapshot, SessionCounters
-    from benchflow.cli._live_progress import _activity_cell
-
-    live_activity.register(
-        "edit-pdf",
-        SimpleNamespace(
-            activity_snapshot=lambda: ActivitySnapshot(
-                "connected", SessionCounters(38, "file_editor", 1500)
-            )
+def test_activity_cell_formats_counters_and_registry_miss():
+    # Pure formatter over one polled snapshot — __rich__ polls the registry
+    # once per running task per frame and shares snapshots between the cells
+    # and the footer sum. distinct_tools=None (unknown, e.g. an old-style
+    # producer) must degrade to the full last:-suffixed cell, never mislabel.
+    snap = ActivitySnapshot("connected", SessionCounters(38, "file_editor", 1500))
+    assert _activity_cell(snap) == "38 calls · 1.5k tok · last: file_editor"
+    # Registry misses (pre-register, teardown races) hand the formatter None —
+    # a live row's cell must never be blank, and never raise.
+    assert _activity_cell(None) == "starting…"
+
+
+@pytest.mark.parametrize(
+    ("counters", "expected"),
+    [
+        # Single-tool agents (prime-agent funnels everything through one
+        # IPython tool): "last: IPython cell" is a constant across all N
+        # calls — once tokens are available they carry the information.
+        (
+            SessionCounters(38, "IPython cell", 412_000, 1),
+            "38 calls · 412.0k tok",
         ),
-    )
-    try:
-        assert _activity_cell("edit-pdf") == "38 calls · 1.5k tok · last: file_editor"
-    finally:
-        live_activity.unregister("edit-pdf")
-    # Unregistered tasks (pre-register, teardown races) degrade to the
-    # fallback label — a live row's cell must never be blank, and never raise.
-    assert _activity_cell("edit-pdf") == "starting…"
+        # Varied tool names: last: still carries information.
+        (
+            SessionCounters(38, "file_editor", 412_000, 5),
+            "38 calls · 412.0k tok · last: file_editor",
+        ),
+        # Constant tool but no tokens yet (first prompt still running): the
+        # name, shown once, is still the only signal — it stays.
+        (
+            SessionCounters(5, "IPython cell", None, 1),
+            "5 calls · last: IPython cell",
+        ),
+        # A lone first call is trivially "constant" but the name is still
+        # news; the drop only pays off once repetition makes it redundant.
+        (
+            SessionCounters(1, "IPython cell", 90_000, 1),
+            "1 calls · 90.0k tok · last: IPython cell",
+        ),
+    ],
+)
+def test_activity_cell_constant_vs_varied_tools(counters, expected):
+    assert _activity_cell(ActivitySnapshot("connected", counters)) == expected
 
 
 def test_rollout_activity_snapshot_reads_acp_session():
@@ -136,7 +268,7 @@ def test_rollout_activity_snapshot_reads_acp_session():
         _acp_client=SimpleNamespace(session=_FakeSession()), _phase="connected"
     )
     assert Rollout.activity_snapshot(connected) == ActivitySnapshot(
-        "connected", SessionCounters(38, "file_editor", 1500)
+        "connected", SessionCounters(38, "file_editor", 1500, 4)
     )
     # Pre-connect (and session-factory) rollouts have no client: counters are
     # None but the lifecycle phase still rides out so the cell can label it.
@@ -148,83 +280,36 @@ def test_rollout_activity_snapshot_reads_acp_session():
 def test_dashboard_renders_activity_for_registered_running_task():
     # End-to-end through __rich__: a registered running task's activity must
     # appear in the rendered panel — reverting the table wiring fails this.
-    import io
-
-    from benchflow._utils import live_activity
-    from benchflow._utils.live_activity import ActivitySnapshot, SessionCounters
-
-    live_activity.register(
-        "edit-pdf",
-        SimpleNamespace(
-            activity_snapshot=lambda: ActivitySnapshot(
-                "connected", SessionCounters(38, "file_editor", None)
-            )
-        ),
-    )
-    try:
+    with _registered_rollouts(
+        {"edit-pdf": _counters_snap(None, calls=38, last="file_editor")}
+    ):
         d = _dash()
         d.on_plan(total=1, done=0, remaining=1)
         d.on_task_start("edit-pdf")
-        out = Console(file=io.StringIO(), width=120)
-        out.print(d.__rich__())
-        text = out.file.getvalue()
+        text = _rendered(d)
         assert "38 calls" in text
         assert "file_editor" in text
-    finally:
-        live_activity.unregister("edit-pdf")
 
 
 def test_activity_cell_shows_phase_label_before_session_exists():
     # Fresh-user dogfood follow-up: ~1.5min of sandbox create / agent install
     # (and the whole verifier) used to render a blank cell — indistinguishable
     # from a hang. Counter-less snapshots must surface the lifecycle phase.
-    from benchflow._utils import live_activity
-    from benchflow._utils.live_activity import ActivitySnapshot
-    from benchflow.cli._live_progress import _activity_cell
-
-    phase = "setup"
-    live_activity.register(
-        "warming-up",
-        SimpleNamespace(activity_snapshot=lambda: ActivitySnapshot(phase, None)),
-    )
-    try:
-        assert _activity_cell("warming-up") == "creating sandbox…"
-        phase = "started"
-        assert _activity_cell("warming-up") == "installing agent…"
-        phase = "verifying"
-        assert _activity_cell("warming-up") == "verifying…"
-        # Unknown phases (e.g. "branched") still never blank the cell.
-        phase = "branched"
-        assert _activity_cell("warming-up") == "starting…"
-        d = _dash()
-        d.on_plan(total=1, done=0, remaining=1)
-        d.on_task_start("warming-up")
-        d.__rich__()  # builds the running-now table row; must not raise
-    finally:
-        live_activity.unregister("warming-up")
+    assert _activity_cell(ActivitySnapshot("setup", None)) == "creating sandbox…"
+    assert _activity_cell(ActivitySnapshot("started", None)) == "installing agent…"
+    assert _activity_cell(ActivitySnapshot("verifying", None)) == "verifying…"
+    # Unknown phases (e.g. "branched") still never blank the cell.
+    assert _activity_cell(ActivitySnapshot("branched", None)) == "starting…"
 
 
 def test_dashboard_renders_phase_label_for_counterless_task():
     # End-to-end through __rich__: the phase label must reach the rendered
     # panel, not just the cell helper — reverting the table wiring fails this.
-    import io
-
-    from benchflow._utils import live_activity
-    from benchflow._utils.live_activity import ActivitySnapshot
-
-    live_activity.register(
-        "edit-pdf",
-        SimpleNamespace(activity_snapshot=lambda: ActivitySnapshot("started", None)),
-    )
-    try:
+    with _registered_rollouts({"edit-pdf": ActivitySnapshot("started", None)}):
         d = _dash()
         d.on_plan(total=1, done=0, remaining=1)
         d.on_task_start("edit-pdf")
-        out = Console(file=io.StringIO(), width=120)
-        out.print(d.__rich__())
-        assert "installing agent…" in out.file.getvalue()
-    finally:
-        live_activity.unregister("edit-pdf")
+        assert "installing agent…" in _rendered(d)
 
 
 def test_rollout_verify_marks_verifying_phase():

{benchflow-0.6.7.dev1840 → benchflow-0.6.7.dev1841}/tests/test_console_progress.py

@@ -97,6 +97,56 @@ def test_progress_snapshot_whitespace_only_title(monkeypatch):
     assert s.progress_snapshot() == (1, "")
 
 
+def test_distinct_tool_titles_constant_vs_varied(monkeypatch):
+    """Incremental distinct-title counter behind the dashboard's single-tool
+    detection (prime-agent: N identical "IPython cell" calls)."""
+    monkeypatch.setenv("BENCHFLOW_PROGRESS", "off")
+    s = ACPSession("sid")
+    assert s.distinct_tool_titles == 0
+    _drive_tool_call(s, "tc1")
+    _drive_tool_call(s, "tc2")
+    assert s.distinct_tool_titles == 1  # same title twice stays constant
+    # The tool_call_update fallback path (unseen id) also records the title.
+    s.handle_update(
+        {
+            "sessionUpdate": "tool_call_update",
+            "toolCallId": "tc3",
+            "title": "file_editor",
+            "kind": "edit",
+            "status": "in_progress",
+        }
+    )
+    assert s.distinct_tool_titles == 2
+
+
+def test_distinct_tool_titles_normalize_like_the_display(monkeypatch):
+    """Distinct means "renders differently": multi-line titles compare by
+    first line (what the cell shows), and empty titles fall back to kind —
+    the same normalization progress_snapshot uses."""
+    monkeypatch.setenv("BENCHFLOW_PROGRESS", "off")
+    s = ACPSession("sid")
+    for call_id, title in (("a", "IPython cell\nx = 1"), ("b", "IPython cell\ny = 2")):
+        s.handle_update(
+            {
+                "sessionUpdate": "tool_call",
+                "toolCallId": call_id,
+                "title": title,
+                "kind": "execute",
+            }
+        )
+    assert s.distinct_tool_titles == 1
+    for call_id in ("c", "d"):
+        s.handle_update(
+            {
+                "sessionUpdate": "tool_call",
+                "toolCallId": call_id,
+                "title": "",
+                "kind": "execute",
+            }
+        )
+    assert s.distinct_tool_titles == 2  # "IPython cell" + kind-fallback "execute"
+
+
 def test_heartbeat_silent_outside_prompt(monkeypatch, caplog):
     """No heartbeat before the first prompt or after mark_prompt_end."""
     monkeypatch.setenv("BENCHFLOW_PROGRESS", "on")