benchflow 0.6.7.dev1843__tar.gz → 0.6.7.dev1845__tar.gz

{benchflow-0.6.7.dev1843 → benchflow-0.6.7.dev1845}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1843
+Version: 0.6.7.dev1845
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1843 → benchflow-0.6.7.dev1845}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1843"
+version = "0.6.7.dev1845"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1843 → benchflow-0.6.7.dev1845}/src/benchflow/_utils/live_activity.py

@@ -18,10 +18,13 @@ from typing import Any, NamedTuple
 class SessionCounters(NamedTuple):
     """The live ACP session counters the console heartbeat logs.
 
-    ``total_tokens`` is None until the session has a usage snapshot (i.e.
-    after a completed prompt). ``distinct_tools`` counts distinct tool
-    display titles seen this session; None means unknown, which renderers
-    treat like varied (keep the ``last:`` cell).
+    ``total_tokens`` is None until a live usage signal exists: the producer
+    (``Rollout.activity_snapshot``) fills it with max(ACP prompt-completion
+    snapshot, LiteLLM gateway live-capture total), so it becomes non-None as
+    soon as the gateway has mirrored a usage-bearing record — mid-prompt —
+    even when no ACP prompt has completed yet. ``distinct_tools`` counts
+    distinct tool display titles seen this session; None means unknown, which
+    renderers treat like varied (keep the ``last:`` cell).
     """
 
     tool_calls: int

{benchflow-0.6.7.dev1843 → benchflow-0.6.7.dev1845}/src/benchflow/cli/_live_progress.py

@@ -162,9 +162,12 @@ def _activity_cell(snap: live_activity.ActivitySnapshot | None) -> str:
     which the Live display otherwise mutes. Until the agent session exists
     (and after it is closed for the verifier) the cell shows the rollout's
     lifecycle phase as a label instead, and a registry miss (None) shows the
-    fallback label, so the row is never blank. Tokens appear only once the
-    session has a usage snapshot (i.e. after a completed prompt); once they
-    do, a single-tool agent's cell is ``38 calls · 412.0k tok`` (no ``last:``).
+    fallback label, so the row is never blank. Tokens appear as soon as
+    either live signal exists — the gateway's live callback capture (per
+    completed LLM request, i.e. mid-prompt) or the ACP usage snapshot (per
+    completed prompt); ``Rollout.activity_snapshot`` reconciles the two as
+    max(). Once tokens are present, a single-tool agent's cell is
+    ``38 calls · 412.0k tok`` (no ``last:``).
     """
     if snap is None:
         return _PHASE_FALLBACK
@@ -379,12 +382,19 @@ class LiveEvalProgress:
             parts.append(tbl)
 
         # Footer: pass-rate (excl errors) + token/cost economics. Tokens sum
-        # the completed tasks' trusted telemetry PLUS the running sessions'
-        # live ACP usage (stepping forward as each prompt completes), so a
-        # 51-minute agent phase shows its spend while it runs, not only at
-        # scoring. The total is NOT monotonic across a task's completion
-        # boundary: scoring swaps the ACP self-report for the gateway
-        # measurement (which can be lower), and a task completing with
+        # the completed tasks' trusted telemetry PLUS the running rollouts'
+        # live usage — per running task, max(ACP prompt-completion snapshot,
+        # gateway live-capture total): the gateway side steps forward per
+        # completed LLM *request*, so a single-prompt agent phase shows its
+        # spend while the prompt runs, not only at scoring (the ACP side
+        # alone stays empty until the prompt completes). Reconciliation
+        # (documented at Rollout.activity_snapshot): max() keeps the running
+        # figure monotonic whichever signal leads, and degrades to the
+        # ACP-only value when the gateway capture is unavailable. The total
+        # is still NOT monotonic across a task's completion boundary: scoring
+        # swaps the live figure for the trusted gateway import (which can be
+        # lower than either live signal — untracked discards, cache
+        # accounting, ACP over-report), and a task completing with
         # untrusted/absent telemetry drops its live contribution entirely —
         # a sole-task footer can collapse back to "— tokens". Clamping would
         # falsify the trusted sum, so the dip is intended. Show "—" (not 0 /

{benchflow-0.6.7.dev1843 → benchflow-0.6.7.dev1845}/src/benchflow/contracts/planes.py

@@ -13,6 +13,22 @@ from typing import Any, Protocol, runtime_checkable
 from benchflow.environment.manifest import EnvironmentManifest
 
 
+class LiveUsageGateway(Protocol):
+    """A provider gateway process that reports live cumulative token usage.
+
+    The eval dashboard's mid-prompt token signal reads this accessor through
+    the rollout kernel (``Rollout.activity_snapshot`` →
+    ``_gateway_live_tokens``), which cannot import the concrete provider
+    plane — so the NAME is agreed here, at the contract boundary.
+    ``LiteLLMProcess`` asserts static conformance in
+    ``benchflow.providers.litellm_runtime``; renaming the accessor on either
+    side fails the type checker there instead of silently blanking the
+    dashboard signal.
+    """
+
+    def live_usage_tokens(self) -> int | None: ...
+
+
 @runtime_checkable
 class RolloutPlanes(Protocol):
     """Concrete-plane operations the rollout kernel needs."""

{benchflow-0.6.7.dev1843 → benchflow-0.6.7.dev1845}/src/benchflow/providers/litellm_runtime.py

@@ -19,7 +19,7 @@ import tempfile
 from collections.abc import Awaitable, Callable, Mapping
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, NoReturn, cast
+from typing import TYPE_CHECKING, Any, NoReturn, cast
 from uuid import uuid4
 
 import httpx
@@ -54,6 +54,9 @@ from benchflow.trajectories._llm_capture import LiveLLMTrajectoryWriter
 from benchflow.trajectories.types import Trajectory
 from benchflow.usage_tracking import UsageTrackingConfig, usage_unavailable
 
+if TYPE_CHECKING:
+    from benchflow.contracts.planes import LiveUsageGateway
+
 logger = logging.getLogger(__name__)
 
 LITELLM_VERSION_SPEC = "litellm[proxy]==1.89.0"
@@ -99,6 +102,10 @@ class LiteLLMProcess:
     trajectory: Trajectory | None
     session_id: str
     agent_name: str
+    # Live-capture token counter (class-level defaults so reads are plain
+    # attribute access even before start_live_capture ever runs).
+    _live_usage_total_tokens: int = 0
+    _live_usage_seen: bool = False
 
     @property
     def base_url(self) -> str:
@@ -126,8 +133,30 @@ class LiteLLMProcess:
         )
         self._live_callback_offset = 0
         self._live_callback_remainder = b""
+        # Clear ``seen`` BEFORE the total so a racing render-thread read can
+        # only observe None (fresh capture) — never a transient "0 tokens".
+        self._live_usage_seen = False
+        self._live_usage_total_tokens = 0
         self._live_capture_task = asyncio.create_task(self._live_capture_loop())
 
+    def live_usage_tokens(self) -> int | None:
+        """Cumulative provider tokens the live callback capture has mirrored.
+
+        Read-only, O(1) — safe to call from the dashboard's render thread while
+        the capture loop appends on the event loop (two monotonic attribute
+        reads; a torn read is at worst one record stale). None until a
+        usage-bearing gateway record has been parsed, so callers can tell "no
+        signal yet" from a genuine zero. The value updates per completed LLM
+        *request* — mid-prompt — which is exactly the granularity the ACP
+        per-completed-prompt snapshots lack (a single-prompt rollout otherwise
+        shows no tokens until scoring). It may lag the gateway log by the
+        capture cadence/backlog and is display-only: trusted usage still comes
+        from the full log import at ``stop()``.
+        """
+        if not self._live_usage_seen:
+            return None
+        return self._live_usage_total_tokens
+
     async def _read_callback_chunk(self, offset: int, limit: int) -> bytes:
         raise NotImplementedError
 
@@ -157,6 +186,19 @@ class LiteLLMProcess:
                 )
                 if parsed.exchanges:
                     trajectory.exchanges.extend(parsed.exchanges)
+                    # Live token counter for the eval dashboard: accumulate the
+                    # same canonical per-exchange accounting scoring sums at
+                    # stop() (Trajectory.total_provider_tokens over the same
+                    # parser's output), so the live figure converges to the
+                    # trusted total as the capture catches up. Failure records
+                    # carry no usage and leave the counter untouched.
+                    if parsed.has_provider_usage:
+                        # Total before ``seen``: a racing reader can never
+                        # observe seen=True with a not-yet-updated total.
+                        self._live_usage_total_tokens += int(
+                            parsed.total_provider_tokens
+                        )
+                        self._live_usage_seen = True
                     changed = True
             if len(chunk) < _LIVE_CAPTURE_CHUNK_BYTES:
                 break
@@ -189,6 +231,19 @@ class LiteLLMProcess:
             writer.reconcile(self.trajectory)
 
 
+def _live_usage_gateway_conformance(process: LiteLLMProcess) -> LiveUsageGateway:
+    """Static assertion: LiteLLMProcess satisfies the contract the rollout
+    kernel's dashboard read binds to (``contracts.planes.LiveUsageGateway``).
+
+    The kernel cannot import this module (#515 architecture invariant), so
+    its ``server.live_usage_tokens()`` call is dynamically typed there; this
+    return-type check is what makes a rename of the accessor — on either the
+    protocol or the process side — fail ``ty`` instead of silently blanking
+    the live token signal. Never called at runtime.
+    """
+    return process
+
+
 class HostLiteLLMProcess(LiteLLMProcess):
     def __init__(
         self,

{benchflow-0.6.7.dev1843 → benchflow-0.6.7.dev1845}/src/benchflow/providers/runtime.py

@@ -9,10 +9,13 @@ from __future__ import annotations
 
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from benchflow.usage_tracking import UsageTrackingConfig
 
+if TYPE_CHECKING:
+    from benchflow.providers.litellm_runtime import LiteLLMProcess
+
 
 @dataclass
 class ProviderRuntime:
@@ -21,7 +24,11 @@ class ProviderRuntime:
     kind: str
     agent_base_url: str
     backend_model: str | None = None
-    server: Any | None = None
+    # Typed (not Any) so cross-module reads of the server's accessors —
+    # e.g. the dashboard's live_usage_tokens dig in rollout — are bound by
+    # the type checker: a rename on LiteLLMProcess breaks ty, not silently
+    # blanks a signal behind a getattr default.
+    server: LiteLLMProcess | None = None
     config_key: str | None = None
     master_key: str | None = None
 

{benchflow-0.6.7.dev1843 → benchflow-0.6.7.dev1845}/src/benchflow/rollout/__init__.py

@@ -581,6 +581,43 @@ async def _run_environment_healthcheck(env: Any, task: Any) -> None:
             await asyncio.sleep(delay)
 
 
+def _gateway_live_tokens(runtime: Any) -> int | None:
+    """Cumulative provider tokens from the LiteLLM gateway's live capture.
+
+    Piggybacks on the llm_trajectory.jsonl mirror the proxy runtime already
+    runs for every rollout (``LiteLLMProcess.start_live_capture``, wired in
+    ``connect()`` and cancelled by the proxy's ``stop()`` in cleanup) — this
+    read generates no additional sandbox exec traffic, at any concurrency.
+
+    ``runtime`` stays ``Any`` because the rollout kernel must not import the
+    concrete provider plane (``benchflow.providers.runtime``; the #515
+    architecture test forbids even a TYPE_CHECKING import, and the planes
+    contract is deliberately Any-typed). Rename-safety for the
+    ``server.live_usage_tokens`` accessor lives where typing IS allowed: the
+    name is agreed in ``contracts.planes.LiveUsageGateway``, LiteLLMProcess
+    statically asserts conformance in the providers plane
+    (``_live_usage_gateway_conformance`` — a rename on either side fails
+    ty), and ``test_gateway_live_tokens_reach_rollout_activity_snapshot``
+    pins THIS call site to the real classes end-to-end (a drift here fails
+    pytest). The attribute access is direct (no getattr default) so the
+    failure mode under a fake without the accessor is the caught exception
+    below, not a permanent silent None.
+    Best-effort by the dashboard contract: any absence (no runtime, no
+    server, the accessor raising, a non-int value) degrades to None —
+    #963's ACP-only behavior — never to a render error.
+    """
+    if runtime is None:
+        return None
+    try:
+        server = runtime.server
+        if server is None:
+            return None
+        tokens = server.live_usage_tokens()
+    except Exception:
+        return None
+    return tokens if isinstance(tokens, int) else None
+
+
 class Rollout:
     """Decomposed trial lifecycle with independently-callable phases."""
 
@@ -765,17 +802,38 @@ class Rollout:
         ("creating sandbox…", "verifying…"), so a 90s sandbox create is not
         indistinguishable from a hang.
 
+        ``total_tokens`` reconciles two cumulative live signals as
+        ``max(acp_snapshot, gateway_live)``: the ACP session's self-reported
+        usage (updates only when a *prompt* completes — a single-prompt
+        rollout stays None for the whole agent phase) and the LiteLLM
+        gateway's live callback capture (updates per completed LLM *request*,
+        mid-prompt). max() because both inputs are non-decreasing counters, so
+        the cell/footer never step down mid-run whichever signal leads —
+        gateway-live can exceed the ACP self-report (requests the agent
+        discarded, cache accounting) and vice versa (capture lag behind large
+        log records) — and a dead gateway signal (None) degrades to exactly
+        the ACP-only behavior. The winner is display-only: at completion the
+        trusted scoring total replaces it (see the footer contract in
+        cli/_live_progress.py for the sanctioned non-monotonic boundary).
+
         Rollout owns the client/session dig so a rename breaks here — in
         typed, tested code — instead of silently blanking the dashboard cell
         (see benchflow._utils.live_activity). Session-factory agents have no
-        ACP client and always report counter-less snapshots.
+        ACP client and always report counter-less snapshots (their
+        gateway-live tokens are not surfaced — the counters seam is the only
+        token carrier).
         """
         session = self._acp_client.session if self._acp_client else None
         if session is None:
             return ActivitySnapshot(self._phase, None)
         calls, last_title = session.progress_snapshot()
         usage = session.latest_usage_totals()
-        tokens = usage.get("total_tokens") if usage else None
+        acp_tokens = usage.get("total_tokens") if usage else None
+        gateway_tokens = _gateway_live_tokens(self._usage_runtime)
+        if acp_tokens is None and gateway_tokens is None:
+            tokens = None
+        else:
+            tokens = max(acp_tokens or 0, gateway_tokens or 0)
         return ActivitySnapshot(
             self._phase,
             SessionCounters(calls, last_title, tokens, session.distinct_tool_titles),

{benchflow-0.6.7.dev1843 → benchflow-0.6.7.dev1845}/tests/test_cli_live_progress.py

@@ -265,7 +265,9 @@ def test_rollout_activity_snapshot_reads_acp_session():
     from benchflow.rollout import Rollout
 
     connected = SimpleNamespace(
-        _acp_client=SimpleNamespace(session=_FakeSession()), _phase="connected"
+        _acp_client=SimpleNamespace(session=_FakeSession()),
+        _phase="connected",
+        _usage_runtime=None,
     )
     assert Rollout.activity_snapshot(connected) == ActivitySnapshot(
         "connected", SessionCounters(38, "file_editor", 1500, 4)
@@ -277,6 +279,101 @@ def test_rollout_activity_snapshot_reads_acp_session():
     ) == ActivitySnapshot("setup", None)
 
 
+class _NoUsageSession(_FakeSession):
+    """An ACP session mid-first-prompt: counters exist, usage doesn't yet."""
+
+    def latest_usage_totals(self):
+        return None
+
+
+def _gateway_server(tokens):
+    if isinstance(tokens, Exception):
+
+        def _boom():
+            raise tokens
+
+        return SimpleNamespace(live_usage_tokens=_boom)
+    return SimpleNamespace(live_usage_tokens=lambda: tokens)
+
+
+def _rollout_ns(session, gateway):
+    return SimpleNamespace(
+        _acp_client=SimpleNamespace(session=session),
+        _phase="connected",
+        _usage_runtime=SimpleNamespace(server=_gateway_server(gateway)),
+    )
+
+
+@pytest.mark.parametrize(
+    ("session", "gateway", "expected_tokens"),
+    [
+        # The round-7 failure case: a single-prompt rollout has NO ACP usage
+        # snapshot until the prompt completes — the gateway live capture is
+        # the only mid-run signal and must carry the counters.
+        (_NoUsageSession(), 412_000, 412_000),
+        # Both signals: max() — whichever cumulative counter leads wins, so
+        # the footer never steps down mid-run (documented at
+        # Rollout.activity_snapshot).
+        (_FakeSession(), 900, 1500),
+        (_FakeSession(), 2_000, 2_000),
+        # Gateway signal dead (no usage-bearing record yet / reader raising /
+        # garbage): degrade to exactly #963's ACP-only behavior.
+        (_FakeSession(), None, 1500),
+        (_FakeSession(), RuntimeError("teardown race"), 1500),
+        (_FakeSession(), "not-an-int", 1500),
+        (_NoUsageSession(), None, None),
+        (_NoUsageSession(), RuntimeError("teardown race"), None),
+    ],
+)
+def test_rollout_activity_snapshot_reconciles_acp_and_gateway_usage(
+    session, gateway, expected_tokens
+):
+    from benchflow.rollout import Rollout
+
+    snap = Rollout.activity_snapshot(_rollout_ns(session, gateway))
+    assert snap.counters is not None
+    assert snap.counters.total_tokens == expected_tokens
+
+
+def test_rollout_activity_snapshot_without_usage_runtime_matches_963():
+    # A rollout whose proxy never started (oracle, native-subscription auth)
+    # has no _usage_runtime server — the ACP-only path must be untouched.
+    from benchflow.rollout import Rollout
+
+    ns = SimpleNamespace(
+        _acp_client=SimpleNamespace(session=_NoUsageSession()),
+        _phase="connected",
+        _usage_runtime=None,
+    )
+    snap = Rollout.activity_snapshot(ns)
+    assert snap.counters is not None
+    assert snap.counters.total_tokens is None
+
+
+def test_footer_and_cell_render_gateway_live_tokens_mid_prompt():
+    # End-to-end through __rich__ with the REAL Rollout.activity_snapshot dig:
+    # a single-prompt run mid-prompt (no ACP usage) whose gateway capture has
+    # tokens must light BOTH the footer sum and the activity cell — the
+    # e2e acceptance criterion, unit-shaped.
+    from benchflow.rollout import Rollout
+
+    session = _NoUsageSession()
+    session.distinct_tool_titles = 1  # single-tool agent (prime-agent shape)
+    ns = _rollout_ns(session, 412_000)
+    rollout = SimpleNamespace(activity_snapshot=lambda: Rollout.activity_snapshot(ns))
+    live_activity.register("dialogue-parser", rollout)
+    try:
+        d = _dash()
+        d.on_plan(total=1, done=0, remaining=1)
+        d.on_task_start("dialogue-parser")
+        text = _rendered(d)
+    finally:
+        live_activity.unregister("dialogue-parser")
+    assert "412.0k tokens" in text  # footer live sum
+    assert "38 calls · 412.0k tok" in text  # constant-tool activity cell
+    assert "— tokens" not in text
+
+
 def test_dashboard_renders_activity_for_registered_running_task():
     # End-to-end through __rich__: a registered running task's activity must
     # appear in the rendered panel — reverting the table wiring fails this.