benchflow 0.5.0__tar.gz → 0.5.1.dev869__tar.gz

{benchflow-0.5.0 → benchflow-0.5.1.dev869}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.5.0
+Version: 0.5.1.dev869
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.5.0 → benchflow-0.5.1.dev869}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.5.0"
+version = "0.5.1.dev869"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.5.0 → benchflow-0.5.1.dev869}/src/benchflow/_utils/evaluation_results.py

@@ -17,6 +17,7 @@ from benchflow.trajectories.metrics import (
     count_skill_invocations,
     result_skill_invocations,
 )
+from benchflow.usage_tracking import is_trusted_usage_source
 
 # Phase keys produced by Rollout (see rollout.py — environment_setup,
 # agent_setup, agent_execution, verifier, total). Kept here so summary
@@ -35,7 +36,7 @@ def agent_result_from_rollout(result: RolloutResult) -> dict[str, Any]:
     n_skill_invocations = result.n_skill_invocations or count_skill_invocations(
         result.trajectory
     )
-    return {
+    agent_result = {
         "n_tool_calls": result.n_tool_calls,
         "n_skill_invocations": n_skill_invocations,
         "n_prompts": result.n_prompts,
@@ -48,6 +49,9 @@ def agent_result_from_rollout(result: RolloutResult) -> dict[str, Any]:
         "usage_source": result.usage_source,
         "price_source": result.price_source,
     }
+    if getattr(result, "usage_details", None) is not None:
+        agent_result["usage_details"] = result.usage_details
+    return agent_result
 
 
 def rollout_result_payload(
@@ -105,7 +109,7 @@ def usage_summary(results: dict[str, dict]) -> dict[str, Any]:
     covered = [
         r
         for r in completed
-        if (r.get("agent_result") or {}).get("usage_source") == "provider_response"
+        if is_trusted_usage_source((r.get("agent_result") or {}).get("usage_source"))
     ]
 
     def total(field: str) -> int:

{benchflow-0.5.0 → benchflow-0.5.1.dev869}/src/benchflow/acp/client.py

@@ -380,6 +380,7 @@ class ACPClient:
         # vendored ``StopReason`` enum so consumers keep ``.value`` / member
         # comparisons working.
         self._session.stop_reason = StopReason(prompt_result.stop_reason)
+        self._session.record_prompt_usage(getattr(prompt_result, "usage", None))
         return prompt_result
 
     async def cancel(self) -> None:

{benchflow-0.5.0 → benchflow-0.5.1.dev869}/src/benchflow/acp/session.py

@@ -3,6 +3,7 @@
 import logging
 from collections.abc import Callable
 from datetime import datetime
+from typing import Any
 
 from benchflow.trajectories.metrics import is_skill_invocation_event
 
@@ -15,6 +16,81 @@ from .types import (
 
 logger = logging.getLogger(__name__)
 
+ACPUsageSnapshot = dict[str, int | None]
+
+_ACP_USAGE_FIELDS: tuple[str, ...] = (
+    "input_tokens",
+    "output_tokens",
+    "total_tokens",
+    "cached_read_tokens",
+    "cached_write_tokens",
+    "thought_tokens",
+)
+
+
+def _coerce_usage_int(value: object) -> int | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return int(value)
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float | str | bytes | bytearray):
+        try:
+            return int(value)
+        except ValueError:
+            return None
+    try:
+        return int(str(value))
+    except ValueError:
+        return None
+
+
+def _usage_mapping(usage: object) -> dict[str, Any]:
+    if isinstance(usage, dict):
+        return {str(key): value for key, value in usage.items()}
+    dump = getattr(usage, "model_dump", None)
+    if callable(dump):
+        data = dump(by_alias=False, exclude_none=True)
+        if isinstance(data, dict):
+            alias_data = dump(by_alias=True, exclude_none=True)
+            if isinstance(alias_data, dict):
+                data = {**alias_data, **data}
+            return data
+    return {
+        field: getattr(usage, field)
+        for field in _ACP_USAGE_FIELDS
+        if hasattr(usage, field)
+    }
+
+
+def normalize_acp_usage(usage: object | None) -> ACPUsageSnapshot | None:
+    """Normalize SDK ACP usage into BenchFlow's snake_case token counters."""
+    if usage is None:
+        return None
+    raw = _usage_mapping(usage)
+    if not raw:
+        return None
+    aliases = {
+        "input_tokens": ("input_tokens", "inputTokens"),
+        "output_tokens": ("output_tokens", "outputTokens"),
+        "total_tokens": ("total_tokens", "totalTokens"),
+        "cached_read_tokens": ("cached_read_tokens", "cachedReadTokens"),
+        "cached_write_tokens": ("cached_write_tokens", "cachedWriteTokens"),
+        "thought_tokens": ("thought_tokens", "thoughtTokens"),
+    }
+    snapshot: ACPUsageSnapshot = {}
+    for field, names in aliases.items():
+        value = None
+        for name in names:
+            if name in raw:
+                value = raw[name]
+                break
+        snapshot[field] = _coerce_usage_int(value)
+    if all(value is None for value in snapshot.values()):
+        return None
+    return snapshot
+
 
 def _is_skill_tool_call(
     kind: object, title: object = "", content: object = None
@@ -93,6 +169,7 @@ class ACPSession:
         self.tool_calls: list[ToolCallRecord] = []
         self._tool_call_map: dict[str, ToolCallRecord] = {}
         self.stop_reason: StopReason | None = None
+        self.usage_snapshots: list[ACPUsageSnapshot] = []
         self.created_at = datetime.now()
         self.events: list[dict] = []
         self._pending_text: list[dict] = []
@@ -124,6 +201,20 @@ class ACPSession:
         self._flush_agent_text()
         self._notify_change()
 
+    def record_prompt_usage(self, usage: object | None) -> None:
+        """Record cumulative ACP token usage returned by session/prompt."""
+        snapshot = normalize_acp_usage(usage)
+        if snapshot is None:
+            return
+        self.usage_snapshots.append(snapshot)
+        self._notify_change()
+
+    def latest_usage_totals(self) -> ACPUsageSnapshot | None:
+        """Return the latest cumulative ACP usage snapshot, if any."""
+        if not self.usage_snapshots:
+            return None
+        return dict(self.usage_snapshots[-1])
+
     def _flush_agent_text(self) -> None:
         """Flush pending text events, merging consecutive same-type chunks."""
         if not self._pending_text:

{benchflow-0.5.0 → benchflow-0.5.1.dev869}/src/benchflow/agents/env.py

@@ -47,9 +47,17 @@ _CODEX_ACCESS_TOKEN_ENV = "CODEX_ACCESS_TOKEN"
 _CODEX_AUTH_JSON_ENV = "CODEX_AUTH_JSON"
 _CLAUDE_CODE_OAUTH_TOKEN_ENV = "CLAUDE_CODE_OAUTH_TOKEN"
 _CLAUDE_OAUTH_TOKEN_ENV = "CLAUDE_OAUTH_TOKEN"
+_SUBSCRIPTION_AUTH_MARKER = "_BENCHFLOW_SUBSCRIPTION_AUTH"
 _CUSTOM_OPENAI_ENDPOINT_KEYS = frozenset(
     {"BENCHFLOW_PROVIDER_BASE_URL", "OPENAI_BASE_URL"}
 )
+_LITELLM_RUNTIME_MARKER_KEYS = frozenset(
+    {
+        "BENCHFLOW_LITELLM_MASTER_KEY",
+        "BENCHFLOW_LITELLM_MODEL_ALIAS",
+        "BENCHFLOW_LITELLM_MODEL_VIA_ENV",
+    }
+)
 _CANONICAL_OPENAI_URL = "https://api.openai.com/v1"
 _GENERIC_PROVIDER_OVERRIDE_KEYS = frozenset(
     {
@@ -368,6 +376,59 @@ def _has_codex_auth_json_auth(
     ) and bool(agent_env.get(_CODEX_AUTH_JSON_ENV))
 
 
+def uses_native_subscription_auth(
+    agent: str,
+    model: str | None,
+    agent_env: dict[str, str],
+) -> bool:
+    """Return True when an agent should use CLI/subscription auth directly.
+
+    This is the Harbor-style split point: API-key runs can be routed through
+    LiteLLM, while subscription-auth runs stay on the native Codex/Claude ACP
+    path and report usage from the agent protocol response.
+    """
+    if agent_env.get("BENCHFLOW_PROVIDER_NAME") == "litellm" or any(
+        agent_env.get(key) for key in _LITELLM_RUNTIME_MARKER_KEYS
+    ):
+        return False
+
+    if agent == "codex-acp":
+        if agent_env.get("OPENAI_API_KEY"):
+            return False
+        required_key = "OPENAI_API_KEY"
+        if not _can_use_codex_subscription_auth(
+            agent,
+            model,
+            required_key,
+            agent_env,
+        ):
+            return False
+        return (
+            bool(agent_env.get(_CODEX_ACCESS_TOKEN_ENV))
+            or bool(agent_env.get(_CODEX_AUTH_JSON_ENV))
+            or agent_env.get(_SUBSCRIPTION_AUTH_MARKER) == "1"
+            or check_subscription_auth(agent, required_key)
+        )
+
+    if agent == "claude-agent-acp":
+        if agent_env.get("ANTHROPIC_API_KEY"):
+            return False
+        if model is not None:
+            from benchflow.agents.registry import infer_env_key_for_model
+
+            if infer_env_key_for_model(model) != "ANTHROPIC_API_KEY":
+                return False
+        return (
+            bool(agent_env.get(_CLAUDE_CODE_OAUTH_TOKEN_ENV))
+            or bool(agent_env.get(_CLAUDE_OAUTH_TOKEN_ENV))
+            or bool(agent_env.get("ANTHROPIC_AUTH_TOKEN"))
+            or agent_env.get(_SUBSCRIPTION_AUTH_MARKER) == "1"
+            or check_subscription_auth(agent, "ANTHROPIC_API_KEY")
+        )
+
+    return False
+
+
 def inject_vertex_credentials(agent_env: dict[str, str], model: str) -> None:
     """Inject ADC credentials and defaults for Vertex AI models."""
     from benchflow.agents.registry import is_vertex_model
@@ -668,7 +729,7 @@ def resolve_agent_env(
                 required_key,
                 agent_env,
             ) and check_subscription_auth(agent, required_key):
-                agent_env["_BENCHFLOW_SUBSCRIPTION_AUTH"] = "1"
+                agent_env[_SUBSCRIPTION_AUTH_MARKER] = "1"
                 logger.info(
                     "Using host subscription auth (no %s set)",
                     required_key,
@@ -701,7 +762,7 @@ def resolve_agent_env(
                     and _can_use_subscription_auth(agent, model, req_key, agent_env)
                     and check_subscription_auth(agent, req_key)
                 ):
-                    agent_env["_BENCHFLOW_SUBSCRIPTION_AUTH"] = "1"
+                    agent_env[_SUBSCRIPTION_AUTH_MARKER] = "1"
                     logger.info(
                         "Using host subscription auth (no %s set)",
                         req_key,

{benchflow-0.5.0 → benchflow-0.5.1.dev869}/src/benchflow/metrics.py

@@ -21,6 +21,7 @@ from benchflow._utils.scoring import (
     pass_rate_excl_errors,
 )
 from benchflow.trajectories.metrics import result_skill_invocations
+from benchflow.usage_tracking import is_trusted_usage_source
 
 logger = logging.getLogger(__name__)
 
@@ -199,7 +200,7 @@ class BenchmarkMetrics:
         return [
             t
             for t in self.tasks
-            if t.completed and t.usage_source == "provider_response"
+            if t.completed and is_trusted_usage_source(t.usage_source)
         ]
 
     @property

{benchflow-0.5.0 → benchflow-0.5.1.dev869}/src/benchflow/models.py

@@ -87,9 +87,10 @@ class RolloutResult:
         total_tokens: Sum of input, output, cache-read, and cache-creation tokens,
                       or None when provider telemetry was unavailable.
         cost_usd:     Provider cost estimate in USD, or None when unavailable.
-        usage_source: Provider telemetry source. One of "provider_response" or
-                      "unavailable".
+        usage_source: Token telemetry source. One of "provider_response",
+                      "agent_native_acp", or "unavailable".
         price_source: Pricing table version used for cost_usd, or None.
+        usage_details: Optional source-specific telemetry details.
         error:        Error description string, or None on success.
         error_category: Stable category for ``error``, or None on success.
         verifier_error: Verifier error description, or None if verifier succeeded
@@ -139,6 +140,7 @@ class RolloutResult:
         cost_usd: float | None = None,
         usage_source: str = "unavailable",
         price_source: str | None = None,
+        usage_details: dict[str, Any] | None = None,
         error: str | None = None,
         error_category: str | None = None,
         verifier_error: str | None = None,
@@ -170,6 +172,7 @@ class RolloutResult:
         self.cost_usd = cost_usd
         self.usage_source = usage_source
         self.price_source = price_source
+        self.usage_details = usage_details
         self.error = error
         self.error_category = error_category
         self.verifier_error = verifier_error

{benchflow-0.5.0 → benchflow-0.5.1.dev869}/src/benchflow/providers/litellm_logging.py

@@ -13,6 +13,7 @@ from benchflow.trajectories.types import (
     LLMResponse,
     Trajectory,
 )
+from benchflow.usage_tracking import usage_unavailable
 
 _PROVIDER_AUTH_STATUS_CODES = (401, 403)
 _STATUS_KEYS = {
@@ -353,19 +354,6 @@ def trajectory_from_litellm_callback_log(
     return trajectory
 
 
-def usage_unavailable() -> dict[str, Any]:
-    return {
-        "n_input_tokens": 0,
-        "n_output_tokens": 0,
-        "n_cache_read_tokens": 0,
-        "n_cache_creation_tokens": 0,
-        "total_tokens": 0,
-        "cost_usd": None,
-        "usage_source": "unavailable",
-        "price_source": None,
-    }
-
-
 def extract_usage_from_trajectory(
     trajectory: Trajectory | None,
     *,

{benchflow-0.5.0 → benchflow-0.5.1.dev869}/src/benchflow/providers/litellm_runtime.py

@@ -25,6 +25,7 @@ import httpx
 import yaml
 
 from benchflow.agents.codex_config import apply_codex_provider_config
+from benchflow.agents.env import uses_native_subscription_auth
 from benchflow.agents.registry import AGENTS
 from benchflow.providers.litellm_config import (
     LITELLM_MASTER_KEY_ENV,
@@ -38,10 +39,9 @@ from benchflow.providers.litellm_logging import (
     callback_module_source,
     extract_usage_from_trajectory,
     trajectory_from_litellm_callback_log,
-    usage_unavailable,
 )
 from benchflow.trajectories.types import Trajectory
-from benchflow.usage_tracking import UsageTrackingConfig
+from benchflow.usage_tracking import UsageTrackingConfig, usage_unavailable
 
 logger = logging.getLogger(__name__)
 
@@ -961,6 +961,13 @@ async def ensure_litellm_runtime(
             reason="usage_tracking=off leaves provider traffic untouched",
         )
 
+    if uses_native_subscription_auth(agent, model, agent_env):
+        return await _skip_litellm_runtime(
+            agent_env,
+            runtime,
+            reason="native subscription auth will use agent ACP usage telemetry",
+        )
+
     if not needs_litellm_runtime(agent, model):
         if usage_cfg.mode == "required" and agent != "oracle":
             raise RuntimeError(

{benchflow-0.5.0 → benchflow-0.5.1.dev869}/src/benchflow/rollout.py

@@ -103,7 +103,13 @@ from benchflow.trajectories._capture import (
 from benchflow.trajectories.metrics import count_skill_invocations
 from benchflow.trajectories.tree import RolloutNode, RolloutTree, Step
 from benchflow.trajectories.types import redact_acp_trajectory_jsonl
-from benchflow.usage_tracking import UsageTrackingConfig
+from benchflow.usage_tracking import (
+    USAGE_SOURCE_AGENT_NATIVE_ACP,
+    USAGE_SOURCE_PROVIDER_RESPONSE,
+    UsageTrackingConfig,
+    is_token_usage_available,
+    usage_unavailable,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -129,6 +135,75 @@ def _provider_auth_status_from_runtime(runtime: Any) -> int | None:
     return None
 
 
+_NATIVE_ACP_USAGE_SNAPSHOT_TO_RESULT = {
+    "input_tokens": "n_input_tokens",
+    "output_tokens": "n_output_tokens",
+    "cached_read_tokens": "n_cache_read_tokens",
+    "cached_write_tokens": "n_cache_creation_tokens",
+    "total_tokens": "total_tokens",
+}
+
+
+def _zero_native_acp_usage_metrics() -> dict[str, Any]:
+    return {**usage_unavailable(), "usage_details": {"thought_tokens": 0}}
+
+
+def _as_nonnegative_int(value: object) -> int:
+    if value is None:
+        return 0
+    if isinstance(value, bool):
+        return int(value)
+    if isinstance(value, int):
+        return max(value, 0)
+    if isinstance(value, float | str | bytes | bytearray):
+        try:
+            return max(int(value), 0)
+        except ValueError:
+            return 0
+    try:
+        return max(int(str(value)), 0)
+    except ValueError:
+        return 0
+
+
+def _native_acp_usage_delta(
+    previous: dict[str, int | None] | None,
+    current: dict[str, int | None],
+) -> dict[str, int]:
+    delta: dict[str, int] = {}
+    for usage_field in (
+        "input_tokens",
+        "output_tokens",
+        "cached_read_tokens",
+        "cached_write_tokens",
+        "thought_tokens",
+    ):
+        current_value = _as_nonnegative_int(current.get(usage_field))
+        previous_value = (
+            _as_nonnegative_int(previous.get(usage_field)) if previous else 0
+        )
+        delta[usage_field] = max(current_value - previous_value, 0)
+
+    current_total = current.get("total_tokens")
+    if current_total is not None:
+        current_value = _as_nonnegative_int(current_total)
+        previous_value = (
+            _as_nonnegative_int(previous.get("total_tokens"))
+            if previous and previous.get("total_tokens") is not None
+            else 0
+        )
+        delta["total_tokens"] = max(current_value - previous_value, 0)
+    else:
+        delta["total_tokens"] = (
+            delta["input_tokens"]
+            + delta["output_tokens"]
+            + delta["cached_read_tokens"]
+            + delta["cached_write_tokens"]
+            + delta["thought_tokens"]
+        )
+    return delta
+
+
 def _task_disallows_internet(task: Any) -> bool:
     """Return True when task.toml requests no internet for the agent task."""
     env_config = getattr(getattr(task, "config", None), "environment", None)
@@ -537,6 +612,7 @@ def _build_rollout_result(
     cost_usd: float | None = None,
     usage_source: str = "unavailable",
     price_source: str | None = None,
+    usage_details: dict[str, Any] | None = None,
     usage_tracking: dict[str, Any] | None = None,
     evolved_skills: dict[str, str] | None = None,
     source_provenance: dict[str, Any] | None = None,
@@ -588,6 +664,7 @@ def _build_rollout_result(
         cost_usd=cost_usd,
         usage_source=usage_source,
         price_source=price_source,
+        usage_details=usage_details,
         error=error,
         error_category=error_category,
         verifier_error=verifier_error,
@@ -615,6 +692,8 @@ def _build_rollout_result(
         "usage_source": result.usage_source,
         "price_source": result.price_source,
     }
+    if result.usage_details is not None:
+        agent_result["usage_details"] = result.usage_details
     final_metrics = final_metrics_from_agent_result(agent_result)
     trajectory_summary = trajectory_summary_from_events(
         trajectory,
@@ -1178,6 +1257,8 @@ class Rollout:
         self._task_skill_policy: TaskSkillPolicy | None = None
         self._usage_runtime: Any = None
         self._usage_metrics: dict[str, Any] = self._planes.extract_usage(None)
+        self._native_usage_metrics: dict[str, Any] = _zero_native_acp_usage_metrics()
+        self._native_usage_checkpoint: dict[str, int | None] | None = None
         # Provider 401/403 status snapshotted during cleanup, after the usage
         # proxy imports its captures (Daytona's SandboxUsageProxy only fills
         # trajectory on stop()). Read by _provider_auth_status() so ACP-error
@@ -1649,6 +1730,7 @@ class Rollout:
             agent_cwd=self._agent_cwd,
             reasoning_effort=cfg.primary_reasoning_effort,
         )
+        self._native_usage_checkpoint = None
         self._reapply_ask_user_handler()
         self._attach_trajectory_writer(rollout_dir)
 
@@ -1827,6 +1909,7 @@ class Rollout:
         self._n_tool_calls += new_tools
         self._executed_prompts.extend(effective_prompts)
         self._trajectory_source = "acp"
+        self._collect_native_acp_usage()
 
         # Grow the tree at Step-level granularity — one Step per ACP event
         # (tool_call, agent_message, agent_thought, user_message). A single
@@ -1859,6 +1942,46 @@ class Rollout:
         self._phase = "executed"
         return trajectory, n_tool_calls
 
+    def _collect_native_acp_usage(self) -> None:
+        """Accumulate ACP PromptResponse.usage deltas for native subscription runs."""
+        session = getattr(self, "_session", None)
+        latest_fn = getattr(session, "latest_usage_totals", None)
+        if not callable(latest_fn):
+            return
+        latest = latest_fn()
+        if not latest:
+            return
+        previous = getattr(self, "_native_usage_checkpoint", None)
+        delta = _native_acp_usage_delta(previous, latest)
+        self._native_usage_checkpoint = dict(latest)
+        if not any(delta.values()):
+            return
+
+        metrics = dict(
+            getattr(self, "_native_usage_metrics", _zero_native_acp_usage_metrics())
+        )
+        for (
+            snapshot_field,
+            result_field,
+        ) in _NATIVE_ACP_USAGE_SNAPSHOT_TO_RESULT.items():
+            if result_field == "total_tokens":
+                continue
+            metrics[result_field] = _as_nonnegative_int(metrics.get(result_field)) + (
+                delta.get(snapshot_field) or 0
+            )
+        metrics["total_tokens"] = _as_nonnegative_int(metrics.get("total_tokens")) + (
+            delta.get("total_tokens") or 0
+        )
+        details = dict(metrics.get("usage_details") or {})
+        details["thought_tokens"] = _as_nonnegative_int(
+            details.get("thought_tokens")
+        ) + (delta.get("thought_tokens") or 0)
+        metrics["usage_details"] = details
+        metrics["usage_source"] = USAGE_SOURCE_AGENT_NATIVE_ACP
+        metrics["cost_usd"] = None
+        metrics["price_source"] = None
+        self._native_usage_metrics = metrics
+
     def _build_step_batch(self, new_events: list[dict], new_tools: int) -> list[Step]:
         """Build one Step per ACP event from the events appended this execute.
 
@@ -2102,7 +2225,9 @@ class Rollout:
                 logger.warning(f"LLM trajectory write failed: {e}")
             finally:
                 self._usage_runtime = None
-            self._enforce_required_usage_tracking()
+
+        self._finalize_usage_metrics()
+        self._enforce_required_usage_tracking()
 
         if self._environment is not None:
             with contextlib.suppress(Exception):
@@ -2126,11 +2251,24 @@ class Rollout:
 
         self._phase = "cleaned"
 
+    def _finalize_usage_metrics(self) -> None:
+        """Prefer LiteLLM usage, otherwise use trusted native ACP usage."""
+        current_metrics = getattr(
+            self, "_usage_metrics", {"usage_source": "unavailable"}
+        )
+        if current_metrics.get("usage_source") == USAGE_SOURCE_PROVIDER_RESPONSE:
+            return
+        native_metrics = getattr(self, "_native_usage_metrics", None)
+        if isinstance(native_metrics, dict) and is_token_usage_available(
+            native_metrics
+        ):
+            self._usage_metrics = native_metrics
+
     def _enforce_required_usage_tracking(self) -> None:
         usage_cfg = self._config.usage_tracking.with_env_defaults()
         if usage_cfg.mode != "required" or self._config.primary_agent == "oracle":
             return
-        if self._usage_metrics.get("usage_source") == "provider_response":
+        if is_token_usage_available(getattr(self, "_usage_metrics", None)):
             return
         if self._error is not None:
             return
@@ -2721,7 +2859,7 @@ class Rollout:
         usage_source = str(self._usage_metrics.get("usage_source", "unavailable"))
         if usage_cfg.mode == "off":
             status = "off"
-        elif usage_source == "provider_response":
+        elif is_token_usage_available(self._usage_metrics):
             status = "enabled"
         else:
             status = "unavailable"

{benchflow-0.5.0 → benchflow-0.5.1.dev869}/src/benchflow/usage_tracking.py

@@ -7,8 +7,15 @@ from dataclasses import dataclass
 from typing import Any, Literal, cast
 
 UsageTrackingMode = Literal["auto", "required", "off"]
+UsageSource = Literal["provider_response", "agent_native_acp", "unavailable"]
 
 USAGE_TRACKING_ENV = "BENCHFLOW_USAGE_TRACKING"
+USAGE_SOURCE_PROVIDER_RESPONSE = "provider_response"
+USAGE_SOURCE_AGENT_NATIVE_ACP = "agent_native_acp"
+USAGE_SOURCE_UNAVAILABLE = "unavailable"
+TRUSTED_USAGE_SOURCES: frozenset[str] = frozenset(
+    {USAGE_SOURCE_PROVIDER_RESPONSE, USAGE_SOURCE_AGENT_NATIVE_ACP}
+)
 
 _MODES: set[str] = {"auto", "required", "off"}
 _LEGACY_USAGE_PROXY_KEYS: frozenset[str] = frozenset(
@@ -36,13 +43,39 @@ def _optional_mode(value: Any) -> UsageTrackingMode | None:
     return normalize_usage_tracking_mode(str(value))
 
 
+def is_trusted_usage_source(value: object) -> bool:
+    """Return True for usage telemetry sources that satisfy required tracking."""
+    return str(value) in TRUSTED_USAGE_SOURCES
+
+
+def is_token_usage_available(metrics: dict[str, Any] | None) -> bool:
+    """Return True when a usage metrics payload has trusted token telemetry."""
+    if not metrics:
+        return False
+    return is_trusted_usage_source(metrics.get("usage_source"))
+
+
+def usage_unavailable() -> dict[str, Any]:
+    """Return the canonical empty token-usage metrics payload."""
+    return {
+        "n_input_tokens": 0,
+        "n_output_tokens": 0,
+        "n_cache_read_tokens": 0,
+        "n_cache_creation_tokens": 0,
+        "total_tokens": 0,
+        "cost_usd": None,
+        "usage_source": USAGE_SOURCE_UNAVAILABLE,
+        "price_source": None,
+    }
+
+
 @dataclass(frozen=True, init=False)
 class UsageTrackingConfig:
     """User-facing token/cost telemetry policy.
 
     ``mode`` is the operator contract:
-    - ``auto`` records usage when the LiteLLM gateway can be started.
-    - ``required`` fails before the agent runs when telemetry cannot be wired.
+    - ``auto`` records usage when LiteLLM or native ACP telemetry can be used.
+    - ``required`` fails when no trusted token telemetry can be captured.
     - ``off`` leaves provider traffic untouched.
     """
 
@@ -119,6 +152,8 @@ class UsageTrackingConfig:
         endpoint_kind = "sandbox" if environment == "daytona" else "host"
         if self.mode == "off":
             endpoint_kind = "none"
+        elif usage_source == USAGE_SOURCE_AGENT_NATIVE_ACP:
+            endpoint_kind = "agent_native"
         return {
             "requested": self.mode,
             "status": status,