benchflow 0.7.6.dev2233__tar.gz → 0.7.6.dev2248__tar.gz

{benchflow-0.7.6.dev2233 → benchflow-0.7.6.dev2248}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.6.dev2233
+Version: 0.7.6.dev2248
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.6.dev2233 → benchflow-0.7.6.dev2248}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.6.dev2233"
+version = "0.7.6.dev2248"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.6.dev2233 → benchflow-0.7.6.dev2248}/src/benchflow/acp/runtime.py

@@ -21,24 +21,27 @@ Does not own:
 import asyncio
 import contextlib
 import logging
-from datetime import UTC, datetime
 from pathlib import Path
 
 from benchflow.acp.client import ACPClient
 from benchflow.acp.container_transport import ContainerTransport
 from benchflow.acp.selection import selected_acp_transport
 from benchflow.acp.types import McpServerSpec
+from benchflow.acp.watchdog import IdleWatchdog
 from benchflow.agents.protocol import ACPSessionAdapter
 from benchflow.agents.providers import (
     find_provider,
     find_provider_for_bare_model,
     strip_provider_prefix,
 )
-from benchflow.agents.registry import AGENTS, OPENCODE_PROXY_PROVIDER_ID
+from benchflow.agents.registry import (
+    ACPX_KEY_PREFIX,
+    AGENTS,
+    OPENCODE_PROXY_PROVIDER_ID,
+)
 from benchflow.diagnostics import (
     AgentPromptTimeoutDiagnostic,
     AgentPromptTimeoutError,
-    IdleTimeoutDiagnostic,
     IdleTimeoutError,
     TransportClosedDiagnostic,
     TransportClosedError,
@@ -269,6 +272,13 @@ def _format_acp_model(model: str, agent: str) -> str:
     models.dev provider prefix when the agent requires it.
     """
     bare = strip_provider_prefix(model)
+    # Gemini CLI accepts bare Google model IDs. ``google/`` is a models.dev
+    # provider prefix rather than a registered BenchFlow provider, so the
+    # generic normalizer intentionally leaves it alone.
+    if agent.removeprefix(ACPX_KEY_PREFIX) == "gemini" and model.startswith(
+        ("google/gemini-", "google/gemma-")
+    ):
+        bare = model.removeprefix("google/")
     agent_cfg = AGENTS.get(agent)
     if agent_cfg and agent_cfg.acp_model_format == "registered-provider/model":
         # Proxy mode: BenchFlow's LiteLLM proxy serves the model under the alias
@@ -820,81 +830,29 @@ async def _prompt_with_idle_watchdog(
     timeout: int,
     idle_timeout: int,
 ):
-    """Run acp_client.prompt() with both a wall-clock and an idle watchdog.
-
-    The watchdog polls session.tool_calls every few seconds and aborts if no
-    progress was made in idle_timeout. This catches agents that hung silently
-    while the local process is still alive (no output to stdout, no tool calls
-    appended).
-    """
-
-    def _activity_count() -> int:
-        # Match the docstring contract: idle = no tool call AND no message
-        # AND no thought. Sum all three so streamed text resets the timer.
-        return (
-            len(session.tool_calls)
-            + len(session.message_chunks)
-            + len(session.thought_chunks)
-        )
-
+    """Run one ACP prompt with wall-clock and idle-watchdog budgets."""
     prompt_task = asyncio.create_task(acp_client.prompt(prompt))
-    last_progress = asyncio.get_event_loop().time()
-    last_activity_at = datetime.now(UTC)
-    last_count = _activity_count()
-    # poll_interval considers BOTH idle_timeout and wall-clock timeout so that
-    # short overall budgets don't overshoot (e.g. timeout=30s with default
-    # poll_interval=30s could overshoot 100%). Cap at 30s, floor at 1s.
-    poll_interval = max(1, min(30, idle_timeout // 4, max(1, timeout // 4)))
-    # deadline is loop-INVARIANT (fixed at loop start), NOT re-derived from
-    # last_progress inside the loop. This is load-bearing: the idle branch below
-    # resets last_progress while a tool call is in-flight, so re-deriving the
-    # deadline from it would let an agent that emits a tool_call and then hangs
-    # forever push the wall-clock backstop out indefinitely. Keep this fixed.
-    deadline = last_progress + timeout
+    loop = asyncio.get_running_loop()
+    watchdog = IdleWatchdog.start(
+        session,
+        idle_timeout_sec=idle_timeout,
+        wall_timeout_sec=timeout,
+        now=loop.time(),
+    )
 
     try:
         while not prompt_task.done():
-            await asyncio.sleep(poll_interval)
+            await asyncio.sleep(watchdog.poll_interval_sec)
             # Re-check done() after the sleep — the prompt may have completed
             # during the poll interval. Without this, we'd cancel an already-
             # completed task and discard a successful result.
             if prompt_task.done():
                 break
-            now = asyncio.get_event_loop().time()
-            cur_count = _activity_count()
-            if cur_count > last_count:
-                last_progress = now
-                last_activity_at = datetime.now(UTC)
-                last_count = cur_count
-            # An in-flight tool call means the agent is actively executing a tool
-            # (e.g. a long build/test/solver shell command), not hung. Those tools
-            # emit no ACP updates until they return, so a >idle_timeout run would
-            # otherwise false-fire the watchdog and discard real work. Treat a
-            # pending tool call as progress and defer to the wall-clock `timeout`
-            # backstop below for a tool that never returns. A genuine model-side
-            # hang has no pending tool call (the prior tool already completed via
-            # tool_call_update), so it still trips the idle path.
-            elif session.pending_tool_call_ids():
-                last_progress = now
-                last_activity_at = datetime.now(UTC)
-            if now - last_progress >= idle_timeout:
-                diag = IdleTimeoutDiagnostic(
-                    idle_timeout_sec=idle_timeout,
-                    idle_duration_sec=int(now - last_progress),
-                    wall_clock_elapsed_sec=int(now - (deadline - timeout)),
-                    n_tool_calls=len(session.tool_calls),
-                    n_message_chunks=len(session.message_chunks),
-                    n_thought_chunks=len(session.thought_chunks),
-                    last_activity_at=last_activity_at.isoformat(),
-                )
-                raise IdleTimeoutError(
-                    f"Agent idle for {idle_timeout}s with no new tool call, "
-                    f"message, or thought "
-                    f"(last activity {int(now - last_progress)}s ago, "
-                    f"{len(session.tool_calls)} tool calls so far)",
-                    diag,
-                )
-            if now > deadline:
+            now = loop.time()
+            watchdog.observe(session, now=now)
+            if watchdog.idle_expired(now):
+                raise watchdog.timeout_error(session, now=now)
+            if watchdog.wall_clock_expired(now):
                 if await _cancel_and_drain_prompt_task(prompt_task):
                     raise _agent_prompt_timeout_error(session, timeout)
                 raise TimeoutError(

{benchflow-0.7.6.dev2233 → benchflow-0.7.6.dev2248}/src/benchflow/acp/session.py

@@ -202,6 +202,11 @@ class ACPSession:
         self.thought_chunks: list[str] = []
         self.tool_calls: list[ToolCallRecord] = []
         self._tool_call_map: dict[str, ToolCallRecord] = {}
+        # Total update count is diagnostic. Per-call pending versions let the
+        # idle watchdog distinguish relevant streaming progress from repeated
+        # terminal updates for another call (PR #1066).
+        self.tool_call_update_count: int = 0
+        self._pending_tool_call_update_counts: dict[str, int] = {}
         # Distinct display titles across recorded tool calls, maintained
         # incrementally at record creation so the eval dashboard can detect
         # single-tool agents (prime-agent funnels everything through one
@@ -301,6 +306,20 @@ class ACPSession:
             if record.status in pending_statuses
         ]
 
+    def pending_tool_call_state(self) -> tuple[tuple[str, int], ...]:
+        """Return stable pending-call identities with progress versions."""
+        pending_statuses = {ToolCallStatus.PENDING, ToolCallStatus.IN_PROGRESS}
+        return tuple(
+            sorted(
+                (
+                    record.tool_call_id,
+                    self._pending_tool_call_update_counts.get(record.tool_call_id, 0),
+                )
+                for record in self.tool_calls
+                if record.status in pending_statuses
+            )
+        )
+
     def record_agent_timeout(
         self,
         *,
@@ -401,6 +420,7 @@ class ACPSession:
             self._record_tool_call(record)
 
         elif update_type == "tool_call_update":
+            self.tool_call_update_count += 1
             tc_id = update.get("toolCallId", "")
             record = self._tool_call_map.get(tc_id)
             if not record:
@@ -420,6 +440,10 @@ class ACPSession:
                 status = ToolCallStatus.IN_PROGRESS
             content = update.get("content")
             record.update_status(status, content)
+            if status in (ToolCallStatus.PENDING, ToolCallStatus.IN_PROGRESS):
+                self._pending_tool_call_update_counts[tc_id] = (
+                    self._pending_tool_call_update_counts.get(tc_id, 0) + 1
+                )
             # Canonicalize legacy OpenHands invoke_skill calls using the same
             # classifier the rescan path uses. Only upgrade to "skill"; never
             # downgrade, and never reclassify a tool that already has a real

benchflow-0.7.6.dev2248/src/benchflow/acp/watchdog.py

@@ -0,0 +1,249 @@
+"""Idle-watchdog state and pending-tool grace policy for ACP prompts."""
+
+from __future__ import annotations
+
+from collections.abc import Sized
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Protocol
+
+from benchflow.diagnostics import IdleTimeoutDiagnostic, IdleTimeoutError
+
+# A pending tool call defers the idle watchdog for at most this many idle
+# budgets while staying silent. Streaming updates restart the grace for their
+# own pending call; unrelated or terminal updates do not.
+PENDING_GRACE_MULTIPLIER = 3
+
+
+class WatchdogSession(Protocol):
+    """The small ACPSession surface consumed by the idle watchdog."""
+
+    tool_calls: Sized
+    message_chunks: Sized
+    thought_chunks: Sized
+    tool_call_update_count: int
+
+    def pending_tool_call_state(self) -> tuple[tuple[str, int], ...]: ...
+
+
+def _activity_count(session: WatchdogSession) -> int:
+    return (
+        len(session.tool_calls)
+        + len(session.message_chunks)
+        + len(session.thought_chunks)
+    )
+
+
+@dataclass
+class IdleWatchdog:
+    """Track prompt activity and enforce bounded pending-tool deferral."""
+
+    idle_timeout_sec: int
+    wall_timeout_sec: int
+    started_at: float
+    last_progress_at: float
+    last_activity_at: datetime
+    last_activity_count: int
+    pending_state: tuple[tuple[str, int], ...]
+    pending_last_progress_at: dict[str, float]
+    pending_last_update_at: dict[str, datetime]
+    pending_set_last_changed_at: datetime | None
+
+    @classmethod
+    def start(
+        cls,
+        session: WatchdogSession,
+        *,
+        idle_timeout_sec: int,
+        wall_timeout_sec: int,
+        now: float,
+        wall_now: datetime | None = None,
+    ) -> IdleWatchdog:
+        """Create a watchdog snapshot at prompt start."""
+        observed_at = wall_now or datetime.now(UTC)
+        pending_state = session.pending_tool_call_state()
+        return cls(
+            idle_timeout_sec=idle_timeout_sec,
+            wall_timeout_sec=wall_timeout_sec,
+            started_at=now,
+            last_progress_at=now,
+            last_activity_at=observed_at,
+            last_activity_count=_activity_count(session),
+            pending_state=pending_state,
+            pending_last_progress_at={call_id: now for call_id, _ in pending_state},
+            pending_last_update_at={},
+            pending_set_last_changed_at=observed_at if pending_state else None,
+        )
+
+    @property
+    def pending_grace_sec(self) -> int:
+        return self.idle_timeout_sec * PENDING_GRACE_MULTIPLIER
+
+    @property
+    def poll_interval_sec(self) -> int:
+        return max(
+            1,
+            min(
+                30,
+                self.idle_timeout_sec // 4,
+                max(1, self.wall_timeout_sec // 4),
+            ),
+        )
+
+    @property
+    def deadline(self) -> float:
+        return self.started_at + self.wall_timeout_sec
+
+    @property
+    def pending_ids(self) -> tuple[str, ...]:
+        return tuple(call_id for call_id, _version in self.pending_state)
+
+    @property
+    def n_pending_tool_updates(self) -> int:
+        return sum(version for _call_id, version in self.pending_state)
+
+    def observe(
+        self,
+        session: WatchdogSession,
+        *,
+        now: float,
+        wall_now: datetime | None = None,
+    ) -> None:
+        """Observe one poll and advance only the relevant grace clocks."""
+        observed_at = wall_now or datetime.now(UTC)
+        current_state = session.pending_tool_call_state()
+        previous_versions = dict(self.pending_state)
+        current_ids = tuple(call_id for call_id, _version in current_state)
+
+        if current_ids != self.pending_ids:
+            self.pending_set_last_changed_at = observed_at
+
+        for call_id, version in current_state:
+            if call_id not in previous_versions:
+                self.pending_last_progress_at[call_id] = now
+            if version > previous_versions.get(call_id, 0):
+                self.pending_last_progress_at[call_id] = now
+                self.pending_last_update_at[call_id] = observed_at
+
+        current_id_set = set(current_ids)
+        self.pending_last_progress_at = {
+            call_id: progress_at
+            for call_id, progress_at in self.pending_last_progress_at.items()
+            if call_id in current_id_set
+        }
+        self.pending_last_update_at = {
+            call_id: update_at
+            for call_id, update_at in self.pending_last_update_at.items()
+            if call_id in current_id_set
+        }
+
+        self.pending_state = current_state
+        current_activity_count = _activity_count(session)
+        if current_activity_count > self.last_activity_count:
+            self.last_progress_at = now
+            self.last_activity_at = observed_at
+            self.last_activity_count = current_activity_count
+        elif current_ids and not self.expired_pending_ids(now):
+            # Silent tools may legitimately run longer than idle_timeout. The
+            # grace, not unrelated session noise, owns this deferral.
+            self.last_progress_at = now
+            self.last_activity_at = observed_at
+
+    def expired_pending_ids(self, now: float) -> tuple[str, ...]:
+        """Pending calls whose own relevant progress grace has expired."""
+        return tuple(
+            call_id
+            for call_id in self.pending_ids
+            if now - self.pending_last_progress_at[call_id] >= self.pending_grace_sec
+        )
+
+    def idle_expired(self, now: float) -> bool:
+        return bool(self.expired_pending_ids(now)) or (
+            now - self.last_progress_at >= self.idle_timeout_sec
+        )
+
+    def wall_clock_expired(self, now: float) -> bool:
+        return now > self.deadline
+
+    def timeout_error(
+        self,
+        session: WatchdogSession,
+        *,
+        now: float,
+        wall_now: datetime | None = None,
+    ) -> IdleTimeoutError:
+        """Build a truthful idle error for generic or pending-grace expiry."""
+        fired_at = wall_now or datetime.now(UTC)
+        pending_ids = list(self.pending_ids)
+        expired_pending_ids = list(self.expired_pending_ids(now))
+        pending_versions = dict(self.pending_state)
+        n_expired_pending_updates = sum(
+            pending_versions[call_id] for call_id in expired_pending_ids
+        )
+        idle_duration = (
+            max(
+                now - self.pending_last_progress_at[call_id]
+                for call_id in expired_pending_ids
+            )
+            if expired_pending_ids
+            else now - self.last_progress_at
+        )
+        expired_update_times = [
+            self.pending_last_update_at[call_id]
+            for call_id in expired_pending_ids
+            if call_id in self.pending_last_update_at
+        ]
+        last_expired_update_at = (
+            max(expired_update_times) if expired_update_times else None
+        )
+        diagnostic = IdleTimeoutDiagnostic(
+            idle_timeout_sec=self.idle_timeout_sec,
+            idle_duration_sec=int(idle_duration),
+            wall_clock_elapsed_sec=int(now - self.started_at),
+            n_tool_calls=len(session.tool_calls),
+            n_message_chunks=len(session.message_chunks),
+            n_thought_chunks=len(session.thought_chunks),
+            last_activity_at=self.last_activity_at.isoformat(),
+            pending_tool_call_ids=pending_ids,
+            expired_pending_tool_call_ids=expired_pending_ids,
+            pending_grace_sec=self.pending_grace_sec,
+            pending_set_last_changed_at=(
+                self.pending_set_last_changed_at.isoformat()
+                if self.pending_set_last_changed_at is not None
+                else None
+            ),
+            last_tool_update_at=(
+                last_expired_update_at.isoformat()
+                if last_expired_update_at is not None
+                else None
+            ),
+            n_tool_call_updates=session.tool_call_update_count,
+            n_pending_tool_call_updates=self.n_pending_tool_updates,
+            n_expired_pending_tool_call_updates=n_expired_pending_updates,
+        )
+        if expired_pending_ids:
+            set_age = _age(fired_at, self.pending_set_last_changed_at)
+            update_age = _age(fired_at, last_expired_update_at)
+            return IdleTimeoutError(
+                f"Agent idle for {self.idle_timeout_sec}s: "
+                f"{len(expired_pending_ids)} pending tool call(s) exceeded the "
+                f"{self.pending_grace_sec}s pending grace "
+                f"({', '.join(expired_pending_ids)}; pending set last changed "
+                f"{set_age}, last pending-call update {update_age}, "
+                f"{n_expired_pending_updates} relevant updates seen, "
+                f"{len(session.tool_calls)} tool calls so far)",
+                diagnostic,
+            )
+        return IdleTimeoutError(
+            f"Agent idle for {self.idle_timeout_sec}s with no new tool call, "
+            f"message, or thought "
+            f"(last activity {int(now - self.last_progress_at)}s ago, "
+            f"{len(session.tool_calls)} tool calls so far)",
+            diagnostic,
+        )
+
+
+def _age(now: datetime, then: datetime | None) -> str:
+    if then is None:
+        return "never"
+    return f"{int((now - then).total_seconds())}s ago"

{benchflow-0.7.6.dev2233 → benchflow-0.7.6.dev2248}/src/benchflow/agents/registry.py

@@ -656,7 +656,11 @@ AGENTS: dict[str, AgentConfig] = {
         description="Google Gemini CLI via ACP",
         skill_paths=["$HOME/.gemini/skills"],
         install_cmd=_js_agent_install("gemini", "@google/gemini-cli@0.42.0"),
-        launch_cmd=_js_agent_launch("gemini", "--acp --yolo"),
+        # BenchFlow already isolates the agent inside the task sandbox and
+        # deliberately grants tool approval with --yolo. Gemini CLI 0.42 also
+        # requires the workspace to be trusted before it will honor that mode;
+        # headless ACP runs cannot answer the interactive trust prompt.
+        launch_cmd=_js_agent_launch("gemini", "--acp --yolo --skip-trust"),
         protocol="acp",
         # The Gemini CLI reads GEMINI_API_KEY natively. GOOGLE_API_KEY is
         # accepted as an alias: auto_inherit_env mirrors it both ways so users

{benchflow-0.7.6.dev2233 → benchflow-0.7.6.dev2248}/src/benchflow/diagnostics.py

@@ -132,18 +132,37 @@ class IdleTimeoutDiagnostic(Diagnostic):
     n_message_chunks: int = 0
     n_thought_chunks: int = 0
     last_activity_at: str = ""
+    # Pending-grace truth (PR #1066): when the fire happened because a
+    # pending tool call exhausted the grace window, these say so instead of
+    # letting the fields above imply pure silence. Additive — older
+    # result.json dicts round-trip through format_issue_from_dict with the
+    # defaults below.
+    pending_tool_call_ids: list[str] = field(default_factory=list)
+    expired_pending_tool_call_ids: list[str] = field(default_factory=list)
+    pending_grace_sec: int | None = None
+    pending_set_last_changed_at: str | None = None
+    last_tool_update_at: str | None = None
+    n_tool_call_updates: int = 0
+    n_pending_tool_call_updates: int = 0
+    n_expired_pending_tool_call_updates: int = 0
 
     field: ClassVar[str] = "idle_timeout_info"
     category: ClassVar[str | None] = "idle_timeout"
     summary_description: ClassVar[str] = "hit idle timeout"
 
     def format_issue(self, task_name: str) -> str:
-        return (
+        line = (
             f"{task_name}: idle timeout after "
             f"{self.idle_duration_sec}s idle "
             f"({self.n_tool_calls} tool calls, "
             f"{self.wall_clock_elapsed_sec}s wall)"
         )
+        if self.pending_tool_call_ids:
+            line += (
+                f" — {len(self.pending_tool_call_ids)} pending tool call(s) "
+                f"exceeded the {self.pending_grace_sec}s pending grace"
+            )
+        return line
 
 
 @dataclass

{benchflow-0.7.6.dev2233 → benchflow-0.7.6.dev2248}/src/benchflow/providers/litellm_config.py

@@ -375,6 +375,9 @@ def resolve_litellm_route(model: str, env: dict[str, str]) -> LiteLLMRoute:
     elif lower.startswith("gemini/"):
         upstream = model
         required = ("GEMINI_API_KEY",)
+    elif lower.startswith(("google/gemini-", "google/gemma-")):
+        upstream = f"gemini/{model.split('/', 1)[1]}"
+        required = ("GEMINI_API_KEY",)
     elif "gemini" in lower:
         upstream = f"gemini/{bare}"
         required = ("GEMINI_API_KEY",)

benchflow-0.7.6.dev2248/src/benchflow/providers/litellm_gemini_passthrough_patch.py

@@ -0,0 +1,116 @@
+"""Backport Gemini pass-through streaming logging for LiteLLM 1.89/1.91.
+
+Those releases classify every native ``generateContent`` URL as Vertex AI and
+send its streamed response through the Vertex logger.  That happens to work for
+Gemini model names present in both catalogs, but it rejects Google AI Studio
+Gemma names and drops the entire callback — including token usage and provider
+trajectory evidence.  LiteLLM added a dedicated Gemini endpoint type later.
+
+The proxy imports this module from ``sitecustomize``.  It patches only the old
+vendor shape (no ``EndpointType.GEMINI``) and only Google AI Studio-style model
+paths, leaving Vertex and every other provider untouched.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+_GOOGLE_AI_STUDIO_GENERATE_PATH_RE = re.compile(
+    r"/v1(?:beta)?/models/[^/:]+:(?:streamGenerateContent|generateContent)$"
+)
+
+
+def _is_google_ai_studio_generate_content(url_route: str) -> bool:
+    """Distinguish native AI Studio model paths from Vertex project paths."""
+    try:
+        path = urlparse(url_route).path
+    except Exception:
+        return False
+    return bool(
+        _GOOGLE_AI_STUDIO_GENERATE_PATH_RE.search(path)
+        and "/projects/" not in path
+        and "/locations/" not in path
+    )
+
+
+def _apply_patch() -> None:
+    try:
+        from litellm.proxy.pass_through_endpoints.llm_provider_handlers.gemini_passthrough_logging_handler import (
+            GeminiPassthroughLoggingHandler,
+        )
+        from litellm.proxy.pass_through_endpoints.streaming_handler import (
+            PassThroughStreamingHandler,
+        )
+        from litellm.types.passthrough_endpoints.pass_through_endpoints import (
+            EndpointType,
+        )
+    except Exception:
+        return
+
+    # Newer LiteLLM releases have their own Gemini dispatch branch.  Do not
+    # replace supported vendor behavior after the pinned dependency advances.
+    if hasattr(EndpointType, "GEMINI"):
+        return
+
+    original = PassThroughStreamingHandler._build_passthrough_logging_result
+    if getattr(original, "__benchflow_gemini_passthrough_patch__", False):
+        return
+
+    def build_passthrough_logging_result(
+        *,
+        litellm_logging_obj: Any,
+        passthrough_success_handler_obj: Any,
+        url_route: str,
+        request_body: dict[str, Any],
+        endpoint_type: Any,
+        start_time: Any,
+        raw_bytes: list[bytes],
+        end_time: Any,
+        model: str | None,
+    ) -> tuple[Any, dict[str, Any]]:
+        if not _is_google_ai_studio_generate_content(url_route):
+            return original(
+                litellm_logging_obj=litellm_logging_obj,
+                passthrough_success_handler_obj=passthrough_success_handler_obj,
+                url_route=url_route,
+                request_body=request_body,
+                endpoint_type=endpoint_type,
+                start_time=start_time,
+                raw_bytes=raw_bytes,
+                end_time=end_time,
+                model=model,
+            )
+
+        all_chunks = PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(
+            raw_bytes
+        )
+        result = (
+            GeminiPassthroughLoggingHandler._handle_logging_gemini_collected_chunks(
+                litellm_logging_obj=litellm_logging_obj,
+                passthrough_success_handler_obj=passthrough_success_handler_obj,
+                url_route=url_route,
+                request_body=request_body,
+                endpoint_type=endpoint_type,
+                start_time=start_time,
+                all_chunks=all_chunks,
+                end_time=end_time,
+                model=model,
+            )
+        )
+        return result["result"], result["kwargs"]
+
+    setattr(  # noqa: B010 - marker on the monkey-patched vendor function
+        build_passthrough_logging_result,
+        "__benchflow_gemini_passthrough_patch__",
+        True,
+    )
+    setattr(  # noqa: B010 - avoids static narrowing on the vendor API
+        PassThroughStreamingHandler,
+        "_build_passthrough_logging_result",
+        staticmethod(build_passthrough_logging_result),
+    )
+
+
+_apply_patch()