benchflow 0.7.6.dev2231__tar.gz → 0.7.6.dev2235__tar.gz

{benchflow-0.7.6.dev2231 → benchflow-0.7.6.dev2235}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.6.dev2231
+Version: 0.7.6.dev2235
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.6.dev2231 → benchflow-0.7.6.dev2235}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.6.dev2231"
+version = "0.7.6.dev2235"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.6.dev2231 → benchflow-0.7.6.dev2235}/src/benchflow/_utils/scoring.py

@@ -186,6 +186,8 @@ def _looks_like_infra_error(error: str) -> bool:
             # Predates the marker above and is kept for strings rebuilt
             # outside that boundary (e.g. a verifier error re-raised as text).
             "failed to get session command",
+            "failed to execute session command",
+            "command timed out after",
             "sandbox not found",
             "workspace not found",
             "api connection",

{benchflow-0.7.6.dev2231 → benchflow-0.7.6.dev2235}/src/benchflow/acp/runtime.py

@@ -21,13 +21,13 @@ Does not own:
 import asyncio
 import contextlib
 import logging
-from datetime import UTC, datetime
 from pathlib import Path
 
 from benchflow.acp.client import ACPClient
 from benchflow.acp.container_transport import ContainerTransport
 from benchflow.acp.selection import selected_acp_transport
 from benchflow.acp.types import McpServerSpec
+from benchflow.acp.watchdog import IdleWatchdog
 from benchflow.agents.protocol import ACPSessionAdapter
 from benchflow.agents.providers import (
     find_provider,
@@ -38,7 +38,6 @@ from benchflow.agents.registry import AGENTS, OPENCODE_PROXY_PROVIDER_ID
 from benchflow.diagnostics import (
     AgentPromptTimeoutDiagnostic,
     AgentPromptTimeoutError,
-    IdleTimeoutDiagnostic,
     IdleTimeoutError,
     TransportClosedDiagnostic,
     TransportClosedError,
@@ -820,81 +819,29 @@ async def _prompt_with_idle_watchdog(
     timeout: int,
     idle_timeout: int,
 ):
-    """Run acp_client.prompt() with both a wall-clock and an idle watchdog.
-
-    The watchdog polls session.tool_calls every few seconds and aborts if no
-    progress was made in idle_timeout. This catches agents that hung silently
-    while the local process is still alive (no output to stdout, no tool calls
-    appended).
-    """
-
-    def _activity_count() -> int:
-        # Match the docstring contract: idle = no tool call AND no message
-        # AND no thought. Sum all three so streamed text resets the timer.
-        return (
-            len(session.tool_calls)
-            + len(session.message_chunks)
-            + len(session.thought_chunks)
-        )
-
+    """Run one ACP prompt with wall-clock and idle-watchdog budgets."""
     prompt_task = asyncio.create_task(acp_client.prompt(prompt))
-    last_progress = asyncio.get_event_loop().time()
-    last_activity_at = datetime.now(UTC)
-    last_count = _activity_count()
-    # poll_interval considers BOTH idle_timeout and wall-clock timeout so that
-    # short overall budgets don't overshoot (e.g. timeout=30s with default
-    # poll_interval=30s could overshoot 100%). Cap at 30s, floor at 1s.
-    poll_interval = max(1, min(30, idle_timeout // 4, max(1, timeout // 4)))
-    # deadline is loop-INVARIANT (fixed at loop start), NOT re-derived from
-    # last_progress inside the loop. This is load-bearing: the idle branch below
-    # resets last_progress while a tool call is in-flight, so re-deriving the
-    # deadline from it would let an agent that emits a tool_call and then hangs
-    # forever push the wall-clock backstop out indefinitely. Keep this fixed.
-    deadline = last_progress + timeout
+    loop = asyncio.get_running_loop()
+    watchdog = IdleWatchdog.start(
+        session,
+        idle_timeout_sec=idle_timeout,
+        wall_timeout_sec=timeout,
+        now=loop.time(),
+    )
 
     try:
         while not prompt_task.done():
-            await asyncio.sleep(poll_interval)
+            await asyncio.sleep(watchdog.poll_interval_sec)
             # Re-check done() after the sleep — the prompt may have completed
             # during the poll interval. Without this, we'd cancel an already-
             # completed task and discard a successful result.
             if prompt_task.done():
                 break
-            now = asyncio.get_event_loop().time()
-            cur_count = _activity_count()
-            if cur_count > last_count:
-                last_progress = now
-                last_activity_at = datetime.now(UTC)
-                last_count = cur_count
-            # An in-flight tool call means the agent is actively executing a tool
-            # (e.g. a long build/test/solver shell command), not hung. Those tools
-            # emit no ACP updates until they return, so a >idle_timeout run would
-            # otherwise false-fire the watchdog and discard real work. Treat a
-            # pending tool call as progress and defer to the wall-clock `timeout`
-            # backstop below for a tool that never returns. A genuine model-side
-            # hang has no pending tool call (the prior tool already completed via
-            # tool_call_update), so it still trips the idle path.
-            elif session.pending_tool_call_ids():
-                last_progress = now
-                last_activity_at = datetime.now(UTC)
-            if now - last_progress >= idle_timeout:
-                diag = IdleTimeoutDiagnostic(
-                    idle_timeout_sec=idle_timeout,
-                    idle_duration_sec=int(now - last_progress),
-                    wall_clock_elapsed_sec=int(now - (deadline - timeout)),
-                    n_tool_calls=len(session.tool_calls),
-                    n_message_chunks=len(session.message_chunks),
-                    n_thought_chunks=len(session.thought_chunks),
-                    last_activity_at=last_activity_at.isoformat(),
-                )
-                raise IdleTimeoutError(
-                    f"Agent idle for {idle_timeout}s with no new tool call, "
-                    f"message, or thought "
-                    f"(last activity {int(now - last_progress)}s ago, "
-                    f"{len(session.tool_calls)} tool calls so far)",
-                    diag,
-                )
-            if now > deadline:
+            now = loop.time()
+            watchdog.observe(session, now=now)
+            if watchdog.idle_expired(now):
+                raise watchdog.timeout_error(session, now=now)
+            if watchdog.wall_clock_expired(now):
                 if await _cancel_and_drain_prompt_task(prompt_task):
                     raise _agent_prompt_timeout_error(session, timeout)
                 raise TimeoutError(

{benchflow-0.7.6.dev2231 → benchflow-0.7.6.dev2235}/src/benchflow/acp/session.py

@@ -202,6 +202,11 @@ class ACPSession:
         self.thought_chunks: list[str] = []
         self.tool_calls: list[ToolCallRecord] = []
         self._tool_call_map: dict[str, ToolCallRecord] = {}
+        # Total update count is diagnostic. Per-call pending versions let the
+        # idle watchdog distinguish relevant streaming progress from repeated
+        # terminal updates for another call (PR #1066).
+        self.tool_call_update_count: int = 0
+        self._pending_tool_call_update_counts: dict[str, int] = {}
         # Distinct display titles across recorded tool calls, maintained
         # incrementally at record creation so the eval dashboard can detect
         # single-tool agents (prime-agent funnels everything through one
@@ -301,6 +306,20 @@ class ACPSession:
             if record.status in pending_statuses
         ]
 
+    def pending_tool_call_state(self) -> tuple[tuple[str, int], ...]:
+        """Return stable pending-call identities with progress versions."""
+        pending_statuses = {ToolCallStatus.PENDING, ToolCallStatus.IN_PROGRESS}
+        return tuple(
+            sorted(
+                (
+                    record.tool_call_id,
+                    self._pending_tool_call_update_counts.get(record.tool_call_id, 0),
+                )
+                for record in self.tool_calls
+                if record.status in pending_statuses
+            )
+        )
+
     def record_agent_timeout(
         self,
         *,
@@ -401,6 +420,7 @@ class ACPSession:
             self._record_tool_call(record)
 
         elif update_type == "tool_call_update":
+            self.tool_call_update_count += 1
             tc_id = update.get("toolCallId", "")
             record = self._tool_call_map.get(tc_id)
             if not record:
@@ -420,6 +440,10 @@ class ACPSession:
                 status = ToolCallStatus.IN_PROGRESS
             content = update.get("content")
             record.update_status(status, content)
+            if status in (ToolCallStatus.PENDING, ToolCallStatus.IN_PROGRESS):
+                self._pending_tool_call_update_counts[tc_id] = (
+                    self._pending_tool_call_update_counts.get(tc_id, 0) + 1
+                )
             # Canonicalize legacy OpenHands invoke_skill calls using the same
             # classifier the rescan path uses. Only upgrade to "skill"; never
             # downgrade, and never reclassify a tool that already has a real

benchflow-0.7.6.dev2235/src/benchflow/acp/watchdog.py

@@ -0,0 +1,249 @@
+"""Idle-watchdog state and pending-tool grace policy for ACP prompts."""
+
+from __future__ import annotations
+
+from collections.abc import Sized
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Protocol
+
+from benchflow.diagnostics import IdleTimeoutDiagnostic, IdleTimeoutError
+
+# A pending tool call defers the idle watchdog for at most this many idle
+# budgets while staying silent. Streaming updates restart the grace for their
+# own pending call; unrelated or terminal updates do not.
+PENDING_GRACE_MULTIPLIER = 3
+
+
+class WatchdogSession(Protocol):
+    """The small ACPSession surface consumed by the idle watchdog."""
+
+    tool_calls: Sized
+    message_chunks: Sized
+    thought_chunks: Sized
+    tool_call_update_count: int
+
+    def pending_tool_call_state(self) -> tuple[tuple[str, int], ...]: ...
+
+
+def _activity_count(session: WatchdogSession) -> int:
+    return (
+        len(session.tool_calls)
+        + len(session.message_chunks)
+        + len(session.thought_chunks)
+    )
+
+
+@dataclass
+class IdleWatchdog:
+    """Track prompt activity and enforce bounded pending-tool deferral."""
+
+    idle_timeout_sec: int
+    wall_timeout_sec: int
+    started_at: float
+    last_progress_at: float
+    last_activity_at: datetime
+    last_activity_count: int
+    pending_state: tuple[tuple[str, int], ...]
+    pending_last_progress_at: dict[str, float]
+    pending_last_update_at: dict[str, datetime]
+    pending_set_last_changed_at: datetime | None
+
+    @classmethod
+    def start(
+        cls,
+        session: WatchdogSession,
+        *,
+        idle_timeout_sec: int,
+        wall_timeout_sec: int,
+        now: float,
+        wall_now: datetime | None = None,
+    ) -> IdleWatchdog:
+        """Create a watchdog snapshot at prompt start."""
+        observed_at = wall_now or datetime.now(UTC)
+        pending_state = session.pending_tool_call_state()
+        return cls(
+            idle_timeout_sec=idle_timeout_sec,
+            wall_timeout_sec=wall_timeout_sec,
+            started_at=now,
+            last_progress_at=now,
+            last_activity_at=observed_at,
+            last_activity_count=_activity_count(session),
+            pending_state=pending_state,
+            pending_last_progress_at={call_id: now for call_id, _ in pending_state},
+            pending_last_update_at={},
+            pending_set_last_changed_at=observed_at if pending_state else None,
+        )
+
+    @property
+    def pending_grace_sec(self) -> int:
+        return self.idle_timeout_sec * PENDING_GRACE_MULTIPLIER
+
+    @property
+    def poll_interval_sec(self) -> int:
+        return max(
+            1,
+            min(
+                30,
+                self.idle_timeout_sec // 4,
+                max(1, self.wall_timeout_sec // 4),
+            ),
+        )
+
+    @property
+    def deadline(self) -> float:
+        return self.started_at + self.wall_timeout_sec
+
+    @property
+    def pending_ids(self) -> tuple[str, ...]:
+        return tuple(call_id for call_id, _version in self.pending_state)
+
+    @property
+    def n_pending_tool_updates(self) -> int:
+        return sum(version for _call_id, version in self.pending_state)
+
+    def observe(
+        self,
+        session: WatchdogSession,
+        *,
+        now: float,
+        wall_now: datetime | None = None,
+    ) -> None:
+        """Observe one poll and advance only the relevant grace clocks."""
+        observed_at = wall_now or datetime.now(UTC)
+        current_state = session.pending_tool_call_state()
+        previous_versions = dict(self.pending_state)
+        current_ids = tuple(call_id for call_id, _version in current_state)
+
+        if current_ids != self.pending_ids:
+            self.pending_set_last_changed_at = observed_at
+
+        for call_id, version in current_state:
+            if call_id not in previous_versions:
+                self.pending_last_progress_at[call_id] = now
+            if version > previous_versions.get(call_id, 0):
+                self.pending_last_progress_at[call_id] = now
+                self.pending_last_update_at[call_id] = observed_at
+
+        current_id_set = set(current_ids)
+        self.pending_last_progress_at = {
+            call_id: progress_at
+            for call_id, progress_at in self.pending_last_progress_at.items()
+            if call_id in current_id_set
+        }
+        self.pending_last_update_at = {
+            call_id: update_at
+            for call_id, update_at in self.pending_last_update_at.items()
+            if call_id in current_id_set
+        }
+
+        self.pending_state = current_state
+        current_activity_count = _activity_count(session)
+        if current_activity_count > self.last_activity_count:
+            self.last_progress_at = now
+            self.last_activity_at = observed_at
+            self.last_activity_count = current_activity_count
+        elif current_ids and not self.expired_pending_ids(now):
+            # Silent tools may legitimately run longer than idle_timeout. The
+            # grace, not unrelated session noise, owns this deferral.
+            self.last_progress_at = now
+            self.last_activity_at = observed_at
+
+    def expired_pending_ids(self, now: float) -> tuple[str, ...]:
+        """Pending calls whose own relevant progress grace has expired."""
+        return tuple(
+            call_id
+            for call_id in self.pending_ids
+            if now - self.pending_last_progress_at[call_id] >= self.pending_grace_sec
+        )
+
+    def idle_expired(self, now: float) -> bool:
+        return bool(self.expired_pending_ids(now)) or (
+            now - self.last_progress_at >= self.idle_timeout_sec
+        )
+
+    def wall_clock_expired(self, now: float) -> bool:
+        return now > self.deadline
+
+    def timeout_error(
+        self,
+        session: WatchdogSession,
+        *,
+        now: float,
+        wall_now: datetime | None = None,
+    ) -> IdleTimeoutError:
+        """Build a truthful idle error for generic or pending-grace expiry."""
+        fired_at = wall_now or datetime.now(UTC)
+        pending_ids = list(self.pending_ids)
+        expired_pending_ids = list(self.expired_pending_ids(now))
+        pending_versions = dict(self.pending_state)
+        n_expired_pending_updates = sum(
+            pending_versions[call_id] for call_id in expired_pending_ids
+        )
+        idle_duration = (
+            max(
+                now - self.pending_last_progress_at[call_id]
+                for call_id in expired_pending_ids
+            )
+            if expired_pending_ids
+            else now - self.last_progress_at
+        )
+        expired_update_times = [
+            self.pending_last_update_at[call_id]
+            for call_id in expired_pending_ids
+            if call_id in self.pending_last_update_at
+        ]
+        last_expired_update_at = (
+            max(expired_update_times) if expired_update_times else None
+        )
+        diagnostic = IdleTimeoutDiagnostic(
+            idle_timeout_sec=self.idle_timeout_sec,
+            idle_duration_sec=int(idle_duration),
+            wall_clock_elapsed_sec=int(now - self.started_at),
+            n_tool_calls=len(session.tool_calls),
+            n_message_chunks=len(session.message_chunks),
+            n_thought_chunks=len(session.thought_chunks),
+            last_activity_at=self.last_activity_at.isoformat(),
+            pending_tool_call_ids=pending_ids,
+            expired_pending_tool_call_ids=expired_pending_ids,
+            pending_grace_sec=self.pending_grace_sec,
+            pending_set_last_changed_at=(
+                self.pending_set_last_changed_at.isoformat()
+                if self.pending_set_last_changed_at is not None
+                else None
+            ),
+            last_tool_update_at=(
+                last_expired_update_at.isoformat()
+                if last_expired_update_at is not None
+                else None
+            ),
+            n_tool_call_updates=session.tool_call_update_count,
+            n_pending_tool_call_updates=self.n_pending_tool_updates,
+            n_expired_pending_tool_call_updates=n_expired_pending_updates,
+        )
+        if expired_pending_ids:
+            set_age = _age(fired_at, self.pending_set_last_changed_at)
+            update_age = _age(fired_at, last_expired_update_at)
+            return IdleTimeoutError(
+                f"Agent idle for {self.idle_timeout_sec}s: "
+                f"{len(expired_pending_ids)} pending tool call(s) exceeded the "
+                f"{self.pending_grace_sec}s pending grace "
+                f"({', '.join(expired_pending_ids)}; pending set last changed "
+                f"{set_age}, last pending-call update {update_age}, "
+                f"{n_expired_pending_updates} relevant updates seen, "
+                f"{len(session.tool_calls)} tool calls so far)",
+                diagnostic,
+            )
+        return IdleTimeoutError(
+            f"Agent idle for {self.idle_timeout_sec}s with no new tool call, "
+            f"message, or thought "
+            f"(last activity {int(now - self.last_progress_at)}s ago, "
+            f"{len(session.tool_calls)} tool calls so far)",
+            diagnostic,
+        )
+
+
+def _age(now: datetime, then: datetime | None) -> str:
+    if then is None:
+        return "never"
+    return f"{int((now - then).total_seconds())}s ago"

{benchflow-0.7.6.dev2231 → benchflow-0.7.6.dev2235}/src/benchflow/diagnostics.py

@@ -132,18 +132,37 @@ class IdleTimeoutDiagnostic(Diagnostic):
     n_message_chunks: int = 0
     n_thought_chunks: int = 0
     last_activity_at: str = ""
+    # Pending-grace truth (PR #1066): when the fire happened because a
+    # pending tool call exhausted the grace window, these say so instead of
+    # letting the fields above imply pure silence. Additive — older
+    # result.json dicts round-trip through format_issue_from_dict with the
+    # defaults below.
+    pending_tool_call_ids: list[str] = field(default_factory=list)
+    expired_pending_tool_call_ids: list[str] = field(default_factory=list)
+    pending_grace_sec: int | None = None
+    pending_set_last_changed_at: str | None = None
+    last_tool_update_at: str | None = None
+    n_tool_call_updates: int = 0
+    n_pending_tool_call_updates: int = 0
+    n_expired_pending_tool_call_updates: int = 0
 
     field: ClassVar[str] = "idle_timeout_info"
     category: ClassVar[str | None] = "idle_timeout"
     summary_description: ClassVar[str] = "hit idle timeout"
 
     def format_issue(self, task_name: str) -> str:
-        return (
+        line = (
             f"{task_name}: idle timeout after "
             f"{self.idle_duration_sec}s idle "
             f"({self.n_tool_calls} tool calls, "
             f"{self.wall_clock_elapsed_sec}s wall)"
         )
+        if self.pending_tool_call_ids:
+            line += (
+                f" — {len(self.pending_tool_call_ids)} pending tool call(s) "
+                f"exceeded the {self.pending_grace_sec}s pending grace"
+            )
+        return line
 
 
 @dataclass

{benchflow-0.7.6.dev2231 → benchflow-0.7.6.dev2235}/src/benchflow/evaluation.py

@@ -1084,31 +1084,60 @@ class Evaluation:
     def _get_completed_tasks(self) -> dict[str, dict]:
         """Load tasks that already have results with rewards or verifier errors.
 
+        Scoreless results whose verifier error is infra-retryable (per
+        ``RetryConfig.should_retry_verifier_error``) are not reused: the
+        verifier never scored the frozen workspace, so the task re-runs.
+
         Scoped to the current job directory (``_jobs_dir / _job_name``) to
-        prevent cross-job contamination.  When multiple result.json files
-        exist for the same task (retry artifacts), the newest by mtime wins.
+        prevent cross-job contamination. When multiple result.json files exist
+        for the same task (retry artifacts), a scored result always wins over
+        a scoreless verifier error; otherwise the newest artifact wins.
 
         Guards ENG-160: orphan retry artifacts no longer pollute resume.
         """
         job_dir = self._jobs_dir / self._job_name
         if not job_dir.exists():
             return {}
-        # Collect every result keyed by (task_name) → keep newest by mtime.
-        best: dict[str, tuple[float, dict]] = {}
+        # A completed score is durable evidence and must not be displaced by a
+        # newer scoreless retry artifact. Within the same scored/unscored tier,
+        # prefer recency and use the path as a deterministic tie-breaker.
+        best: dict[str, tuple[tuple[bool, float, str], dict]] = {}
         for rfile in job_dir.rglob("result.json"):
             try:
                 r = json.loads(rfile.read_text())
                 task = r["task_name"]
                 if r.get("rewards") is not None or r.get("verifier_error"):
-                    mtime = rfile.stat().st_mtime
+                    rank = (
+                        r.get("rewards") is not None,
+                        rfile.stat().st_mtime,
+                        str(rfile),
+                    )
                     prev = best.get(task)
-                    if prev is None or (mtime, str(rfile)) >= (prev[0], ""):
-                        best[task] = (mtime, r)
+                    if prev is None or rank >= prev[0]:
+                        best[task] = (rank, r)
             except Exception as e:
                 logger.debug(f"Skipping corrupt result file {rfile}: {e}")
         completed: dict[str, dict] = {}
-        for task, (_mt, r) in best.items():
+        # Re-running an errored task is only safe when rollouts are
+        # independent. A sequential-shared job advances one persisted learner
+        # state in task order, so replaying an earlier task after later tasks
+        # committed their skills would corrupt the learning curve; there the
+        # errored result stays reused, matching the pre-existing behavior.
+        rerun_ok = self._config.job_mode != "sequential-shared"
+        for task, (_rank, r) in best.items():
             if r.get("verifier_error"):
+                # A scoreless result whose verifier error is infra-retryable
+                # (same taxonomy as the within-run retry) records no signal
+                # about the task; reusing it pins a lost score forever.
+                retryable = self._config.retry.should_retry_verifier_error(
+                    r["verifier_error"]
+                )
+                if rerun_ok and r.get("rewards") is None and retryable:
+                    logger.info(
+                        f"Re-running verifier-errored task on resume: {task} "
+                        f"({truncate_end(r['verifier_error'], 80)})"
+                    )
+                    continue
                 logger.info(
                     f"Reusing completed verifier-errored task on resume: {task} "
                     f"({truncate_end(r['verifier_error'], 80)})"