benchflow 0.6.7.dev1819__tar.gz → 0.6.7.dev1822__tar.gz

{benchflow-0.6.7.dev1819 → benchflow-0.6.7.dev1822}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1819
+Version: 0.6.7.dev1822
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1819 → benchflow-0.6.7.dev1822}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1819"
+version = "0.6.7.dev1822"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1819 → benchflow-0.6.7.dev1822}/src/benchflow/acp/session.py

@@ -1,6 +1,8 @@
 """ACP session lifecycle management."""
 
 import logging
+import os
+import time
 from collections.abc import Callable
 from datetime import datetime
 from typing import Any
@@ -16,6 +18,27 @@ from .types import (
 
 logger = logging.getLogger(__name__)
 
+# Console progress heartbeat contract. ``BENCHFLOW_PROGRESS`` is the operator
+# switch (on/off); unset defers to ``BENCHFLOW_PROGRESS_AUTO``, which the
+# Evaluation layer sets to "0" for multi-concurrency jobs where interleaved
+# per-task lines would be noise. ``bench eval run --quiet`` sets the explicit
+# off value.
+_PROGRESS_ENV = "BENCHFLOW_PROGRESS"
+_PROGRESS_AUTO_ENV = "BENCHFLOW_PROGRESS_AUTO"
+_PROGRESS_INTERVAL_SEC = 45.0
+_PROGRESS_OFF_VALUES = frozenset({"off", "0", "false", "none"})
+_PROGRESS_ON_VALUES = frozenset({"on", "1", "true"})
+
+
+def _console_progress_enabled() -> bool:
+    raw = os.environ.get(_PROGRESS_ENV, "").strip().lower()
+    if raw in _PROGRESS_OFF_VALUES:
+        return False
+    if raw in _PROGRESS_ON_VALUES:
+        return True
+    return os.environ.get(_PROGRESS_AUTO_ENV, "1").strip() != "0"
+
+
 ACPUsageSnapshot = dict[str, int | None]
 
 _ACP_USAGE_FIELDS: tuple[str, ...] = (
@@ -177,8 +200,17 @@ class ACPSession:
         # Optional sink invoked after every public state mutation so callers
         # can stream a trajectory snapshot to disk without polling.
         self.on_change: Callable[[ACPSession], None] | None = None
+        # Console progress heartbeat. Without it a first-run user stares at
+        # total silence between "Prompt 1/1: ..." and the verifier — observed
+        # 18 minutes on a passing rollout (fresh-user dogfood 2026-08-09) with
+        # no way to distinguish "working" from "hung". The throttle keys off
+        # _notify_change, which fires on every streamed update.
+        self._progress_enabled = _console_progress_enabled()
+        self._prompt_started_at: float | None = None
+        self._last_progress_at = 0.0
 
     def _notify_change(self) -> None:
+        self._maybe_log_progress()
         if self.on_change is None:
             return
         try:
@@ -189,15 +221,36 @@ class ACPSession:
             # which is otherwise easy to miss in a 64-concurrency log.
             logger.error(f"ACPSession on_change callback failed: {e}")
 
+    def _maybe_log_progress(self) -> None:
+        if not self._progress_enabled or self._prompt_started_at is None:
+            return
+        now = time.monotonic()
+        if now - self._last_progress_at < _PROGRESS_INTERVAL_SEC:
+            return
+        self._last_progress_at = now
+        elapsed_min = (now - self._prompt_started_at) / 60.0
+        line = f"  … {elapsed_min:.1f}min, {len(self.tool_calls)} tool calls"
+        if self.tool_calls:
+            last = self.tool_calls[-1]
+            title = (last.title or last.kind or "").strip().splitlines()[0][:60]
+            if title:
+                line += f" (last: {title})"
+        logger.info(line)
+
     def record_user_prompt(self, text: str) -> None:
         """Record a user prompt. Call before sending each ACP prompt."""
         self._events_active = True
+        self._prompt_started_at = time.monotonic()
+        # Grace period: the first heartbeat waits a full interval so short
+        # prompts stay single-line.
+        self._last_progress_at = time.monotonic()
         self._flush_agent_text()
         self.events.append({"type": "user_message", "text": text})
         self._notify_change()
 
     def mark_prompt_end(self) -> None:
         """Flush pending agent text after a prompt completes."""
+        self._prompt_started_at = None
         self._flush_agent_text()
         self._notify_change()
 

{benchflow-0.6.7.dev1819 → benchflow-0.6.7.dev1822}/src/benchflow/cli/main.py

@@ -17,6 +17,7 @@ compatibility.
 import asyncio
 import json
 import logging
+import os
 from contextlib import AbstractContextManager, nullcontext
 from pathlib import Path
 from typing import TYPE_CHECKING, Annotated
@@ -383,6 +384,16 @@ def eval_run(
             ),
         ),
     ] = None,
+    quiet: Annotated[
+        bool,
+        typer.Option(
+            "--quiet",
+            help=(
+                "Suppress the per-run console progress heartbeat (the "
+                "'… Nmin, K tool calls' lines during agent execution)."
+            ),
+        ),
+    ] = False,
     jobs_dir: Annotated[
         str | None,
         typer.Option("--jobs-dir", help="Output directory"),
@@ -582,6 +593,10 @@ def eval_run(
     # a backend was added, so it no longer repeats them.
     """Run an evaluation — single task or batch."""
     _apply_dotenv_to_process_env()
+    if quiet:
+        # Explicit off beats the session layer's auto-gate; see
+        # benchflow.acp.session._console_progress_enabled.
+        os.environ["BENCHFLOW_PROGRESS"] = "off"
 
     request = EvalCreateRequest(
         config_file=config_file,

{benchflow-0.6.7.dev1819 → benchflow-0.6.7.dev1822}/src/benchflow/evaluation.py

@@ -1369,6 +1369,11 @@ class Evaluation:
     ) -> list[tuple[str, RunResult]]:
         """The default schedule — rollouts run concurrently and isolated."""
         cfg = self._config
+        # Console heartbeat auto-gate: interleaved per-task progress lines are
+        # noise at high concurrency, so the sessions' heartbeat defaults off
+        # for multi-concurrency jobs. An explicit BENCHFLOW_PROGRESS=on/off
+        # from the operator always wins (checked first in the session layer).
+        os.environ["BENCHFLOW_PROGRESS_AUTO"] = "1" if cfg.concurrency <= 1 else "0"
         # Floor at 1: Semaphore(0) deadlocks on first acquire. eval-create already
         # rejects <1 at plan time, but this guards every other caller (skills eval,
         # SDK) against a silent forever-hang on a bad concurrency.

{benchflow-0.6.7.dev1819 → benchflow-0.6.7.dev1822}/src/benchflow/sandbox/daytona.py

@@ -179,6 +179,12 @@ def _load_daytona_sdk() -> None:
     Resources = _daytona.Resources
     SessionExecuteRequest = _daytona.SessionExecuteRequest
     SnapshotState = _snapshot.SnapshotState
+    # The SDK's PTY transport rides socket.io; engineio's write loop logs
+    # "packet queue is empty, aborting" at ERROR on a benign disconnect race
+    # every time a PTY closes normally (fresh-user dogfood 2026-08-09: two such
+    # lines mid-run on a passing rollout). It is not actionable and reads as a
+    # failure — silence the logger for benchflow processes.
+    logging.getLogger("engineio.client").setLevel(logging.CRITICAL)
     _DAYTONA_SDK_LOADED = True
 
 
@@ -308,10 +314,17 @@ class DaytonaClientManager:
             return self._client
 
     def _cleanup_sync(self) -> None:
+        # Runs as an atexit callback after a (typically successful) run.
+        # ``CancelledError`` is a BaseException, so a bare ``except Exception``
+        # let it escape into a 40-line "Exception ignored in atexit callback"
+        # traceback printed AFTER the final score line (fresh-user dogfood
+        # 2026-08-09) — closing the SDK client tears down its aiohttp/websocket
+        # readers, which surface cancellation on loop shutdown. Nothing here is
+        # actionable at exit: log at debug, never print.
         try:
             asyncio.run(self._cleanup())
-        except Exception as e:
-            print(f"Error during Daytona client cleanup: {e}")
+        except (Exception, asyncio.CancelledError) as e:
+            self._logger.debug(f"Daytona client cleanup swallowed at exit: {e!r}")
 
     async def _cleanup(self) -> None:
         async with self._client_lock:
@@ -319,8 +332,8 @@ class DaytonaClientManager:
                 try:
                     self._logger.debug("Closing AsyncDaytona client at program exit")
                     await self._client.close()
-                except Exception as e:
-                    self._logger.error(f"Error closing AsyncDaytona client: {e}")
+                except (Exception, asyncio.CancelledError) as e:
+                    self._logger.debug(f"Error closing AsyncDaytona client: {e!r}")
                 finally:
                     self._client = None
 

{benchflow-0.6.7.dev1819 → benchflow-0.6.7.dev1822}/src/benchflow/task/config.py

@@ -117,6 +117,16 @@ class PackageInfo(TaskConfigModel):
         ...,
         description="Package name in org/name format (e.g., 'benchflow/hello-world')",
     )
+    version: str = Field(
+        default="",
+        description=(
+            "Package version (Harbor schema 1.3 [task] field). Informational "
+            "only — recorded, never interpreted. Absent from this model it was "
+            "extra_forbidden, which made every Harbor-1.3 task authored by the "
+            "govbench/frontier-bench curation pipeline unloadable (2026-08-09: "
+            "all 6 tasks in the local corpus failed on exactly this field)."
+        ),
+    )
     description: str = Field(
         default="",
         description="Human-readable description of the task",

benchflow-0.6.7.dev1822/tests/test_console_progress.py

@@ -0,0 +1,92 @@
+"""Console progress heartbeat during agent execution.
+
+Fresh-user dogfood (2026-08-09): between "Prompt 1/1: ..." and the verifier
+there was NO console output for 18 minutes on a passing rollout — the single
+biggest first-run confidence gap. The ACP session now emits a throttled
+"  … Nmin, K tool calls" line from the same update stream that feeds the
+trajectory writer; multi-concurrency jobs auto-gate it off, and explicit
+BENCHFLOW_PROGRESS / --quiet always win.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+
+from benchflow.acp import session as session_mod
+from benchflow.acp.session import ACPSession, _console_progress_enabled
+
+
+@pytest.mark.parametrize(
+    ("progress", "auto", "expected"),
+    [
+        ("", "", True),  # bare default: on
+        ("", "0", False),  # auto-gate (multi-concurrency) off
+        ("on", "0", True),  # explicit on beats the auto-gate
+        ("off", "1", False),  # explicit off beats everything
+        ("false", "", False),
+        ("1", "0", True),
+    ],
+)
+def test_progress_enablement_contract(monkeypatch, progress, auto, expected):
+    if progress:
+        monkeypatch.setenv("BENCHFLOW_PROGRESS", progress)
+    else:
+        monkeypatch.delenv("BENCHFLOW_PROGRESS", raising=False)
+    if auto:
+        monkeypatch.setenv("BENCHFLOW_PROGRESS_AUTO", auto)
+    else:
+        monkeypatch.delenv("BENCHFLOW_PROGRESS_AUTO", raising=False)
+    assert _console_progress_enabled() is expected
+
+
+def _drive_tool_call(session: ACPSession, call_id: str) -> None:
+    session.handle_update(
+        {
+            "sessionUpdate": "tool_call",
+            "toolCallId": call_id,
+            "title": "IPython cell",
+            "kind": "execute",
+        }
+    )
+
+
+def test_heartbeat_emits_throttled_line_during_prompt(monkeypatch, caplog):
+    monkeypatch.setenv("BENCHFLOW_PROGRESS", "on")
+    monkeypatch.setattr(session_mod, "_PROGRESS_INTERVAL_SEC", 0.0)
+    s = ACPSession("sid")
+    s.record_user_prompt("do the task")
+    with caplog.at_level(logging.INFO, logger=session_mod.__name__):
+        _drive_tool_call(s, "tc1")
+    lines = [r.message for r in caplog.records if "tool calls" in r.message]
+    assert lines, "no heartbeat line emitted"
+    assert "1 tool calls" in lines[-1]
+    assert "IPython cell" in lines[-1]
+
+
+def test_heartbeat_silent_when_disabled(monkeypatch, caplog):
+    monkeypatch.setenv("BENCHFLOW_PROGRESS", "off")
+    monkeypatch.setattr(session_mod, "_PROGRESS_INTERVAL_SEC", 0.0)
+    s = ACPSession("sid")
+    s.record_user_prompt("do the task")
+    with caplog.at_level(logging.INFO, logger=session_mod.__name__):
+        _drive_tool_call(s, "tc1")
+    assert not [r for r in caplog.records if "tool calls" in r.message]
+
+
+def test_heartbeat_silent_outside_prompt(monkeypatch, caplog):
+    """No heartbeat before the first prompt or after mark_prompt_end."""
+    monkeypatch.setenv("BENCHFLOW_PROGRESS", "on")
+    monkeypatch.setattr(session_mod, "_PROGRESS_INTERVAL_SEC", 0.0)
+    s = ACPSession("sid")
+    with caplog.at_level(logging.INFO, logger=session_mod.__name__):
+        _drive_tool_call(s, "tc0")  # pre-prompt: no active prompt yet
+    assert not [r for r in caplog.records if "tool calls" in r.message]
+
+    s.record_user_prompt("go")
+    s.mark_prompt_end()
+    caplog.clear()
+    with caplog.at_level(logging.INFO, logger=session_mod.__name__):
+        _drive_tool_call(s, "tc1")  # post-prompt: heartbeat window closed
+    assert not [r for r in caplog.records if "tool calls" in r.message]

{benchflow-0.6.7.dev1819 → benchflow-0.6.7.dev1822}/tests/test_task_config.py

@@ -89,6 +89,7 @@ artifacts = [{ source = "/logs/artifacts", destination = "artifacts" }]
 
 [task]
 name = "benchflow/harbor-parity"
+version = "1.0.0"
 description = "Exercises Harbor-compatible task.toml fields"
 authors = [{ name = "BenchFlow", email = "benchflow@example.com" }]
 keywords = ["parity", "task-md"]
@@ -167,6 +168,9 @@ env = { STEP = "scaffold" }
     assert cfg.schema_version == "1.3"
     assert cfg.task is not None
     assert cfg.task.name == "benchflow/harbor-parity"
+    # Harbor 1.3 [task] version — informational, recorded verbatim. Its absence
+    # made every govbench/frontier-bench curated task unloadable (2026-08-09).
+    assert cfg.task.version == "1.0.0"
     assert cfg.metadata["custom"]["kept"] is True
     assert cfg.agent.network_mode == NetworkMode.ALLOWLIST
     assert cfg.agent.allowed_hosts == ["api.example.com"]