benchflow 0.7.6.dev2226__tar.gz → 0.7.6.dev2233__tar.gz

{benchflow-0.7.6.dev2226 → benchflow-0.7.6.dev2233}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.6.dev2226
+Version: 0.7.6.dev2233
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.6.dev2226 → benchflow-0.7.6.dev2233}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.6.dev2226"
+version = "0.7.6.dev2233"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.6.dev2226 → benchflow-0.7.6.dev2233}/src/benchflow/_utils/scoring.py

@@ -186,6 +186,8 @@ def _looks_like_infra_error(error: str) -> bool:
             # Predates the marker above and is kept for strings rebuilt
             # outside that boundary (e.g. a verifier error re-raised as text).
             "failed to get session command",
+            "failed to execute session command",
+            "command timed out after",
             "sandbox not found",
             "workspace not found",
             "api connection",

{benchflow-0.7.6.dev2226 → benchflow-0.7.6.dev2233}/src/benchflow/evaluation.py

@@ -1084,31 +1084,60 @@ class Evaluation:
     def _get_completed_tasks(self) -> dict[str, dict]:
         """Load tasks that already have results with rewards or verifier errors.
 
+        Scoreless results whose verifier error is infra-retryable (per
+        ``RetryConfig.should_retry_verifier_error``) are not reused: the
+        verifier never scored the frozen workspace, so the task re-runs.
+
         Scoped to the current job directory (``_jobs_dir / _job_name``) to
-        prevent cross-job contamination.  When multiple result.json files
-        exist for the same task (retry artifacts), the newest by mtime wins.
+        prevent cross-job contamination. When multiple result.json files exist
+        for the same task (retry artifacts), a scored result always wins over
+        a scoreless verifier error; otherwise the newest artifact wins.
 
         Guards ENG-160: orphan retry artifacts no longer pollute resume.
         """
         job_dir = self._jobs_dir / self._job_name
         if not job_dir.exists():
             return {}
-        # Collect every result keyed by (task_name) → keep newest by mtime.
-        best: dict[str, tuple[float, dict]] = {}
+        # A completed score is durable evidence and must not be displaced by a
+        # newer scoreless retry artifact. Within the same scored/unscored tier,
+        # prefer recency and use the path as a deterministic tie-breaker.
+        best: dict[str, tuple[tuple[bool, float, str], dict]] = {}
         for rfile in job_dir.rglob("result.json"):
             try:
                 r = json.loads(rfile.read_text())
                 task = r["task_name"]
                 if r.get("rewards") is not None or r.get("verifier_error"):
-                    mtime = rfile.stat().st_mtime
+                    rank = (
+                        r.get("rewards") is not None,
+                        rfile.stat().st_mtime,
+                        str(rfile),
+                    )
                     prev = best.get(task)
-                    if prev is None or (mtime, str(rfile)) >= (prev[0], ""):
-                        best[task] = (mtime, r)
+                    if prev is None or rank >= prev[0]:
+                        best[task] = (rank, r)
             except Exception as e:
                 logger.debug(f"Skipping corrupt result file {rfile}: {e}")
         completed: dict[str, dict] = {}
-        for task, (_mt, r) in best.items():
+        # Re-running an errored task is only safe when rollouts are
+        # independent. A sequential-shared job advances one persisted learner
+        # state in task order, so replaying an earlier task after later tasks
+        # committed their skills would corrupt the learning curve; there the
+        # errored result stays reused, matching the pre-existing behavior.
+        rerun_ok = self._config.job_mode != "sequential-shared"
+        for task, (_rank, r) in best.items():
             if r.get("verifier_error"):
+                # A scoreless result whose verifier error is infra-retryable
+                # (same taxonomy as the within-run retry) records no signal
+                # about the task; reusing it pins a lost score forever.
+                retryable = self._config.retry.should_retry_verifier_error(
+                    r["verifier_error"]
+                )
+                if rerun_ok and r.get("rewards") is None and retryable:
+                    logger.info(
+                        f"Re-running verifier-errored task on resume: {task} "
+                        f"({truncate_end(r['verifier_error'], 80)})"
+                    )
+                    continue
                 logger.info(
                     f"Reusing completed verifier-errored task on resume: {task} "
                     f"({truncate_end(r['verifier_error'], 80)})"

{benchflow-0.7.6.dev2226 → benchflow-0.7.6.dev2233}/src/benchflow/sandbox/lockdown.py

@@ -609,7 +609,7 @@ async def _discover_pytest_plugin_flags(env, task: "Task") -> str:
         result = await env.exec(
             f"python3 -c {shlex.quote(_DISCOVER_PYTEST_PLUGINS_SCRIPT)}",
             user="root",
-            timeout_sec=15,
+            timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC,
         )
         if result.stderr:
             logger.debug(f"Plugin discovery stderr: {result.stderr.strip()}")
@@ -684,7 +684,9 @@ async def _distro_pip_env(env) -> dict[str, str]:
     """
     try:
         result = await env.exec(
-            "cat /etc/os-release 2>/dev/null || true", user="root", timeout_sec=5
+            "cat /etc/os-release 2>/dev/null || true",
+            user="root",
+            timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC,
         )
     except Exception as e:
         logger.warning("distro detection failed (%s); skipping pip env tweaks", e)
@@ -709,14 +711,16 @@ async def _trusted_verifier_path(
     checks prove they are root-owned directories and not group/world writable.
     Runtime locations and sandbox-user writable locations stay excluded.
     """
-    path_result = await env.exec("printenv PATH", user="root", timeout_sec=10)
+    path_result = await env.exec(
+        "printenv PATH", user="root", timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC
+    )
     raw_path = path_result.stdout or ""
     if not raw_path.strip():
         return _SAFE_VERIFIER_PATH
     cmd = _trusted_path_extras_cmd(
         raw_path, _blocked_verifier_path_prefixes(sandbox_user, workspace)
     )
-    result = await env.exec(cmd, user="root", timeout_sec=10)
+    result = await env.exec(cmd, user="root", timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC)
     if _exec_return_code(result) != 0:
         logger.debug(
             "Trusted verifier PATH extras unavailable; using safe PATH.%s",
@@ -748,14 +752,16 @@ async def _trusted_verifier_pythonpath(
     is chowned to root before verification.
     """
     pp_result = await env.exec(
-        "printenv PYTHONPATH 2>/dev/null || true", user="root", timeout_sec=10
+        "printenv PYTHONPATH 2>/dev/null || true",
+        user="root",
+        timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC,
     )
     raw_pp = (pp_result.stdout or "").strip()
     if not raw_pp:
         return ""
     blocked = _blocked_verifier_pythonpath_prefixes(sandbox_user)
     cmd = _trusted_path_extras_cmd(raw_pp, blocked)
-    result = await env.exec(cmd, user="root", timeout_sec=10)
+    result = await env.exec(cmd, user="root", timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC)
     try:
         extras = _json.loads(result.stdout or "[]")
     except _json.JSONDecodeError:
@@ -1052,13 +1058,14 @@ async def _kill_sandbox_user_procs(env, sandbox_user: str) -> None:
     await env.exec(
         f"pkill -u {sandbox_user} 2>/dev/null; "
         f"sleep 1; pkill -9 -u {sandbox_user} 2>/dev/null || true",
-        timeout_sec=10,
+        timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC,
     )
     # Second pass: catch any processes that slipped through (e.g. cron/at jobs).
     await env.exec(
         f"! pgrep -u {sandbox_user} > /dev/null 2>&1 || "
         f"(sleep 1 && pkill -9 -u {sandbox_user}; sleep 1)",
         user="root",
+        timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC,
     )
 
 

{benchflow-0.7.6.dev2226 → benchflow-0.7.6.dev2233}/tests/test_sandbox_hardening.py

@@ -268,6 +268,43 @@ class TestVerifierDirWipe:
                 == VERIFIER_SETUP_TIMEOUT_SEC
             )
 
+    @pytest.mark.asyncio
+    async def test_pr_1062_all_verifier_setup_probes_share_budget(self):
+        """Guards PR #1062 against short or unbounded Daytona setup execs."""
+        from benchflow.sandbox.lockdown import (
+            VERIFIER_SETUP_TIMEOUT_SEC,
+            build_reclaim_caches_cmd,
+            harden_before_verify,
+        )
+
+        def side_effect(command, **kwargs):
+            if command == "printenv PATH":
+                stdout = "/usr/local/bin:/usr/bin"
+            elif command.startswith("printenv PYTHONPATH"):
+                stdout = "/usr/local/lib/python3.12/site-packages"
+            elif command.startswith("python3 -c"):
+                stdout = "[]"
+            elif command.startswith("cat /etc/os-release"):
+                stdout = "ID=ubuntu\n"
+            else:
+                stdout = ""
+            return MagicMock(stdout=stdout, stderr="", exit_code=0)
+
+        env = _make_env(side_effect)
+        await harden_before_verify(
+            env, _make_task(), sandbox_user="agent", workspace="/app"
+        )
+
+        reclaim_command = build_reclaim_caches_cmd("/app")
+        setup_calls = [
+            call for call in env.exec.call_args_list if call.args[0] != reclaim_command
+        ]
+        assert len(setup_calls) >= 12
+        assert all(
+            call.kwargs.get("timeout_sec") == VERIFIER_SETUP_TIMEOUT_SEC
+            for call in setup_calls
+        )
+
     @pytest.mark.asyncio
     async def test_pr_942_workspace_chown_failure_is_fatal(self):
         """Guards PR #942: failed ownership freezing cannot reach verification."""

{benchflow-0.7.6.dev2226 → benchflow-0.7.6.dev2233}/tests/test_verify.py

@@ -4,6 +4,7 @@ import asyncio
 import contextlib
 import json
 import logging
+import os
 from datetime import datetime
 from unittest.mock import AsyncMock, MagicMock, patch
 
@@ -60,6 +61,20 @@ def test_classify_verifier_error_substring_order():
     assert classify_verifier_error(msg) == VERIFIER_FAILED
 
 
+def test_pr_1063_daytona_verifier_exec_errors_are_infra():
+    """Guards PR #1063: historical Daytona exec failures must be retryable."""
+    assert (
+        classify_verifier_error(
+            "verifier crashed: Failed to execute session command: (no detail)"
+        )
+        == VERIFIER_INFRA
+    )
+    assert (
+        classify_verifier_error("verifier crashed: Command timed out after 180 seconds")
+        == VERIFIER_INFRA
+    )
+
+
 # RunResult with verifier_error
 
 
@@ -489,11 +504,75 @@ class TestRetry:
 
 
 class TestResume:
-    def test_verifier_errored_is_complete(self, tmp_path, caplog):
-        """Guards the PR #819 fix for issue #542's misleading resume log."""
+    def test_infra_verifier_errored_reruns(self, tmp_path, caplog):
+        """Guards PR #1063: retryable verifier infra must not pin a lost score."""
         task_dir = tmp_path / "task1" / "trial-1"
         task_dir.mkdir(parents=True)
         (task_dir / "result.json").write_text(
+            json.dumps(
+                {
+                    "task_name": "task1",
+                    "rewards": None,
+                    "error": None,
+                    "verifier_error": (
+                        "verifier crashed: Failed to execute session command: "
+                        "(no detail)"
+                    ),
+                }
+            )
+        )
+        from benchflow.evaluation import Evaluation, EvaluationConfig
+
+        job = Evaluation(
+            tasks_dir=tmp_path, jobs_dir=tmp_path, config=EvaluationConfig()
+        )
+        with caplog.at_level(logging.INFO):
+            completed = job._get_completed_tasks()
+        assert "task1" not in completed
+        assert any("Re-running verifier-errored task" in m for m in caplog.messages)
+
+    def test_sequential_shared_reuses_infra_verifier_error(self, tmp_path):
+        """Guards PR #1063: sequential resume must preserve its learning curve."""
+        task_dir = tmp_path / "task1" / "trial-1"
+        task_dir.mkdir(parents=True)
+        (task_dir / "result.json").write_text(
+            json.dumps(
+                {
+                    "task_name": "task1",
+                    "rewards": None,
+                    "error": None,
+                    "verifier_error": "verifier timed out after 900s",
+                }
+            )
+        )
+        from benchflow.evaluation import Evaluation, EvaluationConfig
+
+        job = Evaluation(
+            tasks_dir=tmp_path,
+            jobs_dir=tmp_path,
+            config=EvaluationConfig(job_mode="sequential-shared"),
+        )
+        assert "task1" in job._get_completed_tasks()
+
+    def test_pr_1063_older_scored_result_beats_newer_infra_error(self, tmp_path):
+        """Guards PR #1063: a later failed retry cannot erase a valid score."""
+        scored_dir = tmp_path / "task1" / "trial-1"
+        retry_dir = tmp_path / "task1" / "trial-1-retry-1"
+        scored_dir.mkdir(parents=True)
+        retry_dir.mkdir(parents=True)
+        scored_path = scored_dir / "result.json"
+        retry_path = retry_dir / "result.json"
+        scored_path.write_text(
+            json.dumps(
+                {
+                    "task_name": "task1",
+                    "rewards": {"reward": 1.0},
+                    "error": None,
+                    "verifier_error": None,
+                }
+            )
+        )
+        retry_path.write_text(
             json.dumps(
                 {
                     "task_name": "task1",
@@ -503,6 +582,31 @@ class TestResume:
                 }
             )
         )
+        os.utime(scored_path, (1, 1))
+        os.utime(retry_path, (2, 2))
+
+        from benchflow.evaluation import Evaluation, EvaluationConfig
+
+        job = Evaluation(
+            tasks_dir=tmp_path, jobs_dir=tmp_path, config=EvaluationConfig()
+        )
+        completed = job._get_completed_tasks()
+        assert completed["task1"]["rewards"] == {"reward": 1.0}
+
+    def test_contract_verifier_errored_is_complete(self, tmp_path, caplog):
+        """Guards the PR #819 fix for issue #542's misleading resume log."""
+        task_dir = tmp_path / "task1" / "trial-1"
+        task_dir.mkdir(parents=True)
+        (task_dir / "result.json").write_text(
+            json.dumps(
+                {
+                    "task_name": "task1",
+                    "rewards": None,
+                    "error": None,
+                    "verifier_error": "verifier crashed: No reward file found",
+                }
+            )
+        )
         from benchflow.evaluation import Evaluation, EvaluationConfig
 
         job = Evaluation(