benchflow 0.7.6.dev2058__tar.gz → 0.7.6.dev2072__tar.gz

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2072}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.6.dev2058
+Version: 0.7.6.dev2072
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2072}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.6.dev2058"
+version = "0.7.6.dev2072"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2072}/src/benchflow/sandbox/setup.py

@@ -568,6 +568,37 @@ def _inject_skills_into_dockerfile(
     )
 
 
+def _decode_mountinfo_path(value: str) -> str:
+    """Decode the octal escapes used by ``/proc/self/mountinfo``."""
+    return (
+        value.replace("\\040", " ")
+        .replace("\\011", "\t")
+        .replace("\\012", "\n")
+        .replace("\\134", "\\")
+    )
+
+
+def _mountinfo_path_mapping(cwd: str, mountinfo: str) -> tuple[str, str] | None:
+    """Return host-source/container-destination for a containing bind mount."""
+    current = Path(cwd)
+    best: tuple[str, str] | None = None
+    for line in mountinfo.splitlines():
+        fields = line.split(" - ", 1)[0].split()
+        if len(fields) < 5:
+            continue
+        root = _decode_mountinfo_path(fields[3])
+        dest = _decode_mountinfo_path(fields[4])
+        try:
+            current.relative_to(dest)
+        except ValueError:
+            continue
+        if root == "/" or dest == "/":
+            continue
+        if best is None or len(dest) > len(best[1]):
+            best = (root, dest)
+    return best
+
+
 def _detect_dind_mount() -> tuple[str, str] | None:
     """Detect Docker-in-Docker host path translation.
 
@@ -576,30 +607,33 @@ def _detect_dind_mount() -> tuple[str, str] | None:
 
     Returns (host_source, container_dest) tuple, or None if not in DinD.
     """
-    if not Path("/.dockerenv").exists():
-        return None
     import subprocess as _sp
 
     try:
-        hostname = _sp.check_output(["hostname"], text=True).strip()
-        result = _sp.run(
-            ["docker", "inspect", hostname],
-            capture_output=True,
-            text=True,
-            timeout=5,
-        )
-        if result.returncode != 0:
-            return None
-        data = json.loads(result.stdout)
         cwd = str(Path.cwd())
-        best = None
-        for mount in data[0].get("Mounts", []):
-            if mount.get("Type") != "bind":
-                continue
-            dest = mount.get("Destination", "")
-            if cwd.startswith(dest) and (best is None or len(dest) > len(best[1])):
-                best = (mount["Source"], dest)
-        return best
+        if Path("/.dockerenv").exists():
+            hostname = _sp.check_output(["hostname"], text=True).strip()
+            result = _sp.run(
+                ["docker", "inspect", hostname],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            if result.returncode == 0:
+                data = json.loads(result.stdout)
+                best = None
+                for mount in data[0].get("Mounts", []):
+                    if mount.get("Type") != "bind":
+                        continue
+                    dest = mount.get("Destination", "")
+                    if cwd.startswith(dest) and (
+                        best is None or len(dest) > len(best[1])
+                    ):
+                        best = (mount["Source"], dest)
+                if best is not None:
+                    return best
+        mountinfo = Path("/proc/self/mountinfo").read_text(errors="replace")
+        return _mountinfo_path_mapping(cwd, mountinfo)
     except Exception:
         logger.debug("DinD mount detection failed", exc_info=True)
         return None

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2072}/src/benchflow/trajectories/results.py

@@ -30,6 +30,7 @@ from benchflow.trajectories.export_prime_sft import (
     validate_prime_sft_row,
 )
 from benchflow.trajectories.types import redact_trajectory_obj
+from benchflow.usage_tracking import USAGE_SOURCE_AGENT_NATIVE_ACP
 
 ROLLOUT_RESULTS_FILENAME = "results.jsonl"
 JOB_RESULTS_FILENAME = "results.jsonl"
@@ -465,6 +466,12 @@ def build_rollout_results_record(
         export_error=effective_export_error,
     )
     terminal_health_error = bool(error or verifier_error or partial_trajectory)
+    native_subscription_without_llm = bool(
+        (agent_result or {}).get("usage_source") == USAGE_SOURCE_AGENT_NATIVE_ACP
+        and not (rollout_path / "trajectory" / "llm_trajectory.jsonl").exists()
+        and effective_export_error is None
+        and not terminal_health_error
+    )
     training_ready = bool(
         steps
         and completion
@@ -493,7 +500,7 @@ def build_rollout_results_record(
             training_ready_reason = "verifier_error"
         else:
             training_ready_reason = "missing_healthy_structured_llm_trajectory"
-        if error_obj is None:
+        if error_obj is None and not native_subscription_without_llm:
             error_name = (
                 training_ready_reason
                 if training_ready_reason

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2072}/tests/test_benchflow_experiment_review_validator.py

@@ -133,6 +133,71 @@ def test_validator_accepts_training_ready_results_jsonl(tmp_path: Path) -> None:
     }
 
 
+def test_validator_oracle_mode_accepts_reward_only_rollout(tmp_path: Path) -> None:
+    """Guards BenchFlow PR #1049's explicit oracle artifact exception."""
+    validator = _load_validator()
+    rollout = _rollout(tmp_path)
+    result_path = rollout / "result.json"
+    result = json.loads(result_path.read_text())
+    result.update(
+        {
+            "agent": "oracle",
+            "agent_name": "oracle",
+            "model": None,
+            "n_tool_calls": 0,
+            "agent_result": {
+                "n_tool_calls": 0,
+                "n_input_tokens": 0,
+                "n_output_tokens": 0,
+                "total_tokens": 0,
+                "usage_source": "unavailable",
+            },
+        }
+    )
+    result_path.write_text(json.dumps(result))
+    (rollout / "trajectory" / "llm_trajectory.jsonl").unlink()
+    _write_jsonl(
+        rollout / "results.jsonl",
+        [
+            {
+                "example_id": 0,
+                "prompt": [{"role": "user", "content": "solve"}],
+                "completion": None,
+                "info": {
+                    "task_id": "task-a",
+                    "training_ready": False,
+                    "training_ready_reason": (
+                        "missing_healthy_structured_llm_trajectory"
+                    ),
+                },
+                "reward": 1.0,
+                "error": {
+                    "error": "missing_llm_trajectory",
+                    "error_chain_str": "oracle has no LLM trajectory",
+                },
+                "is_completed": False,
+                "is_truncated": False,
+                "stop_condition": "agent_completed",
+                "metrics": {"n_tool_calls": 0, "reward": 1.0},
+                "tool_defs": [],
+                "token_usage": {
+                    "final_input_tokens": 0,
+                    "final_output_tokens": 0,
+                    "total_tokens": 0,
+                },
+                "trajectory": [],
+            }
+        ],
+    )
+
+    strict = validator.validate_rollout(rollout)
+    oracle = validator.validate_rollout(rollout, allow_oracle_without_llm=True)
+
+    assert strict["healthy"] is False
+    assert oracle["healthy"] is True
+    assert oracle["issues"] == []
+
+
 def test_validator_accepts_provider_total_only_token_usage(tmp_path: Path) -> None:
     """Some providers expose only total tokens; Prime-RL can still render the row."""
     validator = _load_validator()
@@ -240,6 +305,92 @@ def test_validator_rejects_unready_results_for_healthy_rollout(tmp_path: Path) -
     assert any("training_ready=false" in issue for issue in report["issues"])
 
 
+def _native_subscription_rollout(tmp_path: Path) -> Path:
+    rollout = _rollout(tmp_path)
+    (rollout / "trajectory" / "llm_trajectory.jsonl").unlink()
+    result = json.loads((rollout / "result.json").read_text())
+    result["agent_result"]["usage_source"] = "agent_native_acp"
+    (rollout / "result.json").write_text(json.dumps(result))
+    row = json.loads((rollout / "results.jsonl").read_text())
+    row.update(
+        {
+            "completion": None,
+            "error": None,
+            "is_completed": True,
+            "trajectory": [],
+        }
+    )
+    row["info"].update(
+        {
+            "training_ready": False,
+            "training_ready_reason": "missing_healthy_structured_llm_trajectory",
+        }
+    )
+    _write_jsonl(rollout / "results.jsonl", [row])
+    return rollout
+
+
+def test_validator_accepts_explicit_native_subscription_mode(tmp_path: Path) -> None:
+    """Guards this PR's explicit native-subscription evidence mode."""
+    validator = _load_validator()
+    rollout = _native_subscription_rollout(tmp_path)
+
+    strict = validator.validate_rollout(rollout)
+    native = validator.validate_rollout(
+        rollout, allow_native_subscription_without_llm=True
+    )
+
+    assert strict["healthy"] is False
+    assert native["healthy"] is True
+    assert any("native subscription rollout" in item for item in native["warnings"])
+
+
+def test_validator_native_mode_rejects_wrong_usage_source(tmp_path: Path) -> None:
+    """Guards this PR against widening subscription mode to other runs."""
+    validator = _load_validator()
+    rollout = _native_subscription_rollout(tmp_path)
+    result = json.loads((rollout / "result.json").read_text())
+    result["agent_result"]["usage_source"] = "provider_response"
+    (rollout / "result.json").write_text(json.dumps(result))
+
+    report = validator.validate_rollout(
+        rollout, allow_native_subscription_without_llm=True
+    )
+
+    assert report["healthy"] is False
+    assert any("missing required artifact" in issue for issue in report["issues"])
+
+
+def test_validator_native_mode_requires_token_components(tmp_path: Path) -> None:
+    """Guards this PR against accepting usage-free subscription evidence."""
+    validator = _load_validator()
+    rollout = _native_subscription_rollout(tmp_path)
+    row = json.loads((rollout / "results.jsonl").read_text())
+    row["token_usage"] = {"total_tokens": 17}
+    _write_jsonl(rollout / "results.jsonl", [row])
+
+    report = validator.validate_rollout(
+        rollout, allow_native_subscription_without_llm=True
+    )
+
+    assert report["healthy"] is False
+    assert any("positive token components" in issue for issue in report["issues"])
+
+
+def test_validator_native_mode_rejects_malformed_present_llm(tmp_path: Path) -> None:
+    """Guards this PR against bypassing malformed provider evidence."""
+    validator = _load_validator()
+    rollout = _native_subscription_rollout(tmp_path)
+    (rollout / "trajectory" / "llm_trajectory.jsonl").write_text("{\n")
+
+    report = validator.validate_rollout(
+        rollout, allow_native_subscription_without_llm=True
+    )
+
+    assert report["healthy"] is False
+    assert any("JSON parse error" in issue for issue in report["issues"])
+
+
 def test_validator_rejects_results_with_dropped_successful_exchange(
     tmp_path: Path,
 ) -> None:

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2072}/tests/test_rollout_import_no_side_effects.py

@@ -135,3 +135,60 @@ def test_without_dind_environment_patch_does_not_apply() -> None:
         f"subprocess failed: stdout={result.stdout!r} stderr={result.stderr!r}"
     )
     assert "OK" in result.stdout
+
+
+def test_mountinfo_fallback_maps_workspace_subtree() -> None:
+    """Guards the fix from PR #1049 for host-Docker bind-mounted shells."""
+    from benchflow.sandbox.setup import _mountinfo_path_mapping
+
+    mountinfo = (
+        "1001 991 259:5 /workspace/docker /workspace rw,noatime "
+        "- ext4 /dev/nvme1n1p3 rw\n"
+    )
+
+    assert _mountinfo_path_mapping("/workspace/repo", mountinfo) == (
+        "/workspace/docker",
+        "/workspace",
+    )
+
+
+def test_dind_patch_translates_context_and_output_paths() -> None:
+    """Guards the fix from PR #1049 for every host-side Compose path."""
+    result = _run_python(
+        """
+        from unittest.mock import patch
+
+        from benchflow.sandbox.docker import DockerSandboxEnvVars
+        from benchflow.sandbox import setup as sandbox_setup
+        from benchflow.rollout import _install_docker_compat
+
+        with patch.object(
+            sandbox_setup,
+            "_detect_dind_mount",
+            return_value=("/host/work", "/workspace"),
+        ):
+            _install_docker_compat()
+
+        env = DockerSandboxEnvVars(
+            main_image_name="main",
+            context_dir="/workspace/repo/environment",
+            host_verifier_logs_path="/workspace/jobs/verifier",
+            host_agent_logs_path="/workspace/jobs/agent",
+            host_artifacts_path="/workspace/jobs/artifacts",
+            env_verifier_logs_path="/logs/verifier",
+            env_agent_logs_path="/logs/agent",
+            env_artifacts_path="/artifacts",
+        ).to_env_dict(include_os_env=False)
+
+        assert env["CONTEXT_DIR"] == "/workspace/repo/environment"
+        assert env["HOST_VERIFIER_LOGS_PATH"] == "/host/work/jobs/verifier"
+        assert env["HOST_AGENT_LOGS_PATH"] == "/host/work/jobs/agent"
+        assert env["HOST_ARTIFACTS_PATH"] == "/host/work/jobs/artifacts"
+        assert env["ENV_VERIFIER_LOGS_PATH"] == "/logs/verifier"
+        print("OK")
+        """
+    )
+    assert result.returncode == 0, (
+        f"subprocess failed: stdout={result.stdout!r} stderr={result.stderr!r}"
+    )
+    assert "OK" in result.stdout

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2072}/tests/test_train_mode_artifact_emission.py

@@ -1101,6 +1101,83 @@ def test_build_rollout_result_results_jsonl_is_unhealthy_without_llm(tmp_path):
     assert row["stop_condition"] == "partial_trajectory"
 
 
+def test_native_subscription_without_llm_is_completed_but_not_training_ready(
+    tmp_path,
+):
+    """Guards this PR's native-subscription result semantics."""
+    rollout_dir = tmp_path / "native-subscription"
+    rollout_dir.mkdir()
+
+    _build_rollout_result(
+        rollout_dir,
+        task_name="subscription-task",
+        rollout_name="r1",
+        agent="claude-agent-acp",
+        agent_name="Claude Agent",
+        model="claude-opus-5",
+        n_tool_calls=1,
+        prompts=["Solve this."],
+        error=None,
+        verifier_error=None,
+        trajectory=_acp_trajectory(),
+        partial_trajectory=False,
+        trajectory_source="acp",
+        rewards={"reward": 1.0},
+        started_at=datetime.now(),
+        timing={"agent_execution": 1.0, "verifier": 1.0},
+        n_input_tokens=11,
+        n_output_tokens=6,
+        total_tokens=17,
+        usage_source="agent_native_acp",
+    )
+
+    row = json.loads((rollout_dir / "results.jsonl").read_text())
+    assert row["completion"] is None
+    assert row["trajectory"] == []
+    assert row["info"]["training_ready"] is False
+    assert (
+        row["info"]["training_ready_reason"]
+        == "missing_healthy_structured_llm_trajectory"
+    )
+    assert row["error"] is None
+    assert row["is_completed"] is True
+    assert row["is_truncated"] is False
+    assert row["stop_condition"] == "agent_completed"
+
+
+def test_native_subscription_without_llm_preserves_terminal_errors(tmp_path):
+    """Guards this PR against hiding native-subscription verifier failures."""
+    rollout_dir = tmp_path / "native-subscription-error"
+    rollout_dir.mkdir()
+
+    _build_rollout_result(
+        rollout_dir,
+        task_name="subscription-task",
+        rollout_name="r1",
+        agent="claude-agent-acp",
+        agent_name="Claude Agent",
+        model="claude-opus-5",
+        n_tool_calls=1,
+        prompts=["Solve this."],
+        error=None,
+        verifier_error="verifier crashed",
+        trajectory=_acp_trajectory(),
+        partial_trajectory=False,
+        trajectory_source="acp",
+        rewards=None,
+        started_at=datetime.now(),
+        timing={"agent_execution": 1.0},
+        n_input_tokens=11,
+        n_output_tokens=6,
+        total_tokens=17,
+        usage_source="agent_native_acp",
+    )
+
+    row = json.loads((rollout_dir / "results.jsonl").read_text())
+    assert row["error"]["error"] == "verifier_error"
+    assert row["is_completed"] is False
+
+
 def test_results_jsonl_token_usage_falls_back_to_provider_total(tmp_path):
     """Guards PR #828: total-only telemetry still feeds Prime-RL token batches."""
     rollout_dir = tmp_path / "rollout-total-only"