benchflow 0.5.3.dev908__tar.gz → 0.5.3.dev956__tar.gz

{benchflow-0.5.3.dev908 → benchflow-0.5.3.dev956}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.5.3.dev908
+Version: 0.5.3.dev956
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.5.3.dev908 → benchflow-0.5.3.dev956}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.5.3.dev908"
+version = "0.5.3.dev956"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.5.3.dev908 → benchflow-0.5.3.dev956}/src/benchflow/providers/litellm_bedrock_patch.py

@@ -70,6 +70,11 @@ def _patch_bedrock_effort() -> None:
                 reasoning_effort = override
         return original(self, model, reasoning_effort, optional_params)
 
+    # Marker for the fail-closed startup preflight (#602): lets the runtime
+    # verify this override is installed without importing this module (which
+    # would itself apply the patches and mask a sitecustomize load failure).
+    setattr(handle, "__benchflow_bedrock_patch__", True)  # noqa: B010
+
     setattr(  # noqa: B010 - avoids static type narrowing on monkey-patched vendor API
         AmazonConverseConfig,
         "_handle_reasoning_effort_parameter",

benchflow-0.5.3.dev956/src/benchflow/providers/litellm_bedrock_preflight.py

@@ -0,0 +1,181 @@
+"""Fail-closed checks for BenchFlow's Bedrock Claude 4.8+ LiteLLM patch."""
+
+from __future__ import annotations
+
+import shlex
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+from benchflow.providers.litellm_config import LiteLLMRoute
+
+
+class BedrockPatchPreflightError(RuntimeError):
+    """Raised when the Bedrock patch cannot be proven active."""
+
+
+# Behavioral preflight for the Bedrock Claude 4.8+ adaptive-thinking patch
+# (#602). Runs in a fresh interpreter with the same env/PYTHONPATH as the proxy,
+# so site loads the runtime dir's sitecustomize exactly like the proxy process
+# does. It deliberately imports only litellm internals, never the patch module
+# itself, which would apply the patches in-process and mask a load failure.
+BEDROCK_PATCH_PREFLIGHT_SOURCE = """\
+import sys
+
+# Probe with a versioned Bedrock inference-profile ID: stock litellm 1.88.0rc1
+# resolves the bare alias through its cost map, so only the versioned form
+# discriminates patched from stock (#602).
+PROBE = "us.anthropic.claude-opus-4-8-20251101-v1:0"
+
+failures = []
+try:
+    from litellm.llms.anthropic.chat.transformation import AnthropicConfig
+
+    if not AnthropicConfig._is_adaptive_thinking_model(PROBE):
+        failures.append(
+            "adaptive-thinking gate inactive: "
+            f"_is_adaptive_thinking_model({PROBE!r}) is False"
+        )
+except Exception as exc:  # noqa: BLE001 - report any import/shape drift
+    failures.append(f"anthropic transform unavailable: {exc}")
+try:
+    from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
+
+    handler = AmazonConverseConfig._handle_reasoning_effort_parameter
+    if not getattr(handler, "__benchflow_bedrock_patch__", False):
+        failures.append("reasoning-effort override inactive")
+except Exception as exc:  # noqa: BLE001 - report any import/shape drift
+    failures.append(f"bedrock converse transform unavailable: {exc}")
+if failures:
+    print("; ".join(failures))
+    sys.exit(1)
+"""
+
+
+def route_requires_bedrock_patch(route: LiteLLMRoute) -> bool:
+    """True when this resolved route depends on the Bedrock 4.8+ patch (#602)."""
+    return (
+        route.provider_name == "aws-bedrock"
+        and "reasoning_effort" in route.litellm_params
+    )
+
+
+def _python_from_shebang(executable: Path, *, path_env: str | None) -> str | None:
+    try:
+        line = executable.open("rb").readline(4096).decode(errors="ignore").strip()
+    except OSError:
+        return None
+    if not line.startswith("#!"):
+        return None
+    try:
+        parts = shlex.split(line[2:].strip())
+    except ValueError:
+        return None
+    if not parts:
+        return None
+
+    command = parts[0]
+    if Path(command).name == "env":
+        args = parts[1:]
+        while args and args[0].startswith("-"):
+            args = args[1:]
+        if not args:
+            return None
+        command = args[0]
+
+    if not Path(command).name.startswith("python"):
+        return None
+    if "/" in command:
+        return command
+    return shutil.which(command, path=path_env) or command
+
+
+def _host_python_for_litellm(
+    litellm_executable: str, *, env: dict[str, str] | None = None
+) -> str:
+    executable = Path(litellm_executable)
+    shebang_python = _python_from_shebang(
+        executable, path_env=env.get("PATH") if env else None
+    )
+    if shebang_python:
+        return shebang_python
+    sibling = executable.with_name("python")
+    if sibling.exists():
+        return str(sibling)
+    return sys.executable
+
+
+def _failure_message(runtime: str, detail: str) -> str:
+    return (
+        "Bedrock Claude 4.8+ adaptive-thinking patch is NOT active in the "
+        f"{runtime} LiteLLM runtime: {detail[:2000]}. Failing closed before "
+        "agent launch (#602) - a silent fallback would send the legacy thinking "
+        "shape Bedrock rejects."
+    )
+
+
+def preflight_host_bedrock_patch(
+    *,
+    env: dict[str, str],
+    litellm_executable: str,
+) -> None:
+    """Fail closed if the Bedrock 4.8+ patch is not active for the host proxy."""
+    try:
+        result = subprocess.run(
+            [
+                _host_python_for_litellm(litellm_executable, env=env),
+                "-c",
+                BEDROCK_PATCH_PREFLIGHT_SOURCE,
+            ],
+            env=env,
+            capture_output=True,
+            text=True,
+            timeout=120,
+        )
+    except Exception as exc:
+        raise BedrockPatchPreflightError(
+            _failure_message("host", f"preflight execution failed: {exc}")
+        ) from exc
+    if result.returncode != 0:
+        detail = (result.stdout or "").strip() or (result.stderr or "").strip()
+        raise BedrockPatchPreflightError(_failure_message("host", detail))
+
+
+async def preflight_sandbox_bedrock_patch(
+    sandbox: Any,
+    *,
+    python: str,
+    runtime_dir: str,
+    preflight_path: str,
+) -> None:
+    """Fail closed if the Bedrock 4.8+ patch is not active in a sandbox proxy."""
+    command = (
+        f"PYTHONPATH={shlex.quote(runtime_dir)} "
+        f"{shlex.quote(python)} {shlex.quote(preflight_path)}"
+    )
+    try:
+        result = await sandbox.exec(command, timeout_sec=120)
+    except Exception as exc:
+        raise BedrockPatchPreflightError(
+            _failure_message("sandbox", f"preflight execution failed: {exc}")
+        ) from exc
+    if result.return_code != 0:
+        raise BedrockPatchPreflightError(
+            _failure_message(
+                "sandbox",
+                _exec_details("bedrock patch preflight", result),
+            )
+        )
+
+
+def _exec_details(label: str, result: Any) -> str:
+    stdout = (getattr(result, "stdout", "") or "").strip()
+    stderr = (getattr(result, "stderr", "") or "").strip()
+    details = [f"{label} failed with exit code {getattr(result, 'return_code', '?')}"]
+    if stdout:
+        details.append(f"stdout: {stdout[:2000]}")
+    if stderr:
+        details.append(f"stderr: {stderr[:2000]}")
+    return "; ".join(details)

{benchflow-0.5.3.dev908 → benchflow-0.5.3.dev956}/src/benchflow/providers/litellm_runtime.py

@@ -27,6 +27,13 @@ import yaml
 from benchflow.agents.codex_config import apply_codex_provider_config
 from benchflow.agents.env import uses_native_subscription_auth
 from benchflow.agents.registry import AGENTS
+from benchflow.providers.litellm_bedrock_preflight import (
+    BEDROCK_PATCH_PREFLIGHT_SOURCE,
+    BedrockPatchPreflightError,
+    preflight_host_bedrock_patch,
+    preflight_sandbox_bedrock_patch,
+    route_requires_bedrock_patch,
+)
 from benchflow.providers.litellm_config import (
     LITELLM_MASTER_KEY_ENV,
     LITELLM_MODEL_ALIAS_ENV,
@@ -447,12 +454,13 @@ async def _start_host_litellm(
             "BENCHFLOW_LITELLM_LOG_PATH": str(log_path),
         }
     )
+    litellm_executable = _host_litellm_executable()
     stdout = stdout_path.open("ab")
     stderr = stderr_path.open("ab")
     try:
         process = subprocess.Popen(
             [
-                _host_litellm_executable(),
+                litellm_executable,
                 "--config",
                 str(config_path),
                 "--host",
@@ -481,6 +489,14 @@ async def _start_host_litellm(
     )
     try:
         await _poll_host_health(runner)
+        if route_requires_bedrock_patch(route):
+            # Fail closed before the agent launches when the Bedrock 4.8+
+            # thinking patch did not activate (#602).
+            await asyncio.to_thread(
+                preflight_host_bedrock_patch,
+                env=env,
+                litellm_executable=litellm_executable,
+            )
     except BaseException:
         # A proxy that never became healthy still holds provider credentials and
         # an on-disk config with the master key — tear it down, don't leak it.
@@ -573,6 +589,7 @@ async def _upload_runtime_files_to_sandbox(
         "state": f"{runtime_dir}/state.json",
         "venv": f"{runtime_dir}/venv",
         "launch_config": f"{runtime_dir}/launch_config.json",
+        "preflight": f"{runtime_dir}/bedrock_patch_preflight.py",
     }
     result = await sandbox.exec(f"mkdir -p {shlex.quote(runtime_dir)}", timeout_sec=20)
     if result.return_code != 0:
@@ -591,6 +608,9 @@ async def _upload_runtime_files_to_sandbox(
         sandbox, f"import {_PATCH_MODULE}\n", paths["sitecustomize"], ".py"
     )
     await _upload_text(sandbox, _sandbox_launcher_source(), paths["launcher"], ".py")
+    await _upload_text(
+        sandbox, BEDROCK_PATCH_PREFLIGHT_SOURCE, paths["preflight"], ".py"
+    )
     return paths
 
 
@@ -765,6 +785,16 @@ async def _start_sandbox_litellm(
             port=port,
             stderr_path=paths["stderr"],
         )
+        if route_requires_bedrock_patch(route):
+            # Fail closed before the agent launches when the Bedrock 4.8+
+            # thinking patch did not activate (#602). On Daytona this proxy is
+            # the only protection — there is no host proxy to fall back to.
+            await preflight_sandbox_bedrock_patch(
+                sandbox,
+                python=python,
+                runtime_dir=runtime_dir,
+                preflight_path=paths["preflight"],
+            )
     except BaseException:
         # Never leak a half-started proxy (provider keys + master_key on disk).
         await _terminate_sandbox_litellm(
@@ -1054,6 +1084,8 @@ async def ensure_litellm_runtime(
                 session_id=session_id,
                 agent_name=agent,
             )
+    except BedrockPatchPreflightError:
+        raise
     except Exception as exc:
         return await _fallback_or_raise_for_unavailable_litellm(
             usage_cfg=usage_cfg,

{benchflow-0.5.3.dev908 → benchflow-0.5.3.dev956}/tests/test_litellm_hardening.py

@@ -13,6 +13,7 @@ from pathlib import Path
 
 import pytest
 
+from benchflow.providers import litellm_bedrock_preflight as preflight_mod
 from benchflow.providers import litellm_runtime as runtime_mod
 from benchflow.providers.litellm_config import resolve_litellm_route
 from benchflow.providers.litellm_logging import (
@@ -242,10 +243,17 @@ _SUCCESS_LOG = json.dumps(
 class _FakeSandbox:
     """Drives the real sandbox-local LiteLLM launch/teardown code paths."""
 
-    def __init__(self, *, fail_launch: bool = False, log_content: str = _SUCCESS_LOG):
+    def __init__(
+        self,
+        *,
+        fail_launch: bool = False,
+        fail_preflight: bool = False,
+        log_content: str = _SUCCESS_LOG,
+    ):
         self.uploaded: dict[str, str] = {}
         self.exec_calls: list[str] = []
         self.fail_launch = fail_launch
+        self.fail_preflight = fail_preflight
         self.log_content = log_content
         self._started = False
 
@@ -258,6 +266,10 @@ class _FakeSandbox:
             return _ExecResult(0, stdout=str(len(self.log_content)))
         if "urllib.request" in command:
             return _ExecResult(0)
+        if "bedrock_patch_preflight.py" in command:
+            if self.fail_preflight:
+                return _ExecResult(1, stdout="adaptive-thinking gate inactive")
+            return _ExecResult(0)
         if "launcher.py" in command:
             if self.fail_launch:
                 return _ExecResult(1, stderr="boom")
@@ -410,6 +422,180 @@ def test_bedrock_patch_flags_cost_map_when_entry_present():
         assert flagged, "bedrock 4.8+ cost entries should be flagged adaptive-thinking"
 
 
+# --------------------------------------------------------------------------- #
+# Bedrock 4.8+ patch preflight fails closed (issue #602)                       #
+# --------------------------------------------------------------------------- #
+
+_BEDROCK_ENV = {"AWS_BEARER_TOKEN_BEDROCK": "tok", "AWS_REGION": "us-east-1"}
+
+
+@pytest.mark.parametrize(
+    "model,required",
+    [
+        ("aws-bedrock/us.anthropic.claude-opus-4-8-20251101-v1:0", True),
+        ("aws-bedrock/eu.anthropic.claude-sonnet-4-9-20260301-v1:0", True),
+        ("aws-bedrock/us.anthropic.claude-opus-4-7-20251101-v1:0", False),
+    ],
+)
+def test_route_requires_bedrock_patch_gating(model, required):
+    """Guards PR #668's fail-closed gating for issue #602: only Bedrock Claude 4.8+
+    routes require the thinking patch; 4.7 and below stay best-effort."""
+    route = resolve_litellm_route(model, dict(_BEDROCK_ENV))
+    assert preflight_mod.route_requires_bedrock_patch(route) is required
+
+
+def test_route_requires_bedrock_patch_ignores_non_bedrock_providers():
+    """Guards PR #668 / issue #602 scope: a 4.8 model on a non-Bedrock provider does not
+    trigger the fail-closed preflight (direct Anthropic handles 4.8 natively)."""
+    route = resolve_litellm_route(
+        "minimax/MiniMax-M3",
+        {"MINIMAX_API_KEY": "k", "MINIMAX_BASE_URL": "https://api.minimax.io/v1"},
+    )
+    assert preflight_mod.route_requires_bedrock_patch(route) is False
+
+
+def test_host_bedrock_preflight_uses_litellm_shebang_python(tmp_path):
+    """Guards PR #668 against checking a different Python than the LiteLLM CLI
+    will actually run under when the executable is a script with a shebang."""
+    litellm = tmp_path / "litellm"
+    python = tmp_path / "tools" / "python"
+    python.parent.mkdir()
+    litellm.write_text(f"#!{python}\n")
+
+    assert preflight_mod._host_python_for_litellm(str(litellm)) == str(python)
+
+
+def test_host_bedrock_preflight_resolves_env_shebang_with_proxy_path(tmp_path):
+    """Guards PR #668 against resolving `/usr/bin/env python*` with a different
+    PATH than the LiteLLM proxy receives."""
+    bindir = tmp_path / "bin"
+    bindir.mkdir()
+    python = bindir / "python3"
+    python.write_text("")
+    python.chmod(0o755)
+    litellm = tmp_path / "litellm"
+    litellm.write_text("#!/usr/bin/env python3\n")
+
+    assert preflight_mod._host_python_for_litellm(
+        str(litellm), env={"PATH": str(bindir)}
+    ) == str(python)
+
+
+def test_bedrock_patch_preflight_passes_when_runtime_files_on_pythonpath(tmp_path):
+    """End-to-end happy path for the PR #668 preflight (issue #602): a fresh interpreter
+    with the runtime dir on PYTHONPATH loads sitecustomize -> patch module, so
+    the behavioral probe sees the patched litellm and exits 0."""
+    import os
+    import subprocess
+    import sys
+
+    runtime_mod._write_runtime_files(tmp_path, config={"model_list": []})
+    env = dict(os.environ)
+    env["PYTHONPATH"] = str(tmp_path)
+    result = subprocess.run(
+        [sys.executable, "-c", preflight_mod.BEDROCK_PATCH_PREFLIGHT_SOURCE],
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+
+def test_bedrock_patch_preflight_fails_closed_when_patch_not_loaded(tmp_path):
+    """THE regression test for issue #602's fail-open (fixed in PR #668): when the patch never
+    loads (sitecustomize missing from PYTHONPATH — the exact silent-failure
+    mode), the behavioral probe against stock litellm must exit non-zero so
+    setup fails before the agent launches, instead of regressing mid-task."""
+    import os
+    import subprocess
+    import sys
+
+    env = dict(os.environ)
+    env.pop("PYTHONPATH", None)
+    result = subprocess.run(
+        [sys.executable, "-c", preflight_mod.BEDROCK_PATCH_PREFLIGHT_SOURCE],
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    assert result.returncode != 0
+    assert "inactive" in result.stdout
+
+
+@pytest.mark.asyncio
+async def test_sandbox_litellm_bedrock_route_runs_preflight():
+    """Guards PR #668 / issue #602 wiring: a Bedrock 4.8+ sandbox route must run the
+    patch preflight after health and succeed when the patch is active."""
+    route = resolve_litellm_route(
+        "aws-bedrock/us.anthropic.claude-opus-4-8-20251101-v1:0", dict(_BEDROCK_ENV)
+    )
+    sandbox = _FakeSandbox()
+
+    proc = await runtime_mod._start_sandbox_litellm(
+        sandbox=sandbox,
+        route=route,
+        master_key="sk-master",
+        agent_env=dict(_BEDROCK_ENV),
+        session_id="s",
+        agent_name="openhands",
+    )
+
+    preflights = [c for c in sandbox.exec_calls if "bedrock_patch_preflight.py" in c]
+    assert preflights, "Bedrock 4.8+ route must run the patch preflight"
+    assert proc.base_url == "http://127.0.0.1:45999"
+
+
+@pytest.mark.asyncio
+async def test_sandbox_litellm_bedrock_preflight_failure_fails_closed():
+    """THE sandbox fail-closed regression for issue #602 (PR #668): an inactive patch on
+    a Daytona Bedrock 4.8+ route must abort startup (RuntimeError) and tear the
+    half-started proxy down — never continue into the task fail-open."""
+    route = resolve_litellm_route(
+        "aws-bedrock/us.anthropic.claude-opus-4-8-20251101-v1:0", dict(_BEDROCK_ENV)
+    )
+    sandbox = _FakeSandbox(fail_preflight=True)
+
+    with pytest.raises(RuntimeError, match="NOT active"):
+        await runtime_mod._start_sandbox_litellm(
+            sandbox=sandbox,
+            route=route,
+            master_key="sk-master",
+            agent_env=dict(_BEDROCK_ENV),
+            session_id="s",
+            agent_name="openhands",
+        )
+
+    assert any(call.strip().startswith("rm -rf") for call in sandbox.exec_calls)
+
+
+@pytest.mark.asyncio
+async def test_sandbox_litellm_non_bedrock_route_skips_preflight():
+    """Guards PR #668 / issue #602 scope: non-Bedrock routes never run the preflight, so
+    a broken patch can not fail runs that do not depend on it."""
+    route = resolve_litellm_route(
+        "minimax/MiniMax-M3",
+        {"MINIMAX_API_KEY": "k", "MINIMAX_BASE_URL": "https://api.minimax.io/v1"},
+    )
+    sandbox = _FakeSandbox(fail_preflight=True)  # would fail IF it ran
+
+    proc = await runtime_mod._start_sandbox_litellm(
+        sandbox=sandbox,
+        route=route,
+        master_key="sk-master",
+        agent_env={
+            "MINIMAX_API_KEY": "k",
+            "MINIMAX_BASE_URL": "https://api.minimax.io/v1",
+        },
+        session_id="s",
+        agent_name="openhands",
+    )
+
+    assert not [c for c in sandbox.exec_calls if "bedrock_patch_preflight.py" in c]
+    assert proc.base_url == "http://127.0.0.1:45999"
+
+
 # --------------------------------------------------------------------------- #
 # Embedded callback logger actually produces importable JSONL (no drift)       #
 # --------------------------------------------------------------------------- #

{benchflow-0.5.3.dev908 → benchflow-0.5.3.dev956}/tests/test_litellm_runtime.py

@@ -5,6 +5,7 @@ from types import SimpleNamespace
 import pytest
 
 from benchflow.providers import litellm_runtime as runtime_mod
+from benchflow.providers.litellm_bedrock_preflight import BedrockPatchPreflightError
 from benchflow.providers.litellm_config import LITELLM_MODEL_ALIAS_ENV
 from benchflow.providers.runtime import (
     ProviderRuntime,
@@ -153,7 +154,9 @@ async def test_runtime_reuse_and_stop(monkeypatch):
 
 
 @pytest.mark.asyncio
-async def test_required_usage_fails_when_litellm_lacks_provider_key():
+async def test_required_usage_fails_when_litellm_lacks_provider_key(monkeypatch):
+    monkeypatch.setattr(runtime_mod, "uses_native_subscription_auth", lambda *_: False)
+
     with pytest.raises(RuntimeError, match="requires OPENAI_API_KEY"):
         await ensure_litellm_runtime(
             agent="codex-acp",
@@ -332,6 +335,30 @@ async def test_auto_usage_falls_back_when_litellm_start_fails(monkeypatch):
     assert provider_runtime is None
 
 
+@pytest.mark.asyncio
+async def test_auto_usage_does_not_fallback_on_bedrock_patch_preflight(monkeypatch):
+    """Guards PR #668's fail-closed fix for issue #602: an inactive Bedrock patch
+    must abort even when usage_tracking=auto would normally skip LiteLLM."""
+
+    async def fail_start(**_kwargs):
+        raise BedrockPatchPreflightError("patch inactive")
+
+    monkeypatch.setattr(runtime_mod, "_start_host_litellm", fail_start)
+
+    with pytest.raises(BedrockPatchPreflightError, match="patch inactive"):
+        await ensure_litellm_runtime(
+            agent="openhands",
+            agent_env={
+                "AWS_BEARER_TOKEN_BEDROCK": "tok",
+                "AWS_REGION": "us-west-2",
+            },
+            model="aws-bedrock/us.anthropic.claude-opus-4-8-20251101-v1:0",
+            runtime=None,
+            environment="docker",
+            usage_tracking="auto",
+        )
+
+
 @pytest.mark.asyncio
 async def test_auto_usage_requires_sandbox_handle_for_sandbox_local_litellm():
     """Guards the follow-up to PR #620: auto must not hide wiring bugs."""