benchflow 0.7.6.dev2253__tar.gz → 0.7.6.dev2266__tar.gz

{benchflow-0.7.6.dev2253 → benchflow-0.7.6.dev2266}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.6.dev2253
+Version: 0.7.6.dev2266
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.6.dev2253 → benchflow-0.7.6.dev2266}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.6.dev2253"
+version = "0.7.6.dev2266"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.6.dev2253 → benchflow-0.7.6.dev2266}/src/benchflow/contracts/user.py

@@ -62,7 +62,7 @@ class FunctionUser(BaseUser):
         self,
         fn: Callable[
             [int, str, RoundResult | None],
-            str | None | Awaitable[str | None],
+            str | Awaitable[str | None] | None,
         ],
     ) -> None:
         self._fn = fn

{benchflow-0.7.6.dev2253 → benchflow-0.7.6.dev2266}/src/benchflow/evaluation.py

@@ -64,6 +64,7 @@ from benchflow._utils.scoring import (
     classify_verifier_error,
     count_audit_outcomes,
     count_score_outcomes,
+    extract_reward,
     mean_scored_reward,
     pass_rate,
     pass_rate_excl_errors,
@@ -431,8 +432,7 @@ def _classify_completed_outcomes(
     """
     passed = failed = errored = 0
     for r in completed.values():
-        rewards = r.get("rewards") if isinstance(r, dict) else None
-        reward = rewards.get("reward") if rewards else None
+        reward = extract_reward(r) if isinstance(r, dict) else None
         if reward == 1:
             passed += 1
         elif reward is not None:
@@ -1107,6 +1107,17 @@ class Evaluation:
                 r = json.loads(rfile.read_text())
                 task = r["task_name"]
                 if r.get("rewards") is not None or r.get("verifier_error"):
+                    rewards = r.get("rewards")
+                    if rewards is not None and not isinstance(rewards, dict):
+                        logger.warning(
+                            "Malformed rewards field in %s for task %r: "
+                            "expected dict or null, got %s %r — "
+                            "treating as no reward (task will count as errored)",
+                            rfile,
+                            task,
+                            type(rewards).__name__,
+                            rewards,
+                        )
                     rank = (
                         r.get("rewards") is not None,
                         rfile.stat().st_mtime,

{benchflow-0.7.6.dev2253 → benchflow-0.7.6.dev2266}/src/benchflow/providers/litellm_config.py

@@ -276,7 +276,11 @@ def _route_registered_provider(
         )
 
     if provider_cfg.auth_type == "adc":
-        params = {"model": f"vertex_ai/{bare}"}
+        params = {
+            "model": f"vertex_ai/{bare}",
+            "vertex_project": env.get("GOOGLE_CLOUD_PROJECT", ""),
+            "vertex_location": env.get("GOOGLE_CLOUD_LOCATION", ""),
+        }
         return LiteLLMRoute(
             requested_model=model,
             model_alias=safe_model_alias(model),

{benchflow-0.7.6.dev2253 → benchflow-0.7.6.dev2266}/src/benchflow/providers/litellm_runtime.py

@@ -60,7 +60,7 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-LITELLM_VERSION_SPEC = "litellm[proxy]==1.89.0"
+LITELLM_VERSION_SPEC = "litellm[proxy]==1.91.0"
 LITELLM_SANDBOX_ROOT = "/tmp/benchflow-litellm"
 _CALLBACK_MODULE = "benchflow_litellm_callback"
 _PATCH_MODULE = "benchflow_litellm_bedrock_patch"
@@ -1039,7 +1039,11 @@ async def _upload_runtime_files_to_sandbox(
 
 
 async def _ensure_sandbox_litellm(
-    sandbox: Any, *, venv_dir: str, install_timeout_sec: int = 600
+    sandbox: Any,
+    *,
+    venv_dir: str,
+    extra_requirements: tuple[str, ...] = (),
+    install_timeout_sec: int = 600,
 ) -> str:
     vq = shlex.quote(venv_dir)
     # Prefer uv to bootstrap the venv: many sandbox base images ship a python3
@@ -1047,6 +1051,10 @@ async def _ensure_sandbox_litellm(
     # `python -m venv` and `pip install` fail. uv needs neither (it is the same
     # mechanism the openhands agent install already uses in-sandbox), with a
     # stdlib-venv fallback for images that have a working venv and lack uv.
+    requirements = " ".join(
+        shlex.quote(requirement)
+        for requirement in (LITELLM_VERSION_SPEC, "boto3>=1.40", *extra_requirements)
+    )
     command = f"""
 set -eu
 export PATH="$HOME/.local/bin:$PATH"
@@ -1058,7 +1066,7 @@ if [ -z "$UV" ]; then
 fi
 if [ -n "$UV" ]; then
   [ -x {vq}/bin/python ] || "$UV" venv {vq} >/dev/null 2>&1
-  "$UV" pip install --python {vq}/bin/python -q '{LITELLM_VERSION_SPEC}' 'boto3>=1.40'
+  "$UV" pip install --python {vq}/bin/python -q {requirements}
 else
   PY="$(command -v python3 || command -v python)"
   if [ ! -x {vq}/bin/python ]; then
@@ -1068,7 +1076,7 @@ else
     )
   fi
   {vq}/bin/python -m pip install -q --upgrade pip
-  {vq}/bin/python -m pip install -q '{LITELLM_VERSION_SPEC}' 'boto3>=1.40'
+  {vq}/bin/python -m pip install -q {requirements}
 fi
 {vq}/bin/python - <<'PY'
 import litellm
@@ -1178,48 +1186,62 @@ async def _start_sandbox_litellm(
 ) -> SandboxLiteLLMProcess:
     token = uuid4().hex[:16]
     runtime_dir = f"{LITELLM_SANDBOX_ROOT}/{token}"
+    pid_path = f"{runtime_dir}/litellm.pid"
     config = litellm_proxy_config(route, master_key=master_key)
-    paths = await _upload_runtime_files_to_sandbox(
-        sandbox,
-        runtime_dir=runtime_dir,
-        config=config,
-    )
-    python = await _ensure_sandbox_litellm(
-        sandbox, venv_dir=paths["venv"], install_timeout_sec=install_timeout_sec
-    )
-    env = dict(agent_env)
-    env.update(
-        {
-            "PYTHONPATH": f"{runtime_dir}:{env.get('PYTHONPATH', '')}",
-            "LITELLM_MASTER_KEY": master_key,
-            "BENCHFLOW_LITELLM_LOG_PATH": paths["log"],
-            **_PROXY_DOCS_DISABLE_ENV,
-        }
-    )
-    launch_config = {
-        "python": python,
-        "litellm": f"{paths['venv']}/bin/litellm",
-        "config": paths["config"],
-        "stdout": paths["stdout"],
-        "stderr": paths["stderr"],
-        "pid": paths["pid"],
-        "state": paths["state"],
-        "env": env,
-    }
-    await _upload_text(
-        sandbox, json.dumps(launch_config), paths["launch_config"], ".json"
-    )
-    # The launcher reads launch_config.json (provider env + master key) at
-    # startup; unlink it immediately afterwards so the secret does not sit in
-    # the sandbox filesystem for the life of the run.
-    command = (
-        f"rm -f {shlex.quote(paths['state'])} {shlex.quote(paths['pid'])} "
-        f"{shlex.quote(paths['log'])} && "
-        f"{shlex.quote(python)} {shlex.quote(paths['launcher'])} "
-        f"{shlex.quote(paths['launch_config'])}; "
-        f"rc=$?; rm -f {shlex.quote(paths['launch_config'])}; exit $rc"
-    )
+    proxy_agent_env = dict(agent_env)
+    extra_requirements: tuple[str, ...] = ()
+    adc_json = None
+    if route.provider_name in {"google-vertex", "anthropic-vertex"}:
+        extra_requirements = ("google-cloud-aiplatform>=1.133.0,<2.0",)
+        adc_json = proxy_agent_env.pop("GOOGLE_APPLICATION_CREDENTIALS_JSON", None)
     try:
+        paths = await _upload_runtime_files_to_sandbox(
+            sandbox,
+            runtime_dir=runtime_dir,
+            config=config,
+        )
+        python = await _ensure_sandbox_litellm(
+            sandbox,
+            venv_dir=paths["venv"],
+            extra_requirements=extra_requirements,
+            install_timeout_sec=install_timeout_sec,
+        )
+        if adc_json:
+            adc_path = f"{runtime_dir}/application_default_credentials.json"
+            await _upload_text(sandbox, adc_json, adc_path, ".json")
+            proxy_agent_env["GOOGLE_APPLICATION_CREDENTIALS"] = adc_path
+        env = proxy_agent_env
+        env.update(
+            {
+                "PYTHONPATH": f"{runtime_dir}:{env.get('PYTHONPATH', '')}",
+                "LITELLM_MASTER_KEY": master_key,
+                "BENCHFLOW_LITELLM_LOG_PATH": paths["log"],
+                **_PROXY_DOCS_DISABLE_ENV,
+            }
+        )
+        launch_config = {
+            "python": python,
+            "litellm": f"{paths['venv']}/bin/litellm",
+            "config": paths["config"],
+            "stdout": paths["stdout"],
+            "stderr": paths["stderr"],
+            "pid": paths["pid"],
+            "state": paths["state"],
+            "env": env,
+        }
+        await _upload_text(
+            sandbox, json.dumps(launch_config), paths["launch_config"], ".json"
+        )
+        # The launcher reads launch_config.json (provider env + master key) at
+        # startup; unlink it immediately afterwards so the secret does not sit in
+        # the sandbox filesystem for the life of the run.
+        command = (
+            f"rm -f {shlex.quote(paths['state'])} {shlex.quote(paths['pid'])} "
+            f"{shlex.quote(paths['log'])} && "
+            f"{shlex.quote(python)} {shlex.quote(paths['launcher'])} "
+            f"{shlex.quote(paths['launch_config'])}; "
+            f"rc=$?; rm -f {shlex.quote(paths['launch_config'])}; exit $rc"
+        )
         result = await sandbox.exec(command, timeout_sec=20)
         if result.return_code != 0:
             raise RuntimeError(_exec_details("start sandbox LiteLLM", result))
@@ -1248,7 +1270,7 @@ async def _start_sandbox_litellm(
     except BaseException:
         # Never leak a half-started proxy (provider keys + master_key on disk).
         await _terminate_sandbox_litellm(
-            sandbox, pid_path=paths["pid"], runtime_dir=runtime_dir
+            sandbox, pid_path=pid_path, runtime_dir=runtime_dir
         )
         raise
     endpoint = LiteLLMEndpoint(

{benchflow-0.7.6.dev2253 → benchflow-0.7.6.dev2266}/tests/test_litellm_config.py

@@ -38,6 +38,17 @@ def test_bedrock_model_honors_max_thinking_effort_env():
     assert route.litellm_params["reasoning_effort"] == "high"
 
 
+def test_vertex_route_passes_project_and_location_to_litellm():
+    """Guards PR #985 Vertex project/location forwarding to LiteLLM."""
+    route = resolve_litellm_route(
+        "anthropic-vertex/claude-sonnet-4-6",
+        {"GOOGLE_CLOUD_PROJECT": "skillsbench", "GOOGLE_CLOUD_LOCATION": "global"},
+    )
+
+    assert route.litellm_params["vertex_project"] == "skillsbench"
+    assert route.litellm_params["vertex_location"] == "global"
+
+
 def test_azure_openai_route_uses_resource_and_preview_version():
     route = resolve_litellm_route(
         "azure-foundry-openai/gpt-5.5",

{benchflow-0.7.6.dev2253 → benchflow-0.7.6.dev2266}/tests/test_litellm_hardening.py

@@ -288,6 +288,7 @@ class _FakeSandbox:
         self,
         *,
         fail_launch: bool = False,
+        fail_install: bool = False,
         fail_preflight: bool = False,
         log_content: str = _SUCCESS_LOG,
     ):
@@ -296,6 +297,7 @@ class _FakeSandbox:
         self.exec_calls: list[str] = []
         self.exec_timeouts: list[int | None] = []
         self.fail_launch = fail_launch
+        self.fail_install = fail_install
         self.fail_preflight = fail_preflight
         self.log_content = log_content
         self._started = False
@@ -309,6 +311,8 @@ class _FakeSandbox:
     ) -> _ExecResult:
         self.exec_calls.append(command)
         self.exec_timeouts.append(timeout_sec)
+        if "pip install" in command and "litellm" in command and self.fail_install:
+            return _ExecResult(1, stderr="install failed")
         if "stat -c %s" in command:
             return _ExecResult(0, stdout=str(len(self.log_content)))
         if "urllib.request" in command:
@@ -405,6 +409,81 @@ async def test_sandbox_litellm_install_uses_configured_setup_timeout():
     assert sandbox.exec_timeouts[install_index] == 901
 
 
+@pytest.mark.asyncio
+async def test_vertex_sandbox_litellm_has_google_runtime_and_local_adc():
+    """Guards PR #985 Vertex sandbox dependency and ADC staging."""
+    route = resolve_litellm_route(
+        "anthropic-vertex/claude-sonnet-4-6",
+        {"GOOGLE_CLOUD_PROJECT": "project", "GOOGLE_CLOUD_LOCATION": "region"},
+    )
+    sandbox = _FakeSandbox()
+
+    await runtime_mod._start_sandbox_litellm(
+        sandbox=sandbox,
+        route=route,
+        master_key="sk-master",
+        agent_env={"GOOGLE_APPLICATION_CREDENTIALS_JSON": '{"type":"test"}'},
+        session_id="s",
+        agent_name="claude-agent-acp",
+    )
+
+    install_command = next(
+        command
+        for command in sandbox.exec_calls
+        if "pip install" in command and "litellm" in command
+    )
+    assert "google-cloud-aiplatform>=1.133.0,<2.0" in install_command
+    adc_path = next(
+        path
+        for path in sandbox.uploaded
+        if path.endswith("application_default_credentials.json")
+    )
+    launch_path = next(
+        path for path in sandbox.uploaded if path.endswith("launch_config.json")
+    )
+    assert (
+        json.loads(sandbox.uploaded[launch_path])["env"][
+            "GOOGLE_APPLICATION_CREDENTIALS"
+        ]
+        == adc_path
+    )
+    assert (
+        "GOOGLE_APPLICATION_CREDENTIALS_JSON"
+        not in json.loads(sandbox.uploaded[launch_path])["env"]
+    )
+
+
+@pytest.mark.asyncio
+async def test_vertex_install_failure_cleans_runtime_without_uploading_adc():
+    """Guards PR #985 cleanup when Vertex dependency installation fails."""
+    route = resolve_litellm_route(
+        "anthropic-vertex/claude-sonnet-4-6",
+        {"GOOGLE_CLOUD_PROJECT": "project", "GOOGLE_CLOUD_LOCATION": "region"},
+    )
+    sandbox = _FakeSandbox(fail_install=True)
+
+    with pytest.raises(RuntimeError, match="install LiteLLM in sandbox"):
+        await runtime_mod._start_sandbox_litellm(
+            sandbox=sandbox,
+            route=route,
+            master_key="sk-master",
+            agent_env={"GOOGLE_APPLICATION_CREDENTIALS_JSON": '{"type":"test"}'},
+            session_id="s",
+            agent_name="claude-agent-acp",
+        )
+
+    runtime_dir = next(
+        path.rsplit("/", 1)[0]
+        for path in sandbox.uploaded
+        if path.endswith("config.yaml")
+    )
+    assert f"rm -rf {runtime_dir}" in sandbox.exec_calls
+    assert not any(
+        path.endswith("application_default_credentials.json")
+        for path in sandbox.uploaded
+    )
+
+
 @pytest.mark.asyncio
 async def test_sandbox_litellm_stop_imports_usage_and_cleans_up():
     route = resolve_litellm_route(

{benchflow-0.7.6.dev2253 → benchflow-0.7.6.dev2266}/tests/test_litellm_runtime.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
 import json
+from importlib.metadata import version
 from types import SimpleNamespace
 
 import pytest
+from packaging.requirements import Requirement
 
 from benchflow.agents.codex_config import CODEX_DEFAULT_AUTH_REQUEST_ENV
 from benchflow.agents.env import resolve_agent_env
@@ -35,6 +37,12 @@ class FakeLiteLLMServer:
         self.stopped = True
 
 
+def test_sandbox_litellm_version_matches_host_requirement():
+    """Guards PR #985 sandbox/host LiteLLM pin parity."""
+    requirement = Requirement(runtime_mod.LITELLM_VERSION_SPEC)
+    assert requirement.specifier.contains(version("litellm"), prereleases=True)
+
+
 @pytest.mark.asyncio
 async def test_host_litellm_rewrites_codex_env(monkeypatch):
     async def fake_start(**kwargs):