benchflow 0.7.7.dev2305__tar.gz → 0.7.8.dev2321__tar.gz

{benchflow-0.7.7.dev2305 → benchflow-0.7.8.dev2321}/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## [Unreleased]
 
+## 0.7.7 — 2026-09-09
+
 ### Added
 - **`network_mode: denylist` blocks a list of URLs and hosts for the agent on
   Docker and Daytona.** The task keeps internet access; `blocked_urls` and
@@ -10,6 +12,29 @@
   and every refused request lands in `trajectory/egress_denylist.jsonl`.
   Other backends refuse the mode at preflight. (#1113)
 
+### Fixed
+
+- **Denylist egress no longer cuts the agent off from its own model.** The
+  controller registers the running LiteLLM gateway's exact
+  `127.0.0.1:<port>` endpoint with the proxy, so clients that ignore
+  `NO_PROXY` (Gemini's Undici `ProxyAgent` tunnels even plain HTTP) still
+  reach it; the exception comes from the live gateway, never task metadata or
+  agent-supplied environment, and every other private destination stays
+  blocked. Reconnects re-register the current port. (#1118)
+- **The denylist is a shared sandbox policy, not a per-harness one.** Every
+  supported ACP harness — custom registrations included — gets the same proxy,
+  certificates, and UID firewall on primary connections and later roles alike,
+  independent of harness name, model id, or provider. (#1118)
+- **`codex-acp` sessions are configured through native ACP settings.** Hosted
+  search is switched off via `CODEX_CONFIG.web_search` (the CLI ignores `-c`
+  overrides), and Codex runs in `agent-full-access` session mode when
+  BenchFlow has already selected a non-root sandbox user, so its bubblewrap
+  sandbox is not nested inside Docker or Daytona — BenchFlow's own user,
+  filesystem restrictions, proxy, and firewall still apply. (#1118)
+- **`cryptography>=44` is a core dependency.** Denylist egress mints TLS
+  certificates on both Docker and Daytona, so the pin moved out of the
+  `sandbox-agentcore` extra. (#1118)
+
 ## 0.7.6 — 2026-09-04
 
 ### Added

{benchflow-0.7.7.dev2305 → benchflow-0.7.8.dev2321}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.7.dev2305
+Version: 0.7.8.dev2321
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow
@@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.12
 Requires-Dist: agent-client-protocol>=0.10
 Requires-Dist: anyio>=4.0
+Requires-Dist: cryptography>=44
 Requires-Dist: google-cloud-aiplatform<2.0,>=1.133.0
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: litellm[proxy]==1.91.0
@@ -58,7 +59,6 @@ Requires-Dist: openai>=1.40; extra == 'judge'
 Provides-Extra: sandbox-agentcore
 Requires-Dist: bedrock-agentcore>=1.18; extra == 'sandbox-agentcore'
 Requires-Dist: boto3>=1.43.31; extra == 'sandbox-agentcore'
-Requires-Dist: cryptography>=44; extra == 'sandbox-agentcore'
 Requires-Dist: pathspec>=0.12; extra == 'sandbox-agentcore'
 Provides-Extra: sandbox-daytona
 Requires-Dist: daytona>=0.184.0; extra == 'sandbox-daytona'

{benchflow-0.7.7.dev2305 → benchflow-0.7.8.dev2321}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.7.dev2305"
+version = "0.7.8.dev2321"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -12,6 +12,8 @@ dependencies = [
     "pydantic>=2.7",
     "pyyaml>=6.0",
     "rich>=13.0",
+    # Denylist egress mints TLS certificates on both Docker and Daytona.
+    "cryptography>=44",
     # Dataset registry bench_version checks (PEP 440 specifier sets).
     "packaging>=24",
     "litellm[proxy]==1.91.0",
@@ -100,10 +102,6 @@ sandbox-agentcore = [
     # (InvokeAgentRuntimeCommand). The boto3 floor is the first release that
     # models the bedrock-agentcore/-control services.
     "bedrock-agentcore>=1.18",
-    # Used directly by the sealed AgentCore transport (RSA-OAEP key wrap +
-    # AES-CTR/HMAC). It otherwise arrives only transitively via
-    # litellm[proxy]; declaring it here keeps the dependency honest.
-    "cryptography>=44",
     "boto3>=1.43.31",
     # Docker-compatible build-context filtering without requiring a daemon.
     "pathspec>=0.12",

{benchflow-0.7.7.dev2305 → benchflow-0.7.8.dev2321}/src/benchflow/acp/runtime.py

@@ -621,6 +621,7 @@ async def connect_acp(
         agent_env,
         model=model,
         reasoning_effort=reasoning_effort,
+        sandboxed=bool(sandbox_user),
     )
     agent_env = await _prepare_openhands_direct_execution(
         env,

{benchflow-0.7.7.dev2305 → benchflow-0.7.8.dev2321}/src/benchflow/agents/codex_config.py

@@ -91,11 +91,23 @@ def apply_codex_launch_config(
     *,
     model: str | None,
     reasoning_effort: str | None,
+    sandboxed: bool = False,
 ) -> tuple[dict[str, str], bool]:
-    """Apply launch-owned effort and report whether config owns model selection."""
+    """Configure the adapter's sandbox, web policy and launch-owned model effort."""
     if agent != "codex-acp":
         return agent_env, False
-    config = _parse_codex_config(agent_env.get(CODEX_CONFIG_ENV))
+    updated_env = dict(agent_env)
+    if sandboxed:
+        # BenchFlow's non-root sandbox and UID firewall own isolation. The
+        # adapter defaults to workspace-write, whose nested bwrap fails on
+        # Docker/Daytona before tools run. CODEX_CONFIG.sandbox_mode does not
+        # fix this: codex-acp overrides it with its session's AgentMode.
+        updated_env.setdefault("INITIAL_AGENT_MODE", "agent-full-access")
+    disable_search = any(
+        agent_env.get(key) == "1"
+        for key in ("BENCHFLOW_DISALLOW_WEB_TOOLS", "BENCHFLOW_EGRESS_DENYLIST")
+    )
+    config = _parse_codex_config(agent_env.get(CODEX_CONFIG_ENV), strict=disable_search)
     provider_model = agent_env.get(_PROVIDER_MODEL_ENV)
     owns_model = bool(
         model
@@ -105,14 +117,15 @@ def apply_codex_launch_config(
         and config is not None
         and config.get("model") == provider_model
     )
-    if not owns_model or not reasoning_effort:
-        return agent_env, owns_model
-
-    assert config is not None
-    updated_env = dict(agent_env)
-    config["model_reasoning_effort"] = reasoning_effort
-    updated_env[CODEX_CONFIG_ENV] = json.dumps(config, separators=(",", ":"))
-    return updated_env, True
+    if config is not None and (disable_search or (owns_model and reasoning_effort)):
+        if disable_search:
+            # codex-acp 1.6.0 ignores CLI -c flags; its supported CODEX_CONFIG
+            # is forwarded to the Codex app-server's thread configuration.
+            config["web_search"] = "disabled"
+        if owns_model and reasoning_effort:
+            config["model_reasoning_effort"] = reasoning_effort
+        updated_env[CODEX_CONFIG_ENV] = json.dumps(config, separators=(",", ":"))
+    return (updated_env if updated_env != agent_env else agent_env), owns_model
 
 
 def _apply_codex_default_auth_request(

{benchflow-0.7.7.dev2305 → benchflow-0.7.8.dev2321}/src/benchflow/agents/registry.py

@@ -670,8 +670,8 @@ AGENTS: dict[str, AgentConfig] = {
                 HostAuthFile("~/.codex/auth.json", "{home}/.codex/auth.json"),
             ],
         ),
-        disallow_web_tools_launch_suffix=" -c tools.web_search=false",
-        disallow_hosted_search_launch_suffix=" -c tools.web_search=false",
+        # codex-acp ignores CLI -c flags. codex_config applies the web policy
+        # through CODEX_CONFIG before launching the adapter.
     ),
     "gemini": AgentConfig(
         name="gemini",

{benchflow-0.7.7.dev2305 → benchflow-0.7.8.dev2321}/src/benchflow/contracts/planes.py

@@ -112,7 +112,12 @@ class RolloutPlanes(Protocol):
     async def ensure_litellm_runtime(self, *args: Any, **kwargs: Any) -> Any: ...
     async def stop_provider_runtime(self, runtime: Any) -> None: ...
     async def start_egress_denylist(
-        self, env: Any, sandbox_user: str | None, denylist: EgressDenylist
+        self,
+        env: Any,
+        sandbox_user: str | None,
+        denylist: EgressDenylist,
+        *,
+        model_gateway_url: str | None = None,
     ) -> None: ...
     async def stop_egress_denylist(self, env: Any, rollout_dir: Path) -> None: ...
     def extract_usage(self, runtime: Any) -> dict[str, Any]: ...

{benchflow-0.7.7.dev2305 → benchflow-0.7.8.dev2321}/src/benchflow/rollout/__init__.py

@@ -1290,8 +1290,12 @@ class Rollout:
 
     async def _start_egress_denylist(self, denylist: EgressDenylist) -> None:
         """(Re)start the egress proxy before an ACP connection; a restored sandbox has none running."""
+        runtime = self._usage_runtime
         await self._planes.start_egress_denylist(
-            self._env, self._config.sandbox_user, denylist
+            self._env,
+            self._config.sandbox_user,
+            denylist,
+            model_gateway_url=runtime.agent_base_url if runtime is not None else None,
         )
 
     async def connect(self) -> None:

{benchflow-0.7.7.dev2305 → benchflow-0.7.8.dev2321}/src/benchflow/rollout_planes.py

@@ -193,9 +193,16 @@ class DefaultRolloutPlanes:
         await stop_provider_runtime(runtime)
 
     async def start_egress_denylist(
-        self, env: Any, sandbox_user: str | None, denylist: EgressDenylist
+        self,
+        env: Any,
+        sandbox_user: str | None,
+        denylist: EgressDenylist,
+        *,
+        model_gateway_url: str | None = None,
     ) -> None:
-        await start_egress_denylist(env, sandbox_user, denylist)
+        await start_egress_denylist(
+            env, sandbox_user, denylist, model_gateway_url=model_gateway_url
+        )
 
     async def stop_egress_denylist(self, env: Any, rollout_dir: Path) -> None:
         await stop_egress_denylist(env, rollout_dir)

{benchflow-0.7.7.dev2305 → benchflow-0.7.8.dev2321}/src/benchflow/sandbox/_egress_denylist_proxy.py

@@ -99,7 +99,19 @@ def _looks_like_address(host: str) -> bool:
 class Policy:
     """Match hosts and URLs against the denylist; scheme, port and query are ignored."""
 
-    def __init__(self, blocked_urls: list[str], blocked_hosts: list[str]):
+    def __init__(
+        self,
+        blocked_urls: list[str],
+        blocked_hosts: list[str],
+        model_gateway_port: int | None = None,
+    ):
+        # Controller-supplied runtime state, never a task-authored allowlist.
+        if model_gateway_port is not None and (
+            type(model_gateway_port) is not int
+            or not 1024 <= model_gateway_port <= 65535
+        ):
+            raise ValueError("invalid model gateway port")
+        self.model_gateway_port = model_gateway_port
         self.prefixes: list[tuple[str, str]] = []
         for raw in blocked_urls:
             url = raw if "://" in raw else "https://" + raw
@@ -118,9 +130,12 @@ class Policy:
         return cls(
             list(data.get("blocked_urls") or []),
             list(data.get("blocked_hosts") or []),
+            data.get("model_gateway_port"),
         )
 
-    def host_rule(self, host: str) -> str | None:
+    def host_rule(self, host: str, port: int = 0) -> str | None:
+        if (host, port) == ("127.0.0.1", self.model_gateway_port):
+            return None
         name = host.strip().rstrip(".").lower()
         if _looks_like_address(name):
             return "ip-literal"
@@ -129,8 +144,8 @@ class Policy:
                 return f"host:{blocked}"
         return None
 
-    def url_rule(self, host: str, path: str) -> str | None:
-        rule = self.host_rule(host)
+    def url_rule(self, host: str, path: str, port: int = 0) -> str | None:
+        rule = self.host_rule(host, port)
         if rule:
             return rule
         key, pkey = host_key(host), _path_key(path)
@@ -230,8 +245,15 @@ def _upstream_allowed(address: str) -> bool:
         return False
 
 
-def _connect_upstream(host: str, port: int) -> socket.socket:
+def _connect_upstream(
+    host: str, port: int, *, model_gateway_port: int | None = None
+) -> socket.socket:
     """Connect to a vetted address of ``host``; the root proxy must not reach sandbox-internal services."""
+    # Gemini's Undici ProxyAgent ignores NO_PROXY, even for the local model
+    # gateway. Permit only the endpoint BenchFlow created; do not resolve a
+    # hostname or expose any other private address/loopback port.
+    if (host, port) == ("127.0.0.1", model_gateway_port):
+        return socket.create_connection((host, port), timeout=HEAD_TIMEOUT)
     addresses = _resolve(host, port)
     if not addresses or not all(_upstream_allowed(a) for a in addresses):
         raise _PrivateDestination(host)
@@ -363,7 +385,7 @@ class Handler(socketserver.BaseRequestHandler):
         host, _, port_s = target.rpartition(":")
         host = host.strip("[]").rstrip(".").lower()
         port = int(port_s) if port_s.isdigit() else 443
-        rule = self.proxy.policy.host_rule(host)
+        rule = self.proxy.policy.host_rule(host, port)
         if rule:
             self._deny(self.request, "CONNECT", f"{host}:{port}", rule)
             return
@@ -378,7 +400,9 @@ class Handler(socketserver.BaseRequestHandler):
             )
             return
         try:
-            upstream = _connect_upstream(host, port)
+            upstream = _connect_upstream(
+                host, port, model_gateway_port=self.proxy.policy.model_gateway_port
+            )
         except _PrivateDestination:
             self._deny(self.request, "CONNECT", f"{host}:{port}", "private-address")
             return
@@ -422,7 +446,7 @@ class Handler(socketserver.BaseRequestHandler):
             if not host:
                 sock.sendall(_response("400 Bad Request", "absolute URL required\n"))
                 return
-        rule = self.proxy.policy.url_rule(host, path)
+        rule = self.proxy.policy.url_rule(host, path, port)
         if rule:
             self._deny(sock, method, url, rule)
             return
@@ -430,7 +454,9 @@ class Handler(socketserver.BaseRequestHandler):
         headers.insert(0, ("Host", authority))
         rest = _body_prefix(headers, rest)
         try:
-            upstream = _connect_upstream(host, port)
+            upstream = _connect_upstream(
+                host, port, model_gateway_port=self.proxy.policy.model_gateway_port
+            )
             if secure:
                 upstream = self.proxy.upstream_ctx.wrap_socket(
                     upstream, server_hostname=host

{benchflow-0.7.7.dev2305 → benchflow-0.7.8.dev2321}/src/benchflow/sandbox/egress_denylist.py

@@ -279,15 +279,34 @@ async def start_egress_denylist(
     sandbox_user: str | None,
     denylist: EgressDenylist,
     *,
+    model_gateway_url: str | None = None,
     timeout_sec: int = 120,
 ) -> None:
     """Upload policy, certificates and the proxy script, then start the proxy as root."""
     if not sandbox_user:
         raise RuntimeError("network_mode='denylist' requires a sandbox_user")
+    gateway_port = None
+    if model_gateway_url is not None:
+        gateway = urllib.parse.urlsplit(model_gateway_url)
+        if (
+            gateway.scheme != "http"
+            or gateway.hostname != "127.0.0.1"
+            or gateway.username is not None
+            or gateway.path not in ("", "/")
+            or gateway.query
+            or gateway.fragment
+            or gateway.port is None
+            or not 1024 <= gateway.port <= 65535
+        ):
+            raise ValueError(
+                "denylist model gateway must be a controller-owned loopback HTTP endpoint"
+            )
+        gateway_port = gateway.port
     material = certificate_material(denylist.inspect_hosts)
     policy = {
         "blocked_urls": list(denylist.blocked_urls),
         "blocked_hosts": list(denylist.blocked_hosts),
+        "model_gateway_port": gateway_port,
     }
     files = {
         "policy.json": json.dumps(policy, indent=2).encode("utf-8"),

benchflow-0.7.8.dev2321/tests/agents/test_codex_config.py

@@ -0,0 +1,117 @@
+"""Codex launch configuration ownership tests."""
+
+import json
+
+import pytest
+
+from benchflow.agents.codex_config import apply_codex_launch_config
+
+
+@pytest.mark.parametrize(
+    "raw_config",
+    [None, "{", "[]", "{}", '{"model":"another-model"}'],
+    ids=["missing", "malformed", "non-object", "missing-model", "mismatch"],
+)
+def test_launch_config_rejects_missing_invalid_or_mismatched_model(raw_config):
+    """Guards PR #1076: only exact valid Codex config owns model selection."""
+    agent_env = {
+        "BENCHFLOW_PROVIDER_MODEL": "benchflow-openai-gpt-5.4-mini",
+        "BENCHFLOW_LITELLM_MODEL_VIA_ENV": "1",
+    }
+    if raw_config is not None:
+        agent_env["CODEX_CONFIG"] = raw_config
+
+    updated_env, owns_model = apply_codex_launch_config(
+        "codex-acp", agent_env, model="openai/gpt-5.4-mini", reasoning_effort="high"
+    )
+
+    assert updated_env is agent_env
+    assert not owns_model
+
+
+def test_launch_config_applies_effort_to_exact_model():
+    """Guards PR #1076: launch-owned model carries requested effort."""
+    agent_env = {
+        "BENCHFLOW_PROVIDER_MODEL": "benchflow-openai-gpt-5.4-mini",
+        "BENCHFLOW_LITELLM_MODEL_VIA_ENV": "1",
+        "CODEX_CONFIG": '{"model":"benchflow-openai-gpt-5.4-mini"}',
+    }
+
+    updated_env, owns_model = apply_codex_launch_config(
+        "codex-acp", agent_env, model="openai/gpt-5.4-mini", reasoning_effort="high"
+    )
+
+    assert owns_model
+    assert json.loads(updated_env["CODEX_CONFIG"])["model_reasoning_effort"] == "high"
+    assert updated_env is not agent_env
+    assert "model_reasoning_effort" not in agent_env["CODEX_CONFIG"]
+
+
+def test_launch_config_rejects_alias_for_a_different_requested_model():
+    """Guards PR #1076: stale proxy aliases cannot claim requested-model ownership."""
+    agent_env = {
+        "BENCHFLOW_PROVIDER_MODEL": "benchflow-openai-gpt-5.4-mini",
+        "BENCHFLOW_LITELLM_MODEL_VIA_ENV": "1",
+        "CODEX_CONFIG": '{"model":"benchflow-openai-gpt-5.4-mini"}',
+    }
+
+    updated_env, owns_model = apply_codex_launch_config(
+        "codex-acp", agent_env, model="openai/gpt-5.5", reasoning_effort="high"
+    )
+
+    assert updated_env is agent_env
+    assert not owns_model
+
+
+@pytest.mark.parametrize("sandboxed", [True, False])
+@pytest.mark.parametrize("requested_mode", [None, "read-only"])
+def test_nested_sandbox_default_preserves_explicit_mode(sandboxed, requested_mode):
+    """Guards PR #1118's Terra E2E failure: bwrap cannot start inside the sandbox."""
+    env = {"INITIAL_AGENT_MODE": requested_mode} if requested_mode else {}
+    result, owns_model = apply_codex_launch_config(
+        "codex-acp", env, model=None, reasoning_effort=None, sandboxed=sandboxed
+    )
+    expected = requested_mode or ("agent-full-access" if sandboxed else None)
+    assert result.get("INITIAL_AGENT_MODE") == expected
+    assert env == ({"INITIAL_AGENT_MODE": requested_mode} if requested_mode else {})
+    assert not owns_model
+
+
+@pytest.mark.parametrize(
+    "policy", ["BENCHFLOW_EGRESS_DENYLIST", "BENCHFLOW_DISALLOW_WEB_TOOLS"]
+)
+def test_web_policy_overrides_search_without_clobbering_provider_config(policy):
+    """Guards PR #1118: supported config must disable provider-side search."""
+    config = {
+        "web_search": "live",
+        "model_providers": {"custom": {"wire_api": "responses"}},
+    }
+    env = {policy: "1", "CODEX_CONFIG": json.dumps(config)}
+    result, owns_model = apply_codex_launch_config(
+        "codex-acp", env, model="custom/opaque-model", reasoning_effort=None
+    )
+    assert json.loads(result["CODEX_CONFIG"]) == {**config, "web_search": "disabled"}
+    assert json.loads(env["CODEX_CONFIG"])["web_search"] == "live"
+    assert not owns_model
+
+
+@pytest.mark.parametrize("config", ["{", "[]"])
+def test_web_policy_rejects_invalid_config_instead_of_leaving_search_enabled(config):
+    """Guards PR #1118: malformed caller configuration cannot bypass web policy."""
+    with pytest.raises(ValueError, match="CODEX_CONFIG"):
+        apply_codex_launch_config(
+            "codex-acp",
+            {"CODEX_CONFIG": config, "BENCHFLOW_EGRESS_DENYLIST": "1"},
+            model=None,
+            reasoning_effort=None,
+        )
+
+
+def test_other_harness_keeps_its_native_configuration():
+    """Guards PR #1118: Codex's nested sandbox setting belongs only to its adapter."""
+    env = {"BENCHFLOW_EGRESS_DENYLIST": "1"}
+    result, owns_model = apply_codex_launch_config(
+        "custom", env, model=None, reasoning_effort=None, sandboxed=True
+    )
+    assert result is env
+    assert not owns_model

{benchflow-0.7.7.dev2305 → benchflow-0.7.8.dev2321}/tests/test_egress_denylist.py

@@ -240,7 +240,7 @@ def stack(tmp_path: Path, monkeypatch):
         "plain.test": plain.server_address[1],
     }
 
-    def fake_connect_upstream(host, port):
+    def fake_connect_upstream(host, port, *, model_gateway_port=None):
         if host == "internal.test":
             raise proxy_mod._PrivateDestination(host)
         return socket.create_connection(("127.0.0.1", ports[host]), timeout=10)