benchflow 0.7.8.dev2342__tar.gz → 0.7.8.dev2347__tar.gz

{benchflow-0.7.8.dev2342 → benchflow-0.7.8.dev2347}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.8.dev2342
+Version: 0.7.8.dev2347
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.8.dev2342 → benchflow-0.7.8.dev2347}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.8.dev2342"
+version = "0.7.8.dev2347"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.8.dev2342 → benchflow-0.7.8.dev2347}/src/benchflow/sandbox/agentcore.py

@@ -50,6 +50,7 @@ import hashlib
 import io
 import json
 import os
+import re
 import shlex
 import tarfile
 import threading
@@ -92,7 +93,11 @@ _MAX_DOWNLOAD_BYTES = 64 * 1024 * 1024
 # whatever the service currently defaults to.
 _PROTOCOL_CONFIGURATION = {"serverProtocol": "HTTP"}
 #: AgentCore offers PUBLIC or VPC only; see the registry's enforces_no_network.
-_NETWORK_CONFIGURATION = {"networkMode": "PUBLIC"}
+_NETWORK_MODES = ("PUBLIC", "VPC")
+#: ``VpcConfig`` caps both lists at 16 and pins the id shapes.
+_MAX_VPC_IDS = 16
+_SUBNET_ID = re.compile(r"^subnet-[0-9a-zA-Z]{8,17}$")
+_SECURITY_GROUP_ID = re.compile(r"^sg-[0-9a-zA-Z]{8,17}$")
 
 # Environment overrides.
 _ENV_REGION = "BENCHFLOW_AGENTCORE_REGION"
@@ -100,6 +105,9 @@ _ENV_ROLE_ARN = "BENCHFLOW_AGENTCORE_ROLE_ARN"
 _ENV_ECR_REPOSITORY = "BENCHFLOW_AGENTCORE_ECR_REPOSITORY"
 _ENV_IDLE_TIMEOUT = "BENCHFLOW_AGENTCORE_IDLE_TIMEOUT_SEC"
 _ENV_MAX_LIFETIME = "BENCHFLOW_AGENTCORE_MAX_LIFETIME_SEC"
+_ENV_NETWORK_MODE = "BENCHFLOW_AGENTCORE_NETWORK_MODE"
+_ENV_SUBNETS = "BENCHFLOW_AGENTCORE_SUBNETS"
+_ENV_SECURITY_GROUPS = "BENCHFLOW_AGENTCORE_SECURITY_GROUPS"
 
 
 class AgentCoreSandbox(BaseSandbox):
@@ -314,7 +322,7 @@ class AgentCoreSandbox(BaseSandbox):
         contract = {
             "imageDigest": image_digest,
             "lifecycleConfiguration": self._lifecycle_configuration(),
-            "networkConfiguration": _NETWORK_CONFIGURATION,
+            "networkConfiguration": self._network_configuration(),
             "protocolConfiguration": _PROTOCOL_CONFIGURATION,
             "roleArn": self._require_role_arn(),
         }
@@ -341,7 +349,7 @@ class AgentCoreSandbox(BaseSandbox):
                     "containerConfiguration": {"containerUri": image_uri}
                 },
                 roleArn=self._require_role_arn(),
-                networkConfiguration=_NETWORK_CONFIGURATION,
+                networkConfiguration=self._network_configuration(),
                 protocolConfiguration=_PROTOCOL_CONFIGURATION,
                 lifecycleConfiguration=self._lifecycle_configuration(),
                 description=f"BenchFlow sandbox for {self.environment_name}",
@@ -381,7 +389,7 @@ class AgentCoreSandbox(BaseSandbox):
                         "containerConfiguration": {"containerUri": image_uri}
                     },
                     roleArn=self._require_role_arn(),
-                    networkConfiguration=_NETWORK_CONFIGURATION,
+                    networkConfiguration=self._network_configuration(),
                     protocolConfiguration=_PROTOCOL_CONFIGURATION,
                     # Omitting this resets idle/max-lifetime to the service
                     # defaults, silently discarding the caller's configured
@@ -396,7 +404,7 @@ class AgentCoreSandbox(BaseSandbox):
                 image_uri,
                 self._lifecycle_configuration(),
                 self._require_role_arn(),
-                _NETWORK_CONFIGURATION,
+                self._network_configuration(),
                 _PROTOCOL_CONFIGURATION,
             )
             self.logger.info("Adopted existing AgentCore runtime %s", arn)
@@ -510,11 +518,6 @@ class AgentCoreSandbox(BaseSandbox):
         # for — wrong permissions, wrong egress, or a shell that never answers.
         for label, expected, found in (
             ("roleArn", role_arn, detail.get("roleArn")),
-            (
-                "networkConfiguration",
-                dict(network),
-                dict(detail.get("networkConfiguration") or {}),
-            ),
             (
                 "protocolConfiguration",
                 dict(protocol),
@@ -523,6 +526,25 @@ class AgentCoreSandbox(BaseSandbox):
         ):
             if found != expected:
                 drifted[label] = (found, expected)
+
+        def _placement(config: dict[str, Any]) -> tuple[Any, ...]:
+            """Only the placement BenchFlow itself pins.
+
+            ``VpcConfig`` also carries service-managed members that cannot be
+            sent on create, and nothing promises the ids come back in the order
+            they were given — comparing the whole document would refuse every
+            adoption of a healthy VPC runtime.
+            """
+            vpc = config.get("networkModeConfig") or {}
+            return (
+                config.get("networkMode"),
+                tuple(sorted(vpc.get("subnets") or ())),
+                tuple(sorted(vpc.get("securityGroups") or ())),
+            )
+
+        found_network = dict(detail.get("networkConfiguration") or {})
+        if _placement(found_network) != _placement(dict(network)):
+            drifted["networkConfiguration"] = (found_network, dict(network))
         if drifted:
             raise SandboxStartupError(
                 f"AgentCore runtime {runtime_id} does not match this run's "
@@ -564,6 +586,72 @@ class AgentCoreSandbox(BaseSandbox):
             "maxLifetime": lifetime,
         }
 
+    def _network_configuration(self) -> dict[str, Any]:
+        """Where the runtime's microVMs are placed on the network.
+
+        ``PUBLIC`` stays the default so existing deployments do not move, but
+        the ``VPC`` opt-in fails loudly rather than quietly: a half-configured
+        runtime that silently stays public is exactly the exposure the operator
+        set out to remove, and it is invisible once the runtime is READY.
+        """
+
+        def _ids(name: str, pattern: re.Pattern[str], shape: str) -> list[str]:
+            raw = os.environ.get(name) or ""
+            # Canonical order and no duplicates: this list feeds the runtime
+            # contract digest, so "a,b" and "b,a" must not register two
+            # runtimes for the same infrastructure.
+            values: list[str] = sorted(
+                {item.strip() for item in raw.split(",") if item.strip()}
+            )
+            for item in values:
+                if not pattern.match(item):
+                    raise ValueError(
+                        f"{name} must be a comma-separated list of {shape} ids, "
+                        f"got {item!r}."
+                    )
+            if len(values) > _MAX_VPC_IDS:
+                raise ValueError(
+                    f"{name} accepts at most {_MAX_VPC_IDS} ids, got {len(values)}."
+                )
+            return values
+
+        subnets = _ids(_ENV_SUBNETS, _SUBNET_ID, "subnet-*")
+        security_groups = _ids(_ENV_SECURITY_GROUPS, _SECURITY_GROUP_ID, "sg-*")
+        mode = (os.environ.get(_ENV_NETWORK_MODE) or "PUBLIC").strip().upper()
+        if mode not in _NETWORK_MODES:
+            raise ValueError(
+                f"{_ENV_NETWORK_MODE} must be one of {', '.join(_NETWORK_MODES)}, "
+                f"got {mode!r}."
+            )
+        if mode == "PUBLIC":
+            if subnets or security_groups:
+                raise ValueError(
+                    f"{_ENV_SUBNETS}/{_ENV_SECURITY_GROUPS} are set but "
+                    f"{_ENV_NETWORK_MODE} is not VPC, so the runtime would still "
+                    f"be created on the public internet. Set "
+                    f"{_ENV_NETWORK_MODE}=VPC to place it in the VPC."
+                )
+            return {"networkMode": "PUBLIC"}
+        missing = [
+            name
+            for name, values in (
+                (_ENV_SUBNETS, subnets),
+                (_ENV_SECURITY_GROUPS, security_groups),
+            )
+            if not values
+        ]
+        if missing:
+            raise ValueError(
+                f"{_ENV_NETWORK_MODE}=VPC requires {' and '.join(missing)}."
+            )
+        return {
+            "networkMode": "VPC",
+            "networkModeConfig": {
+                "subnets": subnets,
+                "securityGroups": security_groups,
+            },
+        }
+
     def configure_agent_timeout(self, timeout_sec: int) -> None:
         """Keep the remote session alive for the effective rollout budget."""
         self._agent_timeout_sec = max(0, int(timeout_sec))

{benchflow-0.7.8.dev2342 → benchflow-0.7.8.dev2347}/tests/test_agentcore_parallel.py

@@ -743,6 +743,161 @@ class TestAdoptionContract:
             self._verify(self._detail(**{field: value}))
 
 
+_VPC_NETWORK = {
+    "networkMode": "VPC",
+    "networkModeConfig": {
+        "subnets": ["subnet-0a1b2c3d4e5f67890"],
+        "securityGroups": ["sg-0a1b2c3d4e5f67890"],
+    },
+}
+
+
+class TestNetworkConfigurationReachesTheRuntime:
+    """The configured network placement has to survive to the control plane.
+
+    ``_network_configuration`` deciding VPC is worth nothing if registration
+    still sends the old constant, and the mistake is invisible: the runtime
+    reaches READY either way, just reachable from the internet.
+    """
+
+    def _vpc_env(self, monkeypatch):
+        monkeypatch.setenv("BENCHFLOW_AGENTCORE_ROLE_ARN", "arn:aws:iam::1:role/rt")
+        monkeypatch.setenv("BENCHFLOW_AGENTCORE_NETWORK_MODE", "VPC")
+        monkeypatch.setenv("BENCHFLOW_AGENTCORE_SUBNETS", "subnet-0a1b2c3d4e5f67890")
+        monkeypatch.setenv(
+            "BENCHFLOW_AGENTCORE_SECURITY_GROUPS", "sg-0a1b2c3d4e5f67890"
+        )
+
+    def _adopt(self, found):
+        control = MagicMock()
+        control.get_agent_runtime.return_value = {
+            "agentRuntimeArtifact": {
+                "containerConfiguration": {"containerUri": "repo@sha256:a"}
+            },
+            "lifecycleConfiguration": {
+                "idleRuntimeSessionTimeout": 600,
+                "maxLifetime": 7200,
+            },
+            "roleArn": "arn:aws:iam::1:role/rt",
+            "networkConfiguration": found,
+            "protocolConfiguration": {"serverProtocol": "HTTP"},
+        }
+        AgentCoreSandbox._verify_adopted_runtime(
+            control,
+            "rt-1",
+            "repo@sha256:a",
+            {"idleRuntimeSessionTimeout": 600, "maxLifetime": 7200},
+            "arn:aws:iam::1:role/rt",
+            _VPC_NETWORK,
+            {"serverProtocol": "HTTP"},
+        )
+
+    def test_a_service_managed_member_does_not_block_adoption(self):
+        """``VpcConfig`` carries members BenchFlow never sends.
+
+        ``requireServiceS3Endpoint`` cannot even be set on create. Adoption is
+        the path every rollout after the first takes, so comparing the whole
+        document would fail a VPC matrix on everything after its first create.
+
+        Guards PR #1082 against the whole-document comparison it replaced.
+        """
+        self._adopt(
+            {
+                "networkMode": "VPC",
+                "networkModeConfig": {
+                    **_VPC_NETWORK["networkModeConfig"],
+                    "requireServiceS3Endpoint": True,
+                },
+            }
+        )
+
+    def test_a_different_subnet_is_still_refused(self):
+        """Tolerating extra members must not tolerate the wrong network."""
+        from benchflow.sandbox.protocol import SandboxStartupError
+
+        with pytest.raises(SandboxStartupError, match="does not match"):
+            self._adopt(
+                {
+                    "networkMode": "VPC",
+                    "networkModeConfig": {
+                        "subnets": ["subnet-0987654321fedcba"],
+                        "securityGroups": ["sg-0a1b2c3d4e5f67890"],
+                    },
+                }
+            )
+
+    @pytest.mark.asyncio
+    async def test_registration_sends_the_configured_vpc(self, tmp_path, monkeypatch):
+        self._vpc_env(monkeypatch)
+        env = _sandbox(_make_task(tmp_path))
+        control = MagicMock()
+        control.create_agent_runtime.return_value = {
+            "agentRuntimeId": "rt-1",
+            "agentRuntimeArn": "arn:rt",
+        }
+        control.get_agent_runtime.return_value = {"status": "READY"}
+
+        with patch.object(env, "_client", return_value=control):
+            await env._create_or_adopt_runtime("bf_x", "repo@sha256:a")
+
+        sent = control.create_agent_runtime.call_args.kwargs
+        assert sent["networkConfiguration"] == _VPC_NETWORK
+
+    @pytest.mark.asyncio
+    async def test_repointing_a_stale_runtime_keeps_the_vpc(
+        self, tmp_path, monkeypatch
+    ):
+        """``update_agent_runtime`` replaces the whole config it is given.
+
+        Omitting the network here is how a VPC runtime silently reverts to
+        PUBLIC on the next image change.
+        """
+        from botocore.exceptions import ClientError
+
+        self._vpc_env(monkeypatch)
+        env = _sandbox(_make_task(tmp_path))
+        control = MagicMock()
+        control.create_agent_runtime.side_effect = ClientError(
+            {"Error": {"Code": "ConflictException", "Message": "exists"}},
+            "CreateAgentRuntime",
+        )
+        control.get_agent_runtime.return_value = {
+            "status": "READY",
+            "agentRuntimeArtifact": {
+                "containerConfiguration": {"containerUri": "repo@sha256:b"}
+            },
+            "lifecycleConfiguration": env._lifecycle_configuration(),
+            "roleArn": "arn:aws:iam::1:role/rt",
+            "networkConfiguration": _VPC_NETWORK,
+            "protocolConfiguration": {"serverProtocol": "HTTP"},
+        }
+
+        with (
+            patch.object(env, "_client", return_value=control),
+            patch.object(
+                provisioning,
+                "find_runtime_by_name",
+                return_value=("arn:rt", "rt-1", "repo@sha256:a"),
+            ),
+        ):
+            await env._create_or_adopt_runtime("bf_x", "repo@sha256:b")
+
+        sent = control.update_agent_runtime.call_args.kwargs
+        assert sent["networkConfiguration"] == _VPC_NETWORK
+
+    def test_a_vpc_runtime_does_not_share_an_identity_with_a_public_one(
+        self, tmp_path, monkeypatch
+    ):
+        """Same name means the VPC run adopts the existing PUBLIC runtime."""
+        monkeypatch.setenv("BENCHFLOW_AGENTCORE_ROLE_ARN", "arn:aws:iam::1:role/rt")
+        env = _sandbox(_make_task(tmp_path))
+        public = env._runtime_contract_digest("sha256:a")
+
+        self._vpc_env(monkeypatch)
+
+        assert env._runtime_contract_digest("sha256:a") != public
+
+
 class TestDeprecatedCleanupAliasIsSafe:
     """`bench environment cleanup` reaches the same destructive code.
 

{benchflow-0.7.8.dev2342 → benchflow-0.7.8.dev2347}/tests/test_agentcore_sandbox.py

@@ -530,6 +530,107 @@ class TestCapabilityGating:
         )
 
 
+class TestNetworkConfiguration:
+    """Opting a runtime out of the public internet.
+
+    ``networkMode`` accepts ``PUBLIC`` or ``VPC``. Every runtime was
+    registered ``PUBLIC``, so an agent under test was reachable from the
+    internet with no way to opt out. These pin the opt-in and, more
+    importantly, the ways it can fail open.
+    """
+
+    _MODE = "BENCHFLOW_AGENTCORE_NETWORK_MODE"
+    _SUBNETS = "BENCHFLOW_AGENTCORE_SUBNETS"
+    _SECURITY_GROUPS = "BENCHFLOW_AGENTCORE_SECURITY_GROUPS"
+
+    def _vpc_env(self, monkeypatch):
+        monkeypatch.setenv(self._MODE, "VPC")
+        monkeypatch.setenv(self._SUBNETS, "subnet-0a1b2c3d4e5f67890")
+        monkeypatch.setenv(self._SECURITY_GROUPS, "sg-0a1b2c3d4e5f67890")
+
+    def test_the_default_is_public(self, sandbox, monkeypatch):
+        """Unset must stay unchanged, or every existing deployment moves."""
+        for name in (self._MODE, self._SUBNETS, self._SECURITY_GROUPS):
+            monkeypatch.delenv(name, raising=False)
+
+        assert sandbox._network_configuration() == {"networkMode": "PUBLIC"}
+
+    def test_vpc_mode_carries_the_subnets_and_security_groups(
+        self, sandbox, monkeypatch
+    ):
+        """The runtime shape nests these under ``networkModeConfig``.
+
+        Browsers and code interpreters take the same ``VpcConfig`` under
+        ``vpcConfig`` instead; sending that key here is a ValidationException.
+        The ids come back sorted and deduplicated because this list feeds the
+        runtime contract digest: "a,b" and "b,a" must not register two runtimes
+        for the same infrastructure.
+        """
+        monkeypatch.setenv(self._MODE, "VPC")
+        monkeypatch.setenv(
+            self._SUBNETS,
+            "subnet-0a1b2c3d4e5f67890, subnet-01234567, subnet-01234567",
+        )
+        monkeypatch.setenv(self._SECURITY_GROUPS, "sg-0a1b2c3d4e5f67890")
+
+        assert sandbox._network_configuration() == {
+            "networkMode": "VPC",
+            "networkModeConfig": {
+                "subnets": ["subnet-01234567", "subnet-0a1b2c3d4e5f67890"],
+                "securityGroups": ["sg-0a1b2c3d4e5f67890"],
+            },
+        }
+
+    @pytest.mark.parametrize(
+        "missing",
+        ["BENCHFLOW_AGENTCORE_SUBNETS", "BENCHFLOW_AGENTCORE_SECURITY_GROUPS"],
+    )
+    def test_vpc_mode_requires_both_lists(self, sandbox, monkeypatch, missing):
+        """VPC with only half the config is rejected by the service."""
+        self._vpc_env(monkeypatch)
+        monkeypatch.delenv(missing)
+
+        with pytest.raises(ValueError, match=missing):
+            sandbox._network_configuration()
+
+    def test_vpc_resources_without_vpc_mode_are_refused(self, sandbox, monkeypatch):
+        """Ignoring them would leave the runtime on the public internet.
+
+        The operator who sets subnets but forgets the mode believes the agent
+        is isolated. Failing open there is the whole bug this guards.
+
+        Guards PR #1082.
+        """
+        monkeypatch.delenv(self._MODE, raising=False)
+        monkeypatch.setenv(self._SUBNETS, "subnet-0a1b2c3d4e5f67890")
+        monkeypatch.setenv(self._SECURITY_GROUPS, "sg-0a1b2c3d4e5f67890")
+
+        with pytest.raises(ValueError, match=self._MODE):
+            sandbox._network_configuration()
+
+    def test_an_unknown_mode_is_refused(self, sandbox, monkeypatch):
+        """``no-network`` reads as isolation but is not a mode the service has."""
+        monkeypatch.setenv(self._MODE, "ISOLATED")
+
+        with pytest.raises(ValueError, match="PUBLIC"):
+            sandbox._network_configuration()
+
+    def test_a_malformed_subnet_id_is_refused(self, sandbox, monkeypatch):
+        """Otherwise a pasted VPC id surfaces as an opaque ValidationException."""
+        self._vpc_env(monkeypatch)
+        monkeypatch.setenv(self._SUBNETS, "vpc-0a1b2c3d4e5f67890")
+
+        with pytest.raises(ValueError, match="subnet-"):
+            sandbox._network_configuration()
+
+    def test_a_malformed_security_group_id_is_refused(self, sandbox, monkeypatch):
+        self._vpc_env(monkeypatch)
+        monkeypatch.setenv(self._SECURITY_GROUPS, "0a1b2c3d4e5f67890")
+
+        with pytest.raises(ValueError, match="sg-"):
+            sandbox._network_configuration()
+
+
 @pytest.mark.skipif(
     os.environ.get("BENCHFLOW_AGENTCORE_LIVE_TEST") != "1",
     reason="live AWS test; set BENCHFLOW_AGENTCORE_LIVE_TEST=1 to run",