verifiers 0.3.2.dev167__tar.gz → 0.3.2.dev168__tar.gz

{verifiers-0.3.2.dev167 → verifiers-0.3.2.dev168}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev167
+Version: 0.3.2.dev168
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -46,6 +46,8 @@ Requires-Dist: tenacity>=8.5.0
 Requires-Dist: tomli-w>=1.0.0
 Requires-Dist: typing-extensions>=4.12.2
 Requires-Dist: uvicorn>=0.52.0
+Provides-Extra: e2b
+Requires-Dist: e2b<3,>=2.35.0; extra == 'e2b'
 Provides-Extra: harbor
 Requires-Dist: harbor==0.21.0; (python_full_version >= '3.12') and extra == 'harbor'
 Provides-Extra: modal

{verifiers-0.3.2.dev167 → verifiers-0.3.2.dev168}/pyproject.toml

@@ -100,6 +100,11 @@ harbor = [
 modal = [
     "modal>=1.5.4",
 ]
+e2b = [
+    # <3: the runtime swaps the SDK command handle's private stream decoders for
+    # byte-accurate live output (see runtimes/e2b.py _byte_handle).
+    "e2b>=2.35.0,<3",
+]
 openenv = [
     "openenv>=0.4.1",
 ]
@@ -228,6 +233,7 @@ markers = [
     "apptainer: v1 e2e cases touching the apptainer runtime",
     "prime: v1 e2e cases touching the prime runtime (real sandbox + tunnel; local-only, excluded in CI)",
     "modal: v1 e2e cases touching the modal runtime (local-only setup; excluded in CI)",
+    "e2b: v1 e2e cases touching the e2b runtime (real sandbox; local-only, excluded in CI)",
     "colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
     "null: v1 e2e cases on the null harness",
     "bash: v1 e2e cases on the bash harness",

{verifiers-0.3.2.dev167 → verifiers-0.3.2.dev168}/tests/v1/conftest.py

@@ -16,20 +16,21 @@ one without `indirect=True` fails loudly.
 
 Every combination carries its axes' pytest marks, so subsets select with `-m`:
 
-    uv run pytest tests/v1 -n auto                                # everything (needs modal setup)
+    uv run pytest tests/v1 -n auto                                # everything (needs remote setup)
     uv run pytest tests/v1 -n auto -m "not e2e"                   # deterministic CI matrix
-    uv run pytest tests/v1 -n auto -m "e2e and not prime and not modal"  # live CI job
+    uv run pytest tests/v1 -n auto -m "e2e and not prime and not modal and not e2b"  # live CI job
     uv run pytest tests/v1 -n auto -m docker                      # any case touching the docker runtime
     uv run pytest tests/v1 -n auto -m bash                        # only the bash harness
     uv run pytest tests/v1 -n auto -m prime                       # only prime (real sandboxes; local)
     uv run pytest tests/v1 -n auto -m modal                       # only modal (needs local setup)
+    uv run pytest tests/v1 -n auto -m e2b                         # only e2b (real sandboxes; local)
 
-Marks: runtimes `subprocess` / `docker` / `podman` / `apptainer` / `prime` / `modal`, placement `colocated`,
+Marks: runtimes `subprocess` / `docker` / `podman` / `apptainer` / `prime` / `modal` / `e2b`, placement `colocated`,
 harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `openclaw` / `codex` /
 `claude_code` / `hermes_agent`.
 A mark is applied per axis, so it selects every case touching that value on ANY axis; for one exact
 combination use `-k` on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`).
-prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only.
+prime/modal/e2b provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only.
 CI runs deterministic tests across the Python matrix and the remaining live E2Es once.
 """
 

verifiers-0.3.2.dev168/tests/v1/test_configs.py

@@ -0,0 +1,95 @@
+"""Every checked-in v1 eval config parses.
+
+Mirrors prime-rl's config test: glob the configs and assert each validates into its config
+type. The root `configs/*.toml` are the `uv run vf-eval @ <file>` v1 configs (EvalConfig).
+"""
+
+import tomllib
+from pathlib import Path
+
+import pytest
+
+from verifiers.v1.configs.cli.eval import EvalConfig
+from verifiers.v1.runtimes import E2BConfig
+from verifiers.v1.runtimes.e2b import _egress_update
+
+CONFIGS = sorted((Path(__file__).resolve().parents[2] / "configs").glob("*.toml"))
+
+
+@pytest.mark.parametrize("path", CONFIGS, ids=lambda p: p.name)
+def test_eval_config_parses(path: Path) -> None:
+    config = EvalConfig.model_validate(tomllib.load(path.open("rb")))
+    assert config.env.taskset.id
+
+
+@pytest.mark.parametrize(
+    "rule",
+    [
+        "",
+        "https://api.example.com",
+        "example.com:8443",
+        "example.com/path",
+        "user@example.com",
+        "10.0.0.0/99",
+        "example..com",
+    ],
+)
+def test_e2b_config_rejects_unenforceable_egress_rules(rule: str) -> None:
+    with pytest.raises(ValueError, match="allow rules"):
+        E2BConfig(allow=[rule])
+
+
+@pytest.mark.parametrize("rule", ["api.example.com", "*.example.com"])
+def test_e2b_config_rejects_hostname_block_rules(rule: str) -> None:
+    with pytest.raises(ValueError, match="block rules must be IP addresses or CIDR"):
+        E2BConfig(block=[rule])
+
+
+@pytest.mark.parametrize(
+    "rule",
+    [
+        "api.example.com",
+        "*.example.com",
+        "10.0.0.1",
+        "10.0.0.0/24",
+        "2001:db8::1",
+        "2001:db8::/32",
+    ],
+)
+def test_e2b_config_accepts_supported_egress_rules(rule: str) -> None:
+    assert E2BConfig(allow=[rule]).allow == [rule]
+
+
+@pytest.mark.parametrize(
+    "rule", ["10.0.0.1", "10.0.0.0/24", "2001:db8::1", "2001:db8::/32"]
+)
+def test_e2b_config_accepts_supported_block_rules(rule: str) -> None:
+    assert E2BConfig(block=[rule]).block == [rule]
+
+
+def test_e2b_egress_update_states_the_complete_policy() -> None:
+    # The security-critical policy table: whether a "restricted" sandbox actually
+    # gets a deny-everything floor, with framework routes kept reachable. No e2e
+    # placement runs network-restricted, so this table is pinned here.
+    routes = ["https://tunnel.example.com/intercept"]
+
+    unrestricted = _egress_update(E2BConfig(), None)
+    assert unrestricted == {"allow_internet_access": True}
+
+    blocklist = _egress_update(E2BConfig(block=["203.0.113.0/24"]), routes)
+    assert blocklist == {"deny_out": ["203.0.113.0/24"]}
+
+    allowlist = _egress_update(E2BConfig(allow=["api.example.com"]), routes)
+    assert allowlist == {
+        "allow_out": ["tunnel.example.com", "api.example.com"],
+        "deny_out": ["0.0.0.0/0"],
+    }
+
+    framework_only = _egress_update(E2BConfig(allow=[]), routes)
+    assert framework_only == {
+        "allow_out": ["tunnel.example.com"],
+        "deny_out": ["0.0.0.0/0"],
+    }
+
+    no_routes = _egress_update(E2BConfig(allow=[]), [])
+    assert no_routes == {"allow_internet_access": False}

{verifiers-0.3.2.dev167 → verifiers-0.3.2.dev168}/tests/v1/test_e2e.py

@@ -2,8 +2,8 @@
 
 Placement coverage is pairwise (see tests/v1/conftest.py): each list below names the
 combinations a test runs — every axis value at least once plus the cross-boundary pairs
-with distinct networking — instead of fanning the full cross product. prime/modal rows
-are local-only (their marks are excluded in CI)."""
+with distinct networking — instead of fanning the full cross product. prime/modal/e2b
+rows are local-only (their marks are excluded in CI)."""
 
 import shutil
 import subprocess
@@ -142,6 +142,7 @@ CHAT_PLACEMENTS = [
     ),
     pair("bash", "prime", "bash-harness-in-prime"),
     pair("bash", "modal", "bash-harness-in-modal"),
+    pair("bash", "e2b", "bash-harness-in-e2b"),
 ]
 
 # harness x harness runtime for the shell task: every coding agent once (null is a chat
@@ -176,6 +177,7 @@ AGENTIC_PLACEMENTS = [
         marks=[mark.bash, mark.modal],
         id="bash-harness-in-modal-framework-only",
     ),
+    pair("bash", "e2b", "bash-harness-in-e2b"),
 ]
 
 # The scripted user runs in the eval process itself (no placement axis); the harness
@@ -185,6 +187,7 @@ USER_RUNTIMES = [
     pytest.param("docker", marks=[mark.docker], id="harness-in-docker"),
     pytest.param("prime", marks=[mark.prime], id="harness-in-prime"),
     pytest.param("modal", marks=[mark.modal], id="harness-in-modal"),
+    pytest.param("e2b", marks=[mark.e2b], id="harness-in-e2b"),
 ]
 
 # ACP-backed harnesses: each must preserve an exchange across interaction segments and
@@ -218,6 +221,13 @@ ACP_RESUME_PLACEMENTS = [
     ),
     pair("openclaw", "docker", "openclaw-acp-in-docker"),
     pair("rlm", "prime", "rlm-acp-in-prime-vm"),
+    # Give RLM 2 GB for ACP session startup.
+    pytest.param(
+        "rlm",
+        {"type": "e2b", "memory": 2.0},
+        marks=[mark.rlm, mark.e2b],
+        id="rlm-acp-in-e2b",
+    ),
     pytest.param(
         "prime-agent",
         "prime",
@@ -247,6 +257,8 @@ TOOL_PLACEMENTS = [
     pair("prime", "colocated", "harness-in-prime-with-tool-colocated"),
     pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
     pair("subprocess", "modal", "harness-in-subprocess-with-tool-in-modal"),
+    pair("e2b", "colocated", "harness-in-e2b-with-tool-colocated"),
+    pair("subprocess", "e2b", "harness-in-subprocess-with-tool-in-e2b"),
 ]
 
 # The state channel rides the same reachability as TOOL_PLACEMENTS; cover each axis
@@ -256,6 +268,7 @@ TOOL_STATE_PLACEMENTS = [
     pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
     pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
     pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
+    pair("e2b", "colocated", "harness-in-e2b-with-tool-colocated"),
 ]
 
 # Shared servers always run in their own runtime (colocation is per-rollout, shared is
@@ -265,6 +278,7 @@ SHARED_TOOL_PLACEMENTS = [
     pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
     pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
     pair("modal", "modal", "harness-in-modal-with-tool-in-modal"),
+    pair("e2b", "e2b", "harness-in-e2b-with-tool-in-e2b"),
 ]
 
 
@@ -389,7 +403,9 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-acp-resume-v1",
         harness=harness,
-        runtime={"type": harness_runtime},
+        runtime=harness_runtime
+        if isinstance(harness_runtime, dict)
+        else {"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=8,
         max_tokens=8192,

{verifiers-0.3.2.dev167 → verifiers-0.3.2.dev168}/verifiers/v1/runtimes/__init__.py

@@ -25,6 +25,7 @@ from verifiers.v1.runtimes.docker import (
     PodmanRuntime,
     PodmanRuntimeInfo,
 )
+from verifiers.v1.runtimes.e2b import E2BConfig, E2BRuntime, E2BRuntimeInfo
 from verifiers.v1.runtimes.modal import ModalConfig, ModalRuntime, ModalRuntimeInfo
 from verifiers.v1.runtimes.prime import (
     PrimeConfig,
@@ -44,7 +45,8 @@ RuntimeConfig = Annotated[
     | PodmanConfig
     | ApptainerConfig
     | PrimeConfig
-    | ModalConfig,
+    | ModalConfig
+    | E2BConfig,
     Field(discriminator="type"),
 ]
 
@@ -54,7 +56,8 @@ RuntimeInfo = Annotated[
     | PodmanRuntimeInfo
     | ApptainerRuntimeInfo
     | PrimeRuntimeInfo
-    | ModalRuntimeInfo,
+    | ModalRuntimeInfo
+    | E2BRuntimeInfo,
     Field(discriminator="type"),
 ]
 
@@ -67,6 +70,7 @@ def _runtime_cls(config: RuntimeConfig) -> type[Runtime]:
         "apptainer": ApptainerRuntime,
         "prime": PrimeRuntime,
         "modal": ModalRuntime,
+        "e2b": E2BRuntime,
     }[config.type]
 
 
@@ -109,6 +113,9 @@ __all__ = [
     "DockerConfig",
     "DockerRuntime",
     "DockerRuntimeInfo",
+    "E2BConfig",
+    "E2BRuntime",
+    "E2BRuntimeInfo",
     "ModalConfig",
     "ModalRuntime",
     "ModalRuntimeInfo",