verifiers 0.4.1.dev50__tar.gz → 0.4.1.dev53__tar.gz

{verifiers-0.4.1.dev50 → verifiers-0.4.1.dev53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev50
+Version: 0.4.1.dev53
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -46,12 +46,8 @@ Requires-Dist: tenacity>=8.5.0
 Requires-Dist: tomli-w>=1.0.0
 Requires-Dist: typing-extensions>=4.12.2
 Requires-Dist: uvicorn>=0.52.0
-Provides-Extra: e2b
-Requires-Dist: e2b<3,>=2.35.0; extra == 'e2b'
 Provides-Extra: harbor
 Requires-Dist: harbor==0.24.0; (python_full_version >= '3.12') and extra == 'harbor'
-Provides-Extra: modal
-Requires-Dist: modal>=1.5.4; extra == 'modal'
 Provides-Extra: openenv
 Requires-Dist: openenv>=0.4.1; extra == 'openenv'
 Provides-Extra: ta

{verifiers-0.4.1.dev50 → verifiers-0.4.1.dev53}/pyproject.toml

@@ -96,14 +96,6 @@ harbor = [
     # Harbor currently publishes Python 3.12+ packages; keep core verifiers installable on 3.11.
     "harbor==0.24.0 ; python_full_version >= '3.12'",
 ]
-modal = [
-    "modal>=1.5.4",
-]
-e2b = [
-    # <3: the runtime swaps the SDK command handle's private stream decoders for
-    # byte-accurate live output (see runtimes/e2b.py _byte_handle).
-    "e2b>=2.35.0,<3",
-]
 openenv = [
     "openenv>=0.4.1",
 ]
@@ -225,11 +217,7 @@ markers = [
     # is applied per axis, so it matches every cell touching that value on any axis.
     "subprocess: integration cells touching the subprocess (host) runtime",
     "docker: integration cells touching the docker runtime",
-    "podman: integration cells touching the podman runtime (needs the engine on the host; local-only)",
-    "apptainer: integration cells touching the apptainer runtime (needs the engine on the host; local-only)",
     "prime: integration cells touching the prime runtime (real sandbox + tunnel)",
-    "modal: integration cells touching the modal runtime (external sandbox; local-only, excluded in CI)",
-    "e2b: integration cells touching the e2b runtime (external sandbox; local-only, excluded in CI)",
     "colocated: integration cells with a tool server colocated in the harness runtime",
     "service: integration cells with a tool server in a Compose service beside the harness",
     "null: integration cells on the null harness",

{verifiers-0.4.1.dev50 → verifiers-0.4.1.dev53}/tests/integration/conftest.py

@@ -5,8 +5,8 @@ model-driven tests skip.
 
 Each behavior module fixes one task and fans it across the harness x runtime grid in
 `grid.py`, which also says why a cell is skipped. `run` drives the `Env` API in-process,
-the calls the eval CLI makes. The placement fixtures below are indirect-only: they translate a parametrized value, and using one without
-`indirect=True` fails loudly.
+the calls the eval CLI makes. The placement fixtures below are indirect-only: they
+translate a parametrized value, and using one without `indirect=True` fails loudly.
 """
 
 import asyncio
@@ -54,7 +54,7 @@ def tool_runtime(request) -> dict:
         return {"service": "tools"}
     if request.param == "service-vm":
         return {"service": "tools-vm"}
-    return {"runtime": {"type": request.param}}
+    return {"runtime": runtime_config(request.param)}
 
 
 # Built-in harnesses are bundled in the `harnesses` package; the agent CLIs (`rlm` /
@@ -180,7 +180,7 @@ def run():
     makes (`serving`, `slots`, `run_slot`, retries included), without its run
     directory, platform push, or dashboard. The Examples job covers that CLI layer."""
 
-    async def _run(taskset: str, **kwargs) -> list[Episode]:
+    async def _episodes(taskset: str, **kwargs) -> list[Episode]:
         config = _eval_config(taskset, **kwargs)
         env = load_environment(config.env)
         tasks = list(env.taskset.select(config.select))
@@ -198,7 +198,7 @@ def run():
                 )
             )
 
-    return _run
+    return _episodes
 
 
 @pytest.fixture

{verifiers-0.4.1.dev50 → verifiers-0.4.1.dev53}/tests/integration/fixtures/agent_user.py

@@ -90,7 +90,7 @@ class ScriptedAgentUserHarness(Harness[HarnessConfig]):
             "uv_script": script[-1],
             "read_secret_api": read_api,
         }
-        return await runtime.run_program(["sh", "-c", PROBE], {})
+        return await runtime.run(["sh", "-c", PROBE], {})
 
 
 class AgentUserData(vf.TaskData):

{verifiers-0.4.1.dev50 → verifiers-0.4.1.dev53}/tests/integration/grid.py

@@ -2,35 +2,31 @@
 cell is skipped.
 
 Each behavior module fixes one task and fans it over `grid(...)`: every bundled harness
-x every runtime kind. The harness grid runs one local container engine (`docker`);
-`podman` / `apptainer` share its container runtime code and are exercised by the
-runtime-only grids (`placements`, `runtimes_grid`), where the harness is fixed. A cell
-the stack cannot run is collected as a skip with its reason. Every reason derives from a
-capability the code declares, never from a list of names, so the grid follows the code:
-
-- `Harness.NEEDS_CONTAINER`: the harness never runs on the host (`subprocess`)
-- `Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support
-  (`SUPPORTS_RESUME` or a native session): the behavior needs a shell, MCP tools,
-  skills, or a resumable exchange
-- `RuntimeBackend.supports_user`, `RuntimeBackend.supports_expose`, `NetworkPolicyConfig`: the
-  behavior needs a user switch, a tool server reachable from another runtime, or a
-  network policy
-- `podman` / `apptainer` need their engine on the host
-
-CI deselects `podman`, `apptainer`, `modal` and `e2b` by mark: local engines and external
-sandboxes.
+on Prime, the production runtime. The basic single- and multi-turn behaviors also run
+on the host (`DEBUG_RUNTIMES`), which needs no container. Docker cells run on the CI
+runner itself and make the jobs too slow, so the Docker runtime is covered by
+`tests/unit/test_runtimes.py` instead. Every reason a cell cannot run derives
+from a capability the code declares, never from a list of names, so the grid follows
+the code.
+
+A harness without a capability the behavior needs is collected as a skip with its
+reason (`Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support via
+`SUPPORTS_RESUME` or a native session).
+
+A runtime without a feature the behavior needs (`RuntimeBackend.SUPPORTS_*`, or a
+`NetworkPolicyConfig` for a network policy) skips the cell the same way. That a runtime
+refuses an undeclared feature, with a message naming it, is tested in
+`tests/unit/test_runtimes.py`.
 
 Every cell carries its axes' marks, so `-m` selects subsets:
 
-    uv run pytest tests/integration -n 8 -m "not podman and not apptainer and not modal and not e2b"  # CI
-    uv run pytest tests/integration -n 8 -m "docker and codex"
-    uv run pytest tests/integration -n 8 -m e2b                                                      # local
+    uv run pytest tests/integration -n 8 -m "prime and codex"
+    uv run pytest tests/integration -n 8 -m prime
 
 `terminus-2` (drives the host tmux) and `browser-use` (needs its Playwright image, which
 prime cannot pull) stay outside the grid.
 """
 
-import shutil
 from collections.abc import Sequence
 from typing import get_args
 
@@ -40,7 +36,7 @@ from pydantic import TypeAdapter
 import verifiers.v1 as vf
 from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.harness import Harness
-from verifiers.v1.runtimes import RuntimeConfig, _backend_cls
+from verifiers.v1.runtimes import RuntimeConfig, backend_cls
 from verifiers.v1.utils.loaders import harness_class, harness_config_type
 
 HARNESSES: list[str | dict] = [
@@ -58,20 +54,48 @@ HARNESSES: list[str | dict] = [
     "mini-swe-agent",
     "prime-agent",
 ]
-RUNTIMES = ["subprocess", "docker", "podman", "apptainer", "prime", "modal", "e2b"]
-ENGINES = ("podman", "apptainer")
-HARNESS_RUNTIMES = [runtime for runtime in RUNTIMES if runtime not in ENGINES]
-# Runtimes whose VMs run a task's Compose services.
-COMPOSE_VMS = ("prime", "modal")
+DEBUG_RUNTIMES = ("subprocess",)
+RUNTIMES = (*DEBUG_RUNTIMES, "prime")
+
+# The runtime features a behavior can need, as `needs` names them.
+RUNTIME_FEATURES = {
+    "container": "task images",
+    "user": "running as another user",
+    "network": "network policies",
+    "expose": "port exposure",
+    "compose": "Compose services",
+}
 
 # Harnesses that cannot run against the CI model yet.
 # TODO: drop each entry once its fix lands.
 CI_MODEL_GAPS = {
     "kimi-code": "Prime chat completions rejects kimi-code's prompt_cache_key",
-    "openclaw": "openclaw cannot load the custom `internal/` model provider",
 }
 
 
+# Harnesses whose program rewrites earlier turns by design, so a rollout can record
+# several branches. Each test names the ones it lets fork. Recheck on version bumps.
+FORKS = {
+    # Acts only through tool calls: it drops a tool-less reply from its history and
+    # resends only its format error.
+    "mini-swe-agent": "https://github.com/SWE-agent/mini-swe-agent/blob/v2.4.6/src/minisweagent/agents/default.py#L114",
+    # Moves its runtime-context message to the tail of every request and drops the
+    # earlier one, unless the provider's replay policy is append-only.
+    "openclaw": "https://github.com/openclaw/openclaw/blob/v2026.9.5/src/agents/internal-runtime-context.ts#L402-L408",
+}
+
+
+def assert_one_branch(
+    trace: vf.Trace, harness: vf.HarnessConfig, forks: Sequence[str] = ()
+) -> None:
+    """A rollout records one branch, unless its harness is one of `forks` (see `FORKS`)."""
+    assert set(forks) <= FORKS.keys(), (
+        f"no FORKS reason for {set(forks) - FORKS.keys()}"
+    )
+    if harness.id not in forks:
+        assert trace.num_branches == 1, f"{trace.num_branches} branches"
+
+
 def agent_seats(taskset: str, env_id: str = "") -> list[str]:
     """The agent seats the env declares for `taskset`: `agent` for a plain taskset, its
     own roles for a recipe env."""
@@ -94,26 +118,31 @@ def mark(name: str) -> pytest.MarkDecorator:
 def runtime_config(runtime: str, harness: str | dict | None = None) -> dict:
     config: dict = {"type": runtime}
     program = harness is not None and harness_class(harness_id(harness)).NEEDS_CONTAINER
-    # The default sandboxes are too small to install and run an agent CLI: e2b's box
-    # starves rlm's ACP startup, and prime's 1 CPU / 2 GB gets openclaw terminated.
-    if runtime == "e2b":
-        config["memory"] = 2.0
+    # Prime's default 1 CPU / 2 GB is too small to install and run an agent CLI.
     if runtime == "prime" and program:
         config.update(cpu=2.0, memory=4.0)
     return config
 
 
 def runtime_cls(runtime: str) -> type:
-    return _backend_cls(TypeAdapter(RuntimeConfig).validate_python({"type": runtime}))
-
-
-def runtime_supports_user(runtime: str) -> bool:
-    return runtime_cls(runtime).supports_user
+    return backend_cls(
+        TypeAdapter(RuntimeConfig).validate_python(runtime_config(runtime))
+    )
 
 
-def runtime_supports_network_policy(runtime: str) -> bool:
-    config = TypeAdapter(RuntimeConfig).validate_python({"type": runtime})
-    return isinstance(config, NetworkPolicyConfig)
+def supports(runtime: str, feature: str) -> bool:
+    """Whether the runtime declares `feature`: a backend flag, or, for a network policy,
+    a config that inherits `NetworkPolicyConfig`."""
+    if feature == "network":
+        config = TypeAdapter(RuntimeConfig).validate_python(runtime_config(runtime))
+        return isinstance(config, NetworkPolicyConfig)
+    flag = {
+        "container": "SUPPORTS_TASK_IMAGES",
+        "user": "SUPPORTS_USER_SWITCHING",
+        "expose": "SUPPORTS_PORT_EXPOSURE",
+        "compose": "SUPPORTS_COMPOSE",
+    }[feature]
+    return getattr(runtime_cls(runtime), flag)
 
 
 def resumes(cls: type[Harness]) -> bool:
@@ -129,13 +158,10 @@ def resumes(cls: type[Harness]) -> bool:
 def unsupported(
     harness: str | dict, runtime: str, needs: tuple[str, ...]
 ) -> str | None:
-    """Why this cell cannot run, or None when it can."""
+    """Why this cell cannot run, or None when it can: a capability the behavior needs
+    that the harness or the runtime does not declare."""
     name = harness_id(harness)
     cls = harness_class(name)
-    if runtime in ENGINES and shutil.which(runtime) is None:
-        return f"needs {runtime} on the host"
-    if runtime == "subprocess" and cls.NEEDS_CONTAINER:
-        return f"{name} needs a container runtime"
     if "shell" in needs and not cls.EXECUTES_CODE:
         return f"{name} gives the model no shell"
     if "mcp" in needs and not cls.SUPPORTS_MCP:
@@ -144,43 +170,48 @@ def unsupported(
         return f"{name} has no native skill support"
     if "resume" in needs and not resumes(cls):
         return f"{name} cannot resume an exchange"
-    if "user" in needs and not runtime_supports_user(runtime):
-        return f"{runtime} cannot run commands as another user"
-    if "network" in needs and not runtime_supports_network_policy(runtime):
-        return f"{runtime} has no network policy"
-    return None
+    runtime_needs = (*(("container",) if cls.NEEDS_CONTAINER else ()), *needs)
+    for feature in runtime_needs:
+        if feature in RUNTIME_FEATURES and not supports(runtime, feature):
+            return f"{runtime} does not support {RUNTIME_FEATURES[feature]}"
+    return CI_MODEL_GAPS.get(name)
+
+
+def cell_marks(
+    harness: str | dict, runtime: str, needs: tuple[str, ...]
+) -> list[pytest.MarkDecorator]:
+    if reason := unsupported(harness, runtime, needs):
+        return [pytest.mark.skip(reason=reason)]
+    return []
 
 
 def runtimes_grid(
-    runtimes: Sequence[str] = RUNTIMES, *, needs: tuple[str, ...] = ()
+    runtimes: Sequence[str] = ("prime",), *, needs: tuple[str, ...] = ()
 ) -> list:
     """`harness_runtime` cells for a behavior with no harness axis."""
     cells = []
     for runtime in runtimes:
-        marks = [mark(runtime)]
-        if reason := unsupported("null", runtime, needs):
-            marks.append(pytest.mark.skip(reason=reason))
+        marks = [mark(runtime), *cell_marks("null", runtime, needs)]
         cells.append(pytest.param(runtime, marks=marks, id=f"harness-in-{runtime}"))
     return cells
 
 
 def grid(
     harnesses: Sequence[str | dict] = HARNESSES,
-    runtimes: Sequence[str] = HARNESS_RUNTIMES,
+    runtimes: Sequence[str] = ("prime",),
     *,
     needs: tuple[str, ...] = (),
 ) -> list:
-    """`(harness, harness_runtime)` cells, each marked by its axes and skipped with a
-    reason where the behavior's `needs` or the runtime rule it out."""
+    """`(harness, harness_runtime)` cells, each marked by its axes, skipped where the
+    harness or the runtime rules the behavior out."""
     cells = []
     for harness in harnesses:
         for runtime in runtimes:
-            marks = [mark(harness_id(harness)), mark(runtime)]
-            reason = unsupported(harness, runtime, needs) or CI_MODEL_GAPS.get(
-                harness_id(harness)
-            )
-            if reason:
-                marks.append(pytest.mark.skip(reason=reason))
+            marks = [
+                mark(harness_id(harness)),
+                mark(runtime),
+                *cell_marks(harness, runtime, needs),
+            ]
             cells.append(
                 pytest.param(
                     harness,
@@ -239,45 +270,30 @@ def skip_harness(cells: list, harness: str, reason: str) -> list:
 
 
 def placements(
-    harness_runtimes: tuple[str, ...] = tuple(RUNTIMES),
-    tool_runtimes: tuple[str, ...] = ("colocated", *RUNTIMES),
+    harness_runtimes: tuple[str, ...] = ("prime",),
+    tool_runtimes: tuple[str, ...] = ("colocated", "subprocess", "prime"),
     *,
     shared: bool = False,
 ) -> list:
     """`(harness_runtime, tool_runtime)` cells for a tool server placed colocated in the
     harness's box, in a Compose service beside it (`service`), or in its own runtime
-    (on Prime beside a Prime harness, a VM on the rollout's private network). A `shared`
-    server runs once per eval, so it joins no rollout's network."""
+    (on Prime beside a Prime harness, a VM on the rollout's private network). A
+    `subprocess` tool runtime is a server on the host, reached from the sandbox through
+    the tunnel. A `shared` server runs once per eval, so it joins no rollout's network."""
     cells = []
     for harness_runtime in harness_runtimes:
         for tool_runtime in tool_runtimes:
             marks = [mark(harness_runtime), mark(tool_runtime)]
-            missing = [
-                engine
-                for engine in {harness_runtime, tool_runtime} & set(ENGINES)
-                if shutil.which(engine) is None
-            ]
-            if missing:
-                marks.append(pytest.mark.skip(reason=f"needs {missing[0]} on the host"))
-            elif tool_runtime == "service" and harness_runtime not in COMPOSE_VMS:
-                marks.append(
-                    pytest.mark.skip(
-                        reason="a tool server in a Compose service needs Compose on a "
-                        "Prime or Modal VM"
-                    )
-                )
-            elif (
-                tool_runtime not in ("colocated", "service")
-                and not runtime_cls(tool_runtime).supports_expose
-                # Beside a Prime harness, a rollout's Prime server joins its private network.
-                and not (not shared and tool_runtime == harness_runtime == "prime")
+            reason = None
+            if tool_runtime == "service":
+                reason = unsupported("null", harness_runtime, ("compose",))
+            # Beside a Prime harness, a rollout's Prime server joins its private network.
+            elif tool_runtime != "colocated" and not (
+                not shared and tool_runtime == harness_runtime == "prime"
             ):
-                marks.append(
-                    pytest.mark.skip(
-                        reason=f"{tool_runtime} cannot expose a port; a tool server "
-                        "there is reachable only colocated"
-                    )
-                )
+                reason = unsupported("null", tool_runtime, ("expose",))
+            if reason:
+                marks.append(pytest.mark.skip(reason=reason))
             cells.append(
                 pytest.param(
                     harness_runtime,

{verifiers-0.4.1.dev50 → verifiers-0.4.1.dev53}/tests/integration/test_agent_user.py

@@ -3,7 +3,7 @@
 upstream (agent-user-harness), on the runtimes that can switch users."""
 
 import pytest
-from grid import grid, runtimes_grid
+from grid import assert_one_branch, grid, runtimes_grid
 
 
 @pytest.mark.parametrize(
@@ -52,11 +52,6 @@ async def test_agent_user_harness(run, harness, harness_runtime, scripted_model)
         client=scripted_model,
     )
     (trace,) = episode.traces
-    # mini-swe-agent only acts through tool calls: it drops a tool-less reply (the
-    # scripted "OK") from its history and resends only its format error, so the
-    # trace forks by design. Recheck on version bumps:
-    # https://github.com/SWE-agent/mini-swe-agent/blob/v2.4.6/src/minisweagent/agents/default.py#L114
-    if harness.id != "mini-swe-agent":
-        assert trace.num_branches == 1
+    assert_one_branch(trace, harness, forks=("mini-swe-agent",))
     assert trace.ok, [error.message for error in trace.errors]
     assert trace.num_turns >= 1

{verifiers-0.4.1.dev50 → verifiers-0.4.1.dev53}/tests/integration/test_agentic_judge.py

@@ -13,7 +13,7 @@ async def test_agentic_judge_shared_verifier(run, tmp_path):
             "id": "agentic-judge",
             "solver": {
                 "harness": {"id": "bash"},
-                "runtime": {"type": "docker", "block": ["example.com"]},
+                "runtime": {"type": "prime", "block": ["example.com"]},
                 "scoring": {"mode": "shared"},
             },
             "judge": {
@@ -59,7 +59,7 @@ async def test_env_id_agentic_judge(run, tmp_path):
             "id": "agentic-judge",
             "solver": {
                 "harness": {"id": "bash"},
-                "runtime": {"type": "docker", "block": ["example.com"]},
+                "runtime": {"type": "prime", "block": ["example.com"]},
                 "scoring": {"mode": "isolated"},
             },
             "judge": {
@@ -84,7 +84,7 @@ async def test_env_id_agentic_judge(run, tmp_path):
     assert judge.agent.trainable is False
     assert solver.agent.runtime is not None and judge.agent.runtime is not None
     assert solver.agent.runtime.id != judge.agent.runtime.id
-    assert solver.agent.runtime.type == judge.agent.runtime.type == "docker"
+    assert solver.agent.runtime.type == judge.agent.runtime.type == "prime"
     assert "/app/answer.txt" in solver.state.artifacts
     assert solver.rewards["wrote_phrase"].score == 1.0
     assert solver.rewards["wrote_phrase"].weight == 0.5

{verifiers-0.4.1.dev50 → verifiers-0.4.1.dev53}/tests/integration/test_compose_scoring.py

@@ -1,12 +1,12 @@
-"""Compose scoring uses live shared services or a released, collected submission."""
+"""Compose scoring uses live shared services or a released, collected submission, on
+a Prime VM."""
 
-import shutil
+import os
 
 import pytest
 
 import verifiers.v1 as vf
 from verifiers.v1.configs.agent import TimeoutConfig
-from verifiers.v1.runtimes.container import cli
 from verifiers.v1.tasksets.harbor.taskset import (
     CollectHook,
     HarborArtifact,
@@ -14,10 +14,8 @@ from verifiers.v1.tasksets.harbor.taskset import (
     HarborTask,
 )
 
-pytestmark = [
-    pytest.mark.docker,
-    pytest.mark.skipif(not shutil.which("docker"), reason="needs Docker"),
-]
+# Solver service backends by trace id, to check their VM is gone before grading.
+SOLVERS: dict[str, list] = {}
 
 IMAGE = "python:3.11-slim"
 COMPOSE = f"""
@@ -44,7 +42,7 @@ services:
 
 class ComposeHarness(vf.Harness):
     async def launch(self, ctx, trace, runtime, endpoint, secret, mcp_urls, data):
-        return await runtime.run_program(
+        return await runtime.run(
             [
                 "python",
                 "-c",
@@ -61,23 +59,18 @@ class ComposeTask(HarborTask):
     async def setup(self, trace, runtime, deployment=None):
         await super().setup(runtime)
         if deployment is not None:
-            trace.info["solver_containers"] = [
-                runtime.info.id for runtime in deployment.target_backends.values()
-            ]
+            SOLVERS[trace.id] = list(deployment.target_backends.values())
         else:
-            # Fresh verifier setup happens only after the solver project is removed.
-            for container in trace.info["solver_containers"]:
-                result = await cli("docker", "inspect", container)
-                assert result.exit_code != 0, "solver service survived into grading"
+            # Fresh verifier setup happens only after the solver VM is removed.
+            assert all(
+                backend._host_backend.stopped for backend in SOLVERS[trace.id]
+            ), "solver services survived into grading"
             trace.info["solver_released_before_grading"] = True
 
     async def finalize_services(self, trace, targets):
-        result = await cli(
-            "docker",
-            "inspect",
-            "--format",
-            "{{.State.Running}}",
-            targets["main"].info.id,
+        main = targets["main"].backend
+        result = await main._run_host(
+            "docker", "inspect", "--format", "{{.State.Running}}", main._container
         )
         assert result.stdout.strip() == "false", (
             "workspace must stop before sidecar collection"
@@ -101,7 +94,43 @@ class ComposeTask(HarborTask):
         "isolated-scoring-timeout",
     ],
 )
+@pytest.mark.prime
+@pytest.mark.skipif(not os.environ.get("PRIME_API_KEY"), reason="needs PRIME_API_KEY")
 async def test_compose_scoring(tmp_path, mode, collect_fails, scoring_timeout):
+    task = compose_task(tmp_path, collect_fails, scoring_timeout, mode)
+    config = vf.AgentConfig(
+        model="unused",
+        runtime=vf.PrimeConfig(labels=["vf-ci"]),
+        scoring=vf.ScoringConfig(
+            mode=mode,
+            env={"EXPECT_SIDECAR": "1" if mode == "isolated" else "0"},
+            retries=vf.RetryConfig(max_retries=0),
+        ),
+        timeout=TimeoutConfig(
+            setup=600, finalize=60, scoring=1 if scoring_timeout else 120
+        ),
+    )
+    async with vf.Agent(config) as agent:
+        agent.harness = ComposeHarness(vf.HarnessConfig())
+        trace = await agent.run(task)
+    assert all(backend._host_backend.stopped for backend in SOLVERS.pop(trace.id, []))
+    if mode == "isolated" and collect_fails:
+        assert not trace.ok
+        assert "collect hook failed" in str(trace.errors)
+        assert "solver_released_before_grading" not in trace.info
+    elif scoring_timeout:
+        assert trace.ok and trace.is_timeout and not trace.errors
+        assert trace.stop_condition == "scoring_timeout"
+        assert all(reward is None for reward in trace.rewards.values())
+    else:
+        assert trace.ok and trace.reward == 1, trace.errors
+        assert bool(trace.info.get("collected_sidecars")) == (mode == "isolated")
+        assert bool(trace.info.get("solver_released_before_grading")) == (
+            mode == "isolated"
+        )
+
+
+def compose_task(tmp_path, collect_fails: bool, scoring_timeout: bool, mode: str):
     environment = tmp_path / "environment"
     environment.mkdir()
     (environment / "docker-compose.yaml").write_text(COMPOSE)
@@ -114,7 +143,7 @@ async def test_compose_scoring(tmp_path, mode, collect_fails, scoring_timeout):
         + ("sleep 30\n" if scoring_timeout else "")
         + "echo 1 > /logs/verifier/reward.txt\n"
     )
-    task = ComposeTask(
+    return ComposeTask(
         HarborData(
             prompt="Copy the sidecar answer",
             task_dir=str(tmp_path),
@@ -135,35 +164,3 @@ async def test_compose_scoring(tmp_path, mode, collect_fails, scoring_timeout):
             ],
         )
     )
-    config = vf.AgentConfig(
-        model="unused",
-        runtime=vf.DockerConfig(),
-        deployment=vf.DeploymentConfig(trust_compose=True),
-        scoring=vf.ScoringConfig(
-            mode=mode,
-            env={"EXPECT_SIDECAR": "1" if mode == "isolated" else "0"},
-            retries=vf.RetryConfig(max_retries=0),
-        ),
-        timeout=TimeoutConfig(
-            setup=180, finalize=30, scoring=1 if scoring_timeout else 30
-        ),
-    )
-    async with vf.Agent(config) as agent:
-        agent.harness = ComposeHarness(vf.HarnessConfig())
-        trace = await agent.run(task)
-    for container in trace.info.get("solver_containers", []):
-        assert (await cli("docker", "inspect", container)).exit_code != 0
-    if mode == "isolated" and collect_fails:
-        assert not trace.ok
-        assert "collect hook failed" in str(trace.errors)
-        assert "solver_released_before_grading" not in trace.info
-    elif scoring_timeout:
-        assert trace.ok and trace.is_timeout and not trace.errors
-        assert trace.stop_condition == "scoring_timeout"
-        assert all(reward is None for reward in trace.rewards.values())
-    else:
-        assert trace.ok and trace.reward == 1, trace.errors
-        assert bool(trace.info.get("collected_sidecars")) == (mode == "isolated")
-        assert bool(trace.info.get("solver_released_before_grading")) == (
-            mode == "isolated"
-        )

{verifiers-0.4.1.dev50 → verifiers-0.4.1.dev53}/tests/integration/test_limits.py

@@ -19,7 +19,7 @@ async def test_cap(run, cap, stop_condition):
     (episode,) = await run(
         "counter-tool",
         harness="null",
-        runtime={"type": "subprocess"},
+        runtime={"type": "prime"},
         env={"agent": cap},
     )
     (trace,) = episode.traces
@@ -33,7 +33,7 @@ async def test_rollout_timeout(run):
     (episode,) = await run(
         "counter-tool",
         harness="null",
-        runtime={"type": "subprocess"},
+        runtime={"type": "prime"},
         rollout_timeout=1,
     )
     (trace,) = episode.traces