verifiers 0.4.1.dev3__tar.gz → 0.4.1.dev4__tar.gz

{verifiers-0.4.1.dev3 → verifiers-0.4.1.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev3
+Version: 0.4.1.dev4
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -91,6 +91,34 @@ uv tool install prime
 - The [docs](docs/) contain short, human-written guides and overviews about the architecture.
 - The [AGENTS.md](AGENTS.md) and [skills](skills/) are for coding agents and go into more details.
 
+## Contributing
+
+We welcome community contributions. We use [issues](https://github.com/PrimeIntellect-ai/verifiers/issues) to track bugs and feature requests. If you hit a bug, a pain point, or have an idea for a feature, please open an issue.
+
+Contributions are welcome via PR. Please follow these guidelines:
+
+1. Install the [pre-commit hooks](#pre-commit-hooks) so your code is formatted and type-checked.
+2. Keep your PR in "Draft" until it is ready for review.
+3. If your PR resolves an issue, link the issue in the PR description.
+4. Run the [tests](#tests) for the areas you touched.
+
+### Pre-commit hooks
+
+```bash
+uv run pre-commit install
+```
+
+### Tests
+
+Unit tests run one component in isolation, with no network, and mirror the `verifiers/v1` layout. Integration tests run real evals: they need `PRIME_API_KEY` and skip without it.
+
+```bash
+uv run pytest tests/unit -n auto                       # unit
+uv run pytest tests/integration -n 8                   # integration
+uv run pytest tests/integration/test_single_turn.py    # one behavior
+uv run pytest tests/integration -m "docker and codex"  # one runtime and harness
+```
+
 ## Citation
 
 Originally created by Will Brown ([@willccbb](https://github.com/willccbb)).

{verifiers-0.4.1.dev3 → verifiers-0.4.1.dev4}/README.md

@@ -30,6 +30,34 @@ uv tool install prime
 - The [docs](docs/) contain short, human-written guides and overviews about the architecture.
 - The [AGENTS.md](AGENTS.md) and [skills](skills/) are for coding agents and go into more details.
 
+## Contributing
+
+We welcome community contributions. We use [issues](https://github.com/PrimeIntellect-ai/verifiers/issues) to track bugs and feature requests. If you hit a bug, a pain point, or have an idea for a feature, please open an issue.
+
+Contributions are welcome via PR. Please follow these guidelines:
+
+1. Install the [pre-commit hooks](#pre-commit-hooks) so your code is formatted and type-checked.
+2. Keep your PR in "Draft" until it is ready for review.
+3. If your PR resolves an issue, link the issue in the PR description.
+4. Run the [tests](#tests) for the areas you touched.
+
+### Pre-commit hooks
+
+```bash
+uv run pre-commit install
+```
+
+### Tests
+
+Unit tests run one component in isolation, with no network, and mirror the `verifiers/v1` layout. Integration tests run real evals: they need `PRIME_API_KEY` and skip without it.
+
+```bash
+uv run pytest tests/unit -n auto                       # unit
+uv run pytest tests/integration -n 8                   # integration
+uv run pytest tests/integration/test_single_turn.py    # one behavior
+uv run pytest tests/integration -m "docker and codex"  # one runtime and harness
+```
+
 ## Citation
 
 Originally created by Will Brown ([@willccbb](https://github.com/willccbb)).

{verifiers-0.4.1.dev3 → verifiers-0.4.1.dev4}/pyproject.toml

@@ -61,7 +61,6 @@ dev = [
     "ty>=0.0.63",
     "pytest>=7.0.0,<9.1.0",
     "pytest-asyncio>=0.21.0",
-    "pytest-cov>=4.0.0",
     "pytest-xdist>=3.8.0",
 ]
 # V1 examples and support packages, resolved from their local sources below.
@@ -213,7 +212,7 @@ known-first-party = ["verifiers"]
 [tool.pytest.ini_options]
 minversion = "7.0"
 testpaths = ["tests"]
-pythonpath = ["tests/v1/fixtures"]
+pythonpath = ["tests/integration", "tests/integration/fixtures"]
 python_files = ["test_*.py"]
 python_classes = ["Test*"]
 python_functions = ["test_*"]
@@ -224,27 +223,27 @@ addopts = [
     "--quiet",
 ]
 markers = [
-    "e2e: marks end-to-end eval-run tests (need a model API key; skipped without one)",
-    # v1 e2e matrix axes — select subsets with `-m` (see tests/v1/conftest.py). A mark is applied
-    # per axis, so it matches every case touching that value on any axis.
-    "subprocess: v1 e2e cases touching the subprocess (host) runtime",
-    "docker: v1 e2e cases touching the docker runtime",
-    "podman: v1 e2e cases touching the podman runtime",
-    "apptainer: v1 e2e cases touching the apptainer runtime",
-    "prime: v1 e2e cases touching the prime runtime (real sandbox + tunnel; local-only, excluded in CI)",
-    "modal: v1 e2e cases touching the modal runtime (local-only setup; excluded in CI)",
-    "e2b: v1 e2e cases touching the e2b runtime (real sandbox; local-only, excluded in CI)",
-    "colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
-    "null: v1 e2e cases on the null harness",
-    "bash: v1 e2e cases on the bash harness",
-    "browser_use: v1 e2e cases on the browser_use harness",
-    "claude_code: v1 e2e cases on the claude-code harness",
-    "openclaw: v1 e2e cases on the OpenClaw harness",
-    "hermes_agent: v1 e2e cases on the hermes-agent harness",
-    "rlm: v1 e2e cases on the rlm harness",
-    "kimi_code: v1 e2e cases on the kimi-code harness",
-    "pi: v1 e2e cases on the pi harness",
-    "codex: v1 e2e cases on the codex harness",
+    # Integration grid axes — select subsets with `-m` (see tests/integration/grid.py). A mark
+    # is applied per axis, so it matches every cell touching that value on any axis.
+    "subprocess: integration cells touching the subprocess (host) runtime",
+    "docker: integration cells touching the docker runtime",
+    "podman: integration cells touching the podman runtime (needs the engine on the host; local-only)",
+    "apptainer: integration cells touching the apptainer runtime (needs the engine on the host; local-only)",
+    "prime: integration cells touching the prime runtime (real sandbox + tunnel)",
+    "modal: integration cells touching the modal runtime (external sandbox; local-only, excluded in CI)",
+    "e2b: integration cells touching the e2b runtime (external sandbox; local-only, excluded in CI)",
+    "colocated: integration cells with a tool server colocated in the harness runtime",
+    "null: integration cells on the null harness",
+    "bash: integration cells on the bash harness",
+    "claude_code: integration cells on the claude-code harness",
+    "openclaw: integration cells on the OpenClaw harness",
+    "hermes_agent: integration cells on the hermes-agent harness",
+    "rlm: integration cells on the rlm harness",
+    "kimi_code: integration cells on the kimi-code harness",
+    "pi: integration cells on the pi harness",
+    "codex: integration cells on the codex harness",
+    "mini_swe_agent: integration cells on the mini-swe-agent harness",
+    "prime_agent: integration cells on the prime-agent harness",
     "asyncio: marks tests as async tests",
 ]
 filterwarnings = [
@@ -268,23 +267,3 @@ redundant-cast = "ignore"
 # verifiers/v1 is vendored (heavily typed upstream for a different checker); it is not part
 # of verifiers' ty gate. environments/ are user-authored envs, also out of scope.
 exclude = ["environments", "verifiers/v1"]
-
-
-[tool.coverage.run]
-source = ["verifiers"]
-omit = [
-    "*/tests/*",
-    "*/__pycache__/*",
-    "*/site-packages/*",
-]
-
-[tool.coverage.report]
-exclude_lines = [
-    "pragma: no cover",
-    "def __repr__",
-    "if self.debug:",
-    "raise AssertionError",
-    "raise NotImplementedError",
-    "if __name__ == .__main__.:",
-    "if TYPE_CHECKING:",
-]

{verifiers-0.4.1.dev3/tests/v1 → verifiers-0.4.1.dev4/tests/integration}/conftest.py

@@ -1,37 +1,12 @@
-"""Shared fixtures + helpers for the v1 end-to-end eval tests.
-
-These tests run REAL eval runs (a live model endpoint, real runtimes) with the smallest
-settings that still exercise the path, then assert on the resulting `Trace`(s) — they are
-not unit tests of individual components. They need a model API key (`PRIME_API_KEY`);
-without one the `e2e`-marked tests skip (config parsing still runs).
-
-`run_v1` mirrors the eval CLI (`run_eval`, in-process); `run_v1_server` drives the same env
-through an env-server worker pool, the path prime-rl trains through. Placement coverage (harness x harness runtime x tool
-server runtime) is PAIRWISE, not a full cross product: each test carries a curated list of
-combinations (in test_e2e.py) that hits every axis value and the cross-boundary pairs with
-distinct networking. The full cross bought flake exposure and CI minutes, not coverage — add
-a combination to a test's list when it exercises a genuinely new reachability pair. The
-placement fixtures below are indirect-only: they translate a parametrized value, and using
-one without `indirect=True` fails loudly.
-
-Every combination carries its axes' pytest marks, so subsets select with `-m`:
-
-    uv run pytest tests/v1 -n auto                                # everything (needs remote setup)
-    uv run pytest tests/v1 -n auto -m "not e2e"                   # deterministic CI matrix
-    uv run pytest tests/v1 -n auto -m "e2e and not prime and not modal and not e2b"  # live CI job
-    uv run pytest tests/v1 -n auto -m docker                      # any case touching the docker runtime
-    uv run pytest tests/v1 -n auto -m bash                        # only the bash harness
-    uv run pytest tests/v1 -n auto -m prime                       # only prime (real sandboxes; local)
-    uv run pytest tests/v1 -n auto -m modal                       # only modal (needs local setup)
-    uv run pytest tests/v1 -n auto -m e2b                         # only e2b (real sandboxes; local)
-
-Marks: runtimes `subprocess` / `docker` / `podman` / `apptainer` / `prime` / `modal` / `e2b`, placement `colocated`,
-harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `openclaw` / `codex` /
-`claude_code` / `hermes_agent`.
-A mark is applied per axis, so it selects every case touching that value on ANY axis; for one exact
-combination use `-k` on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`).
-prime/modal/e2b provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only.
-CI runs deterministic tests across the Python matrix and the remaining live E2Es once.
+"""Shared fixtures for the integration suite: REAL eval runs (a live model endpoint, real
+runtimes) with the smallest settings that still exercise the path, asserting on the
+resulting `Trace`(s). They need a model API key (`PRIME_API_KEY`); without one the
+model-driven tests skip.
+
+Each behavior module fixes one task and fans it across the harness x runtime grid in
+`grid.py`, which also says why a cell is skipped. `run` drives the `Env` API in-process,
+the calls the eval CLI makes. The placement fixtures below are indirect-only: they translate a parametrized value, and using one without
+`indirect=True` fails loudly.
 """
 
 import asyncio
@@ -39,27 +14,29 @@ import os
 from pathlib import Path
 
 import pytest
+from grid import agent_seats, runtime_config
 
-import verifiers.v1 as vf
-from verifiers.v1.cli.eval.runner import run_eval
+from verifiers.v1.clients import EvalClientConfig, ModelContext
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.configs.harness import HarnessConfig
-from verifiers.v1.trace import Trace
-from verifiers.v1.utils.loaders import harness_config_type
+from verifiers.v1.episode import Episode
+from verifiers.v1.types import SamplingConfig
+from verifiers.v1.utils.loaders import harness_config_type, load_environment
 
 CI_MODEL = "openai/gpt-5.6-luna"
 
-# Fixture tasksets (echo-v1, echo-agentic-v1) live in tests/v1/fixtures, added to the
-# path via `pythonpath` in pyproject so the loader resolves them by id (no install).
+# Fixture tasksets (echo-v1, echo-agentic-v1, ...) live in tests/integration/fixtures,
+# added to the path via `pythonpath` in pyproject so the loader resolves them by id.
 
-# The placement fixtures translate one parametrized value each; the combinations live on
-# the tests (`indirect=True`), so coverage is a visible, curated list — never an implicit
-# cross product.
+LIVE_FIXTURES = {"run", "live_ctx"}
 
 
 @pytest.fixture
-def harness_runtime(request) -> str | dict:
-    return request.param
+def harness_runtime(request) -> dict:
+    """The runtime config placing the harness: a config built by the grid, or a
+    runtime type from a runtime-only grid."""
+    param = request.param
+    return dict(param) if isinstance(param, dict) else runtime_config(param)
 
 
 @pytest.fixture
@@ -68,15 +45,13 @@ def tool_runtime(request) -> dict:
     runtime) or its own runtime, by type."""
     if request.param == "colocated":
         return {"colocated": True}
-    if request.param == "docker":
-        return {"runtime": {"type": "docker", "allow": ["*"]}}
     return {"runtime": {"type": request.param}}
 
 
 # Built-in harnesses are bundled in the `harnesses` package; the agent CLIs (`rlm` /
 # `kimi-code` / `openclaw` / `codex` / `claude-code` / `hermes-agent`) install their
 # dependencies at rollout.
-# `compact` (an example harness) and `terminus-2` (drives the host tmux) are excluded from e2e.
+# `compact` (an example harness) and `terminus-2` (drives the host tmux) are outside the grid.
 @pytest.fixture
 def harness(request) -> HarnessConfig:
     config = {"id": request.param} if isinstance(request.param, str) else request.param
@@ -97,33 +72,29 @@ def pytest_configure(config) -> None:
 
 
 def pytest_collection_modifyitems(config, items) -> None:
-    """Skip the live-model tests (marked `e2e`) when no model endpoint is configured, so the
-    rest of the suite (e.g. config parsing) still runs in a keyless environment."""
+    """Skip the model-driven tests when no model endpoint is configured; the rest
+    (scripted harness behaviors, the examples' own key check) decide for themselves."""
     if os.environ.get("PRIME_API_KEY"):
         return
     skip = pytest.mark.skip(reason="needs PRIME_API_KEY")
     for item in items:
-        if "e2e" in item.keywords:
+        if LIVE_FIXTURES & set(getattr(item, "fixturenames", ())):
             item.add_marker(skip)
 
 
-def _configure_prime_runtimes(config: dict) -> None:
-    """Configure every prime runtime config (nested — harness / tool): tag a `vf-ci` label
-    for optional cleanup, and pin a region that supports port exposure."""
+def _label_prime_runtimes(config: dict) -> None:
+    """Tag every prime runtime config (nested: harness / tool) with a `vf-ci` label, so
+    leftover sandboxes can be found and cleaned up."""
     if isinstance(config, dict):
         if config.get("type") == "prime":
             config.setdefault("labels", ["vf-ci"])
-            # `us` is required for prime's port exposure, which a tool server hosted in a
-            # sandbox needs to be reachable from outside it.
-            config.setdefault("region", "us")
         for value in config.values():
-            _configure_prime_runtimes(value)
+            _label_prime_runtimes(value)
 
 
 def _eval_config(
     taskset: str,
     *,
-    output_dir: Path,
     harness: str | HarnessConfig | None = "null",
     n: int = 1,
     num_tasks: int = 1,
@@ -136,8 +107,8 @@ def _eval_config(
     reasoning_effort: str | None = None,
     client: dict | None = None,
 ) -> EvalConfig:
-    """Build the smallest `EvalConfig` that still exercises the path, shared by the in-process
-    (`run_v1`) and env-server (`run_v1_server`) fixtures. `taskset_overrides` merges onto the
+    """Build the smallest `EvalConfig` that still exercises the path, used by the `run`
+    fixture. `taskset_overrides` merges onto the
     `{id: ...}` config; `runtime` places the `agent` seat's harness (an agent field, not a
     harness one).
 
@@ -146,7 +117,7 @@ def _eval_config(
     seat and a multi-agent test pins its seats through `env` role fields instead."""
     taskset_cfg = {"id": taskset, **(taskset_overrides or {})}
     env_cfg = dict(env or {})
-    _configure_prime_runtimes(taskset_cfg)
+    _label_prime_runtimes(taskset_cfg)
     if harness:
         env_cfg.setdefault("agent", {})["harness"] = (
             harness_config_type(harness)(id=harness)
@@ -155,7 +126,7 @@ def _eval_config(
         )
     if runtime:
         runtime_cfg = dict(runtime)
-        _configure_prime_runtimes(runtime_cfg)
+        _label_prime_runtimes(runtime_cfg)
         env_cfg.setdefault("agent", {})["runtime"] = runtime_cfg
     retries = {
         "max_retries": 0,
@@ -165,15 +136,9 @@ def _eval_config(
         ],
     }
     env_cfg.setdefault("retries", retries)
-    # Per-run caps live on the seats: resolve the env's declared roles and cap
-    # each one (a test's own seat dict wins over the shared defaults).
-    config_cls = vf.env_config_type(taskset, env_cfg.get("id", ""))
-    seats = [
-        name
-        for name, field in config_cls.model_fields.items()
-        if isinstance(field.default, vf.AgentConfig)
-    ]
-    for seat in seats:
+    # Per-run caps live on the seats: cap each role the env declares (a test's own
+    # seat dict wins over the shared defaults).
+    for seat in agent_seats(taskset, env_cfg.get("id", "")):
         seat_cfg = env_cfg.setdefault(seat, {})
         seat_cfg.setdefault("max_turns", max_turns)
         seat_cfg.setdefault("max_output_tokens", max_tokens)
@@ -191,91 +156,42 @@ def _eval_config(
             "max_tokens": max_tokens,
             "reasoning_effort": reasoning_effort,
         },
-        rich=None,
-        output_dir=output_dir.parent,
-        run={"dir": output_dir.name},
         model=CI_MODEL,
-        push=False,
         **({"client": client} if client else {}),
     )
 
 
 @pytest.fixture
-def run_v1():
-    """Run a v1 taskset end-to-end in-process (`run_eval`) and return its traces."""
+def run():
+    """Run a taskset's episodes in-process through the `Env` API: the calls `vf-eval`
+    makes (`serving`, `slots`, `run_slot`, retries included), without its run
+    directory, platform push, or dashboard. The Examples job covers that CLI layer."""
 
-    async def _run(taskset: str, **kwargs) -> list[Trace]:
+    async def _run(taskset: str, **kwargs) -> list[Episode]:
         config = _eval_config(taskset, **kwargs)
-        records = await run_eval(config)
-        # The runner answers durability envelopes; the tests assert on traces.
-        return [t for r in records for t in r.traces]
-
-    return _run
-
-
-@pytest.fixture
-def run_v1_server():
-    """Run a v1 taskset through an env-server worker pool — the path prime-rl trains
-    through. Spawns the broker + a worker, so it's the only fixture that exercises
-    serving resources (shared tool servers, interception pool) being stood up by the
-    *server* rather than the in-process runner. Pinned to a single static worker for
-    determinism."""
-    import multiprocessing as mp
-
-    from verifiers.v1.serve import EnvClient, env_config_data, serve_env
-    from verifiers.v1.utils.loaders import load_taskset
-
-    async def _run(taskset: str, **kwargs) -> list[Trace]:
-        config = _eval_config(taskset, **kwargs)
-        tasks = list(load_taskset(config.env.taskset).select(config.select))
-        mpctx = mp.get_context("spawn")
-        address_queue: mp.Queue = mpctx.Queue()
-        proc = mpctx.Process(
-            target=serve_env,
-            kwargs={
-                "max_workers": 1,
-                "elastic": False,
-                "address": "tcp://127.0.0.1:0",
-                "address_queue": address_queue,
-                "config_data": env_config_data(config.env),
-                "max_concurrent": config.max_concurrent,
-            },
+        env = load_environment(config.env)
+        tasks = list(env.taskset.select(config.select))
+        ctx = ModelContext(
+            client=config.client, model=config.model, sampling=config.sampling
         )
-        proc.start()
-        try:
-            address = await asyncio.to_thread(address_queue.get, timeout=600)
-            client = EnvClient(address=address)
-            try:
-                await client.wait_for_server_startup(timeout=600)
-                episodes = await asyncio.gather(
+        async with env.serving():
+            return list(
+                await asyncio.gather(
                     *(
-                        client.run(
-                            client=config.client,
-                            model=config.model,
-                            sampling=config.sampling,
-                            task_data=task.data.model_dump(mode="json"),
-                        )
+                        env.run_slot(slot, ctx)
                         for task in tasks
-                        for _ in range(config.num_rollouts)
+                        for slot in env.slots(task, config.num_rollouts)
                     )
                 )
-            finally:
-                await client.close()
-        finally:
-            proc.terminate()
-            await asyncio.to_thread(proc.join, 10)
-        return [t for e in episodes for t in e.traces]
+            )
 
     return _run
 
 
 @pytest.fixture
 async def live_ctx():
-    """The e2e `ModelContext` (default model + endpoint config, provider-default sampling)
-    for driving `Agent` directly — the agent-surface counterpart of `run_v1`."""
-    from verifiers.v1.clients import EvalClientConfig, ModelContext
-    from verifiers.v1.types import SamplingConfig
-
+    """The live `ModelContext` (default model + endpoint config, provider-default sampling)
+    for driving `Agent` directly — the agent-surface counterpart of `run`."""
     # Endpoint config only — each rollout builds and closes its own client.
     yield ModelContext(
         model=CI_MODEL,

{verifiers-0.4.1.dev3/tests/v1 → verifiers-0.4.1.dev4/tests/integration}/fixtures/agent_user_harness_v1.py

@@ -2,7 +2,7 @@
 
 Root setup creates the task's `agent_user` (images normally ship one) and hands it the
 workdir; the harness then installs, launches as that user, and answers a one-line prompt.
-With the e2e suite's scripted upstream answering, the run is deterministic: it fails only if
+With the integration suite's scripted upstream answering, the run is deterministic: it fails only if
 the harness can't set up or run without root.
 """
 

{verifiers-0.4.1.dev3/tests/v1 → verifiers-0.4.1.dev4/tests/integration}/fixtures/duet_v1.py

@@ -1,6 +1,6 @@
 """duet: the smallest multi-agent environment — two roles echo the same phrase.
 
-The multi-agent fixture for the v1 e2e suite (resolved by id `duet-v1`): an
+The multi-agent fixture for the v1 integration suite (resolved by id `duet-v1`): an
 `Env` subclass exported alongside its taskset, with two roles ("a", a trainable
 seat on the run's model; "b", pinned untrainable), a `run()` that fans both out on
 the task, and a `finalize()` that records a sibling-dependent signal. One episode

{verifiers-0.4.1.dev3/tests/v1 → verifiers-0.4.1.dev4/tests/integration}/fixtures/echo_agentic_v1.py

@@ -3,7 +3,7 @@
 Like harbor's hello-world but with no Dockerfile to build — it runs on the runtime's default
 image, so the whole agentic loop works end to end: the model issues bash commands and the
 reward reads the file back out of the runtime (`runtime.read`, runtime-opaque like harbor's
-verifier). The minimal reward-1 agentic task for the e2e suite. Needs an agentic (shell) harness,
+verifier). The minimal reward-1 agentic task for the integration suite. Needs an agentic (shell) harness,
 e.g. `bash`.
 """
 

{verifiers-0.4.1.dev3/tests/v1 → verifiers-0.4.1.dev4/tests/integration}/fixtures/echo_tool_v1.py

@@ -1,6 +1,6 @@
 """echo (v1, MCP tool): retrieve a stamped echo from a `vf.Toolset`, then report it.
 
-The v1 tool fixture for the e2e matrix. The task constructs an `EchoToolset`
+The v1 tool fixture for the integration grid. The task constructs an `EchoToolset`
 (`vf.Toolset`) in `Task.toolsets`, with one `@vf.tool` method whose placement is
 CLI-tunable (`--taskset.task.tools.colocated`, `--taskset.task.tools.runtime.type`):
 it runs colocated in the harness's runtime or in its own runtime, and the harness

{verifiers-0.4.1.dev3/tests/v1 → verifiers-0.4.1.dev4/tests/integration}/fixtures/echo_user_sim_v1.py

@@ -9,7 +9,6 @@ closes the interaction (the trace stops as `user_closed`).
 import verifiers.v1 as vf
 
 PHRASES = ["hello world", "goodbye world"]
-SYSTEM = "Repeat the user's message back to them exactly, with no extra words."
 
 
 def _key(text: str) -> str:
@@ -27,12 +26,14 @@ class EchoUserSimData(vf.TaskData):
 class EchoUserSimTask(vf.Task[EchoUserSimData, vf.State, vf.TaskConfig]):
     @vf.reward(weight=1.0)
     async def echoed(self, trace: vf.Trace) -> float:
-        replies = [m.content for m in trace.assistant_messages]
-        phrases = self.data.phrases
-        if len(replies) < len(phrases):
-            return 0.0
-        matched = sum(_key(p) in _key(r or "") for r, p in zip(replies, phrases))
-        return matched / len(phrases)
+        # Match each phrase to a later reply, in order: a harness may spend turns on
+        # tool calls, so replies and phrases need not pair up one to one.
+        replies = iter(_key(m.content or "") for m in trace.assistant_messages)
+        matched = sum(
+            any(_key(phrase) in reply for reply in replies)
+            for phrase in self.data.phrases
+        )
+        return matched / len(self.data.phrases)
 
 
 class EchoUserSimEnv(vf.SingleAgentEnv):
@@ -43,7 +44,11 @@ class EchoUserSimEnv(vf.SingleAgentEnv):
         # first turn opens the conversation.
         async with agents.agent.interaction(task) as interaction:
             for phrase in task.data.phrases:
-                if (await interaction.turn(phrase)).terminated:
+                instruction = (
+                    "Answer directly without using tools or running code. Your "
+                    f"reply must contain this exact phrase: {phrase}"
+                )
+                if (await interaction.turn(instruction)).terminated:
                     break
 
 
@@ -55,7 +60,6 @@ class EchoUserSimTaskset(vf.Taskset[EchoUserSimTask, EchoUserSimConfig]):
                     idx=0,
                     # No prompt: the scripted user opens the conversation.
                     prompt=None,
-                    system_prompt=SYSTEM,
                     phrases=self.config.phrases,
                 )
             )

{verifiers-0.4.1.dev3/tests/v1 → verifiers-0.4.1.dev4/tests/integration}/fixtures/echo_v1.py

@@ -2,16 +2,14 @@
 
 The smallest possible reward-1 taskset — no dataset, no tools, no reasoning required — so an
 end-to-end eval run is fast and deterministic. The reward is 1.0 when the phrase appears in
-the model's reply. It's a fixture taskset for the v1 e2e suite (in tests/v1/fixtures, resolved
-by id `echo-v1` via pytest's `pythonpath`).
+the model's reply. It's a fixture taskset for the integration suite (in tests/integration/fixtures,
+resolved by id `echo-v1` via pytest's `pythonpath`).
 """
 
 from pydantic import Field
 
 import verifiers.v1 as vf
 
-SYSTEM = "Repeat the user's message back to them exactly, with no extra words."
-
 
 def _key(text: str) -> str:
     """Lenient comparison key: lowercase, alphanumerics only."""
@@ -64,7 +62,6 @@ class EchoTaskset(vf.Taskset[EchoTask, EchoConfig]):
                         "Answer directly without using tools or running code. Your reply "
                         f"must contain this exact phrase: {phrase}"
                     ),
-                    system_prompt=SYSTEM,
                     answer=phrase,
                 ),
                 self.config.task,