verifiers 0.2.2.dev17__tar.gz → 0.2.2.dev19__tar.gz

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev17
+Version: 0.2.2.dev19
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev19}/pyproject.toml

@@ -79,6 +79,7 @@ examples = [
     "gsm8k-v1", "glossary-v1", "deepwiki-v1", "wiki-search-v1", "code-golf-v1",
     "proposer-solver-v1", "reverse-text-v1", "color-codeword-v1",
     "wordle-v1", "openenv-wordle-v1", "alphabet-sort-v1", "scratchpad-v1",
+    "kuhn-poker-v1",
 ]
 
 [project.optional-dependencies]
@@ -131,6 +132,7 @@ openenv-wordle-v1 = { path = "environments/openenv_wordle_v1", editable = true }
 color-codeword-v1 = { path = "environments/color_codeword_v1", editable = true }
 alphabet-sort-v1 = { path = "environments/alphabet_sort_v1", editable = true }
 scratchpad-v1 = { path = "environments/scratchpad_v1", editable = true }
+kuhn-poker-v1 = { path = "environments/kuhn_poker_v1", editable = true }
 
 [tool.uv.exclude-newer-package]
 # PrimeIntellect-published on PyPI (trusted publisher)
@@ -229,6 +231,8 @@ markers = [
     "bash: v1 e2e cases on the bash harness",
     "rlm: v1 e2e cases on the rlm harness",
     "kimi_code: v1 e2e cases on the kimi-code harness",
+    "pi: v1 e2e cases on the pi harness",
+    "pool: v1 e2e cases on the pool harness",
     "codex: v1 e2e cases on the codex harness",
     "unit: marks tests as unit tests",
     "asyncio: marks tests as async tests",

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev19}/tests/v1/conftest.py

@@ -6,7 +6,7 @@ not unit tests of individual components. They need a model API key (`PRIME_API_K
 without one the `e2e`-marked tests skip (config parsing still runs).
 
 `run_v1` / `run_v0` mirror the eval CLI's two paths (`run_eval` for a v1 taskset,
-`run_legacy_eval` for a v0 env). Placement coverage (harness x harness runtime x user/tool
+`run_legacy_eval` for a v0 env). Placement coverage (harness x harness runtime x tool
 server runtime) is PAIRWISE, not a full cross product: each test carries a curated list of
 combinations (in test_e2e.py) that hits every axis value and the cross-boundary pairs with
 distinct networking. The full cross bought flake exposure and CI minutes, not coverage — add
@@ -24,11 +24,11 @@ Every combination carries its axes' pytest marks, so subsets select with `-m`:
     uv run pytest tests/v1 -n auto -m modal                       # only modal (needs local setup)
 
 Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placement `colocated`,
-harnesses `null` / `bash` / `rlm` / `kimi_code` / `codex` / `claude_code`. A mark is applied per
-axis, so it selects every case touching that value on ANY axis; for one exact combination use `-k`
-on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). prime/modal provision real
-remote sandboxes (slow, infra-flaky, need setup), so they're local-only — CI runs
-`-m "not prime and not modal"`.
+harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `pool` / `codex` / `claude_code`.
+A mark is applied per axis, so it selects every case touching that value on ANY axis; for one exact
+combination use `-k` on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`).
+prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only
+— CI runs `-m "not prime and not modal"`.
 """
 
 import os
@@ -56,15 +56,6 @@ def harness_runtime(request) -> str:
     return request.param
 
 
-@pytest.fixture
-def user_runtime(request) -> dict:
-    """A `taskset.task.user` override placing the user simulator: `colocated` (inside the harness's
-    runtime) or its own runtime, by type."""
-    if request.param == "colocated":
-        return {"colocated": True}
-    return {"colocated": False, "runtime": {"type": request.param}}
-
-
 @pytest.fixture
 def tool_runtime(request) -> dict:
     """A `taskset.task.tools` override placing the tool server: `colocated` (inside the harness's
@@ -83,9 +74,9 @@ def harness(request) -> str:
 
 
 def pytest_configure(config) -> None:
-    """Self-launching tool/user servers run `python -m <module>` in a fresh subprocess, which
+    """Self-launching tool servers run `python -m <module>` in a fresh subprocess, which
     inherits `PYTHONPATH` but not pytest's in-process `pythonpath`. Put the fixture dir on
-    `PYTHONPATH` so a fixture server module (e.g. `echo_user_sim_v1`, `tool_response_image_v1`)
+    `PYTHONPATH` so a fixture server module (e.g. `tool_response_image_v1`)
     resolves there too — an installed example package (e.g. `glossary_v1`) already would."""
     fixtures = str(Path(__file__).parent / "fixtures")
     existing = os.environ.get("PYTHONPATH", "")
@@ -107,12 +98,12 @@ def pytest_collection_modifyitems(config, items) -> None:
 
 
 def _configure_prime_runtimes(config: dict) -> None:
-    """Configure every prime runtime config (nested — harness / tool / user): tag a `vf-ci` label
+    """Configure every prime runtime config (nested — harness / tool): tag a `vf-ci` label
     for optional cleanup, and pin a region that supports port exposure."""
     if isinstance(config, dict):
         if config.get("type") == "prime":
             config.setdefault("labels", ["vf-ci"])
-            # `us` is required for prime's port exposure, which a tool/user server hosted in a
+            # `us` is required for prime's port exposure, which a tool server hosted in a
             # sandbox needs to be reachable from outside it.
             config.setdefault("region", "us")
         for value in config.values():
@@ -221,6 +212,24 @@ def run_v1_server():
     return _run
 
 
+@pytest.fixture
+async def live_ctx():
+    """A live `ModelContext` (the e2e default model + endpoint, greedy) for driving
+    `Agent` directly — the agent-surface counterpart of `run_v1`."""
+    from verifiers.v1.clients import EvalClientConfig, ModelContext, resolve_client
+    from verifiers.v1.types import SamplingConfig
+
+    client = resolve_client(EvalClientConfig())
+    try:
+        yield ModelContext(
+            model="deepseek/deepseek-v4-flash",
+            client=client,
+            sampling=SamplingConfig(max_tokens=2048, temperature=0),
+        )
+    finally:
+        await client.close()
+
+
 @pytest.fixture
 def run_v0():
     """Run a legacy v0 env through the v1 bridge (the eval CLI's `--id` path)."""

verifiers-0.2.2.dev19/tests/v1/fixtures/echo_acp_resume_v1.py

@@ -0,0 +1,100 @@
+"""Two-segment ACP continuation with an MCP call after resume."""
+
+import verifiers.v1 as vf
+
+CODEWORD = "violet-cascade-731"
+TOOL_STAMP = "resume-ok-9d2"
+
+
+def _key(text: str) -> str:
+    return "".join(character for character in text.casefold() if character.isalnum())
+
+
+class ResumeToolset(vf.Toolset[vf.ToolsetConfig]):
+    TOOL_PREFIX = "resume"
+
+    @vf.tool
+    def recall(self, codeword: str) -> str:
+        """Return the supplied codeword with a private verification stamp."""
+        return f"{codeword} [{TOOL_STAMP}]"
+
+
+class ACPResumeTaskConfig(vf.TaskConfig):
+    tools: vf.ToolsetConfig = vf.ToolsetConfig(colocated=True)
+
+
+class ACPResumeConfig(vf.TasksetConfig):
+    task: ACPResumeTaskConfig = ACPResumeTaskConfig()
+
+
+class ACPResumeTask(vf.Task[vf.TaskData, vf.State, ACPResumeTaskConfig]):
+    tools = (ResumeToolset,)
+
+    @vf.reward(weight=1.0)
+    async def resumed(self, trace: vf.Trace) -> float:
+        segments = trace.info.get("acp_segments", [])
+        if len(segments) != 2:
+            return 0.0
+        first, second = segments
+        resumed_tool_output = "\n".join(second["tool_outputs"])
+        return float(
+            bool(first["last_reply"].strip())
+            and _key(CODEWORD) in _key(second["last_reply"])
+            and _key(TOOL_STAMP) in _key(second["last_reply"])
+            and _key(CODEWORD) in _key(resumed_tool_output)
+            and _key(TOOL_STAMP) in _key(resumed_tool_output)
+            and "tool" in second["roles"]
+        )
+
+
+class ACPResumeEnv(vf.SingleAgentEnv):
+    async def run(self, task, agents):
+        async with agents.agent.interaction(task) as interaction:
+            first = await interaction.turn(
+                f"Remember the codeword {CODEWORD}. Reply with exactly READY."
+            )
+            segments = [first]
+            if not first.terminated:
+                segments.append(
+                    await interaction.turn(
+                        "Call `resume_recall` with the codeword from my previous "
+                        "message, then reply with exactly the tool result."
+                    )
+                )
+            interaction.trace.info["acp_segments"] = [
+                {
+                    "roles": [message.role for message in segment.messages],
+                    "tool_outputs": [
+                        str(message.content)
+                        for message in segment.messages
+                        if message.role == "tool"
+                    ],
+                    "last_reply": segment.last_reply,
+                    "terminated": segment.terminated,
+                }
+                for segment in segments
+            ]
+
+
+class ACPResumeTaskset(vf.Taskset[ACPResumeTask, ACPResumeConfig]):
+    def load(self) -> list[ACPResumeTask]:
+        return [
+            ACPResumeTask(
+                vf.TaskData(
+                    idx=0,
+                    prompt=None,
+                    system_prompt=(
+                        "Follow each user instruction exactly. Preserve conversational "
+                        "context between turns and use requested tools."
+                    ),
+                ),
+                self.config.task,
+            )
+        ]
+
+
+__all__ = ["ACPResumeTaskset", "ACPResumeEnv"]
+
+
+if __name__ == "__main__":
+    ResumeToolset.run()

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev19}/tests/v1/fixtures/echo_tool_v1.py

@@ -33,10 +33,11 @@ class EchoToolTask(vf.Task[vf.TaskData, vf.State, EchoToolTaskConfig]):
 
     @vf.reward(weight=1.0)
     async def echoed(self, trace: vf.Trace) -> float:
-        # The stamped token reaches the answer only if the model called the MCP tool.
-        last = trace.assistant_messages[-1].content if trace.assistant_messages else ""
-        last = (last or "").lower()
-        return float(PHRASE in last and ECHO_TOKEN in last)
+        # The stamped token surfaces in an ASSISTANT message only if the model called
+        # the MCP tool and relayed its result — wherever in the exchange that happened
+        # (in a conversation the last turn may be a closing pleasantry).
+        replies = ((m.content or "").lower() for m in trace.assistant_messages)
+        return float(any(PHRASE in r and ECHO_TOKEN in r for r in replies))
 
 
 class EchoToolConfig(vf.TasksetConfig):

verifiers-0.2.2.dev19/tests/v1/fixtures/echo_user_sim_v1.py

@@ -0,0 +1,65 @@
+"""Multi-turn echo driven by a scripted user — the single user-sim mechanism.
+
+The env's `run()` scripts the user through an interaction. The task is
+prompt-less, so the first `turn(phrase)` opens the conversation; each later turn
+resumes the harness onto the accreted conversation, and leaving the `async with`
+closes the interaction (the trace stops as `user_closed`).
+"""
+
+import verifiers.v1 as vf
+
+PHRASES = ["hello world", "goodbye world"]
+SYSTEM = "Repeat the user's message back to them exactly, with no extra words."
+
+
+def _key(text: str) -> str:
+    return "".join(c for c in text.casefold() if c.isalnum())
+
+
+class EchoUserSimConfig(vf.TasksetConfig):
+    phrases: list[str] = PHRASES
+
+
+class EchoUserSimData(vf.TaskData):
+    phrases: list[str]
+
+
+class EchoUserSimTask(vf.Task[EchoUserSimData, vf.State, vf.TaskConfig]):
+    @vf.reward(weight=1.0)
+    async def echoed(self, trace: vf.Trace) -> float:
+        replies = [m.content for m in trace.assistant_messages]
+        phrases = self.data.phrases
+        if len(replies) < len(phrases):
+            return 0.0
+        matched = sum(_key(p) in _key(r or "") for r, p in zip(replies, phrases))
+        return matched / len(phrases)
+
+
+class EchoUserSimEnv(vf.SingleAgentEnv):
+    """Scripts the user side: opens with the first phrase, follows with the rest."""
+
+    async def run(self, task, agents):
+        # An interaction scripting the user: the task carries no prompt, so the
+        # first turn opens the conversation.
+        async with agents.agent.interaction(task) as interaction:
+            for phrase in task.data.phrases:
+                if (await interaction.turn(phrase)).terminated:
+                    break
+
+
+class EchoUserSimTaskset(vf.Taskset[EchoUserSimTask, EchoUserSimConfig]):
+    def load(self) -> list[EchoUserSimTask]:
+        return [
+            EchoUserSimTask(
+                EchoUserSimData(
+                    idx=0,
+                    # No prompt: the scripted user opens the conversation.
+                    prompt=None,
+                    system_prompt=SYSTEM,
+                    phrases=self.config.phrases,
+                )
+            )
+        ]
+
+
+__all__ = ["EchoUserSimTaskset", "EchoUserSimEnv"]

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev19}/tests/v1/fixtures/pyproject.toml

@@ -9,13 +9,13 @@ dependencies = ["verifiers"]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
-# Flat fixture modules whose tool/user servers self-launch in a sandbox (docker/prime) and so must
-# be importable there. The others (echo_v1, echo_*_v0) have no server, so they never get installed.
+# Flat fixture modules whose tool servers self-launch in a sandbox (docker/prime) and so must
+# be importable there. The others (echo_v1, echo_user_sim_v1, echo_*_v0) have no server, so they
+# never get installed.
 [tool.hatch.build]
 include = [
     "counter_tool_v1.py",
     "echo_tool_v1.py",
-    "echo_user_sim_v1.py",
     "tool_response_image_v1.py",
     "pyproject.toml",
 ]

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev19}/tests/v1/test_e2e.py

@@ -15,43 +15,52 @@ def _pair(a: str, b: str, id: str, *extra_marks):
     return pytest.param(a, b, marks=[*marks, *extra_marks], id=id)
 
 
-# harness x harness runtime: every harness once, both local runtimes hit, one remote row
-# per provider. codex/claude-code are excluded here (unreliable on a no-op echo chat
+# harness x harness runtime: every harness once, both local runtimes hit (subprocess
+# only carries the in-house loops — the rest NEEDS_CONTAINER), one remote row per
+# provider. codex/claude-code are excluded here (unreliable on a no-op echo chat
 # task) — test_agentic covers them.
 CHAT_PLACEMENTS = [
     _pair("null", "subprocess", "null-harness-in-subprocess"),
     _pair("bash", "docker", "bash-harness-in-docker"),
-    _pair("rlm", "subprocess", "rlm-harness-in-subprocess"),
+    _pair("rlm", "docker", "rlm-harness-in-docker"),
     _pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
     _pair("bash", "prime", "bash-harness-in-prime"),
     _pair("bash", "modal", "bash-harness-in-modal"),
 ]
 
 # harness x harness runtime for the shell task: every coding agent once (null is a chat
-# loop with no shell), both local runtimes hit, one remote row per provider.
+# loop with no shell), both local runtimes hit (subprocess only carries bash), one
+# remote row per provider.
 AGENTIC_PLACEMENTS = [
     _pair("bash", "subprocess", "bash-harness-in-subprocess"),
     _pair("rlm", "docker", "rlm-harness-in-docker"),
-    _pair("kimi-code", "subprocess", "kimi-code-harness-in-subprocess"),
+    _pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
     _pair("codex", "docker", "codex-harness-in-docker"),
-    _pair("claude-code", "subprocess", "claude-code-harness-in-subprocess"),
+    _pair("claude-code", "docker", "claude-code-harness-in-docker"),
     _pair("bash", "prime", "bash-harness-in-prime"),
     _pair("bash", "modal", "bash-harness-in-modal"),
 ]
 
-# harness runtime x user placement: colocated in both local runtimes, each own-runtime
-# across the opposite boundary, modal rows local-only. No prime rows: a user sim in a
-# prime sandbox needs prime port exposure (unreachable from the host here).
-USER_PLACEMENTS = [
-    _pair("subprocess", "colocated", "harness-in-subprocess-with-user-colocated"),
-    _pair("docker", "colocated", "harness-in-docker-with-user-colocated"),
-    _pair("subprocess", "docker", "harness-in-subprocess-with-user-in-docker"),
-    _pair("docker", "subprocess", "harness-in-docker-with-user-in-subprocess"),
-    _pair("modal", "colocated", "harness-in-modal-with-user-colocated"),
-    _pair("subprocess", "modal", "harness-in-subprocess-with-user-in-modal"),
+# The scripted user runs in the eval process itself (no placement axis); the harness
+# runtime is the exchange's only axis.
+USER_RUNTIMES = [
+    pytest.param("subprocess", marks=[_m.subprocess], id="harness-in-subprocess"),
+    pytest.param("docker", marks=[_m.docker], id="harness-in-docker"),
+    pytest.param("prime", marks=[_m.prime], id="harness-in-prime"),
+    pytest.param("modal", marks=[_m.modal], id="harness-in-modal"),
+]
+
+# ACP-backed harnesses: each must preserve an exchange across process relaunches and
+# retain MCP access after resuming. Cover every harness in the local container runtime,
+# plus one remote placement for the sandbox/tunnel boundary.
+ACP_RESUME_PLACEMENTS = [
+    _pair("kimi-code", "docker", "kimi-code-acp-in-docker"),
+    _pair("pi", "docker", "pi-acp-in-docker"),
+    _pair("pool", "docker", "pool-acp-in-docker"),
+    _pair("pool", "prime", "pool-acp-in-prime"),
 ]
 
-# harness runtime x tool placement: as USER_PLACEMENTS plus the two-container case
+# harness runtime x tool placement: every axis value once plus the two-container case
 # (harness and tool in separate docker boxes) and a prime-colocated row (a tool in its
 # OWN prime sandbox needs port exposure; colocated rides the harness's box).
 TOOL_PLACEMENTS = [
@@ -97,6 +106,7 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     )
     assert trace.ok
     assert trace.num_turns == 1
+    assert trace.stop_condition == "agent_completed"
     assert trace.reward == 1.0
     # The seat's resolved identity rides the trace (policy metadata for trainers).
     assert trace.agent is not None and trace.agent.sampling.temperature == 0
@@ -109,25 +119,94 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
 
 
 @pytest.mark.e2e
-@pytest.mark.parametrize("harness_runtime,user_runtime", USER_PLACEMENTS, indirect=True)
-async def test_user(run_v1, harness_runtime, user_runtime, tmp_path):
-    """Multi-turn, driven by a (container-safe) `vf.User` simulator, across the user's
-    placement (`user_runtime`: colocated in the harness's runtime, or its own runtime) x
-    the harness `runtime`. Either way the framework drives the user and must reach it
-    from wherever the harness runs."""
+@pytest.mark.parametrize("harness_runtime", USER_RUNTIMES, indirect=True)
+async def test_user(run_v1, harness_runtime, tmp_path):
+    """Multi-turn, driven by a scripted user — an interaction loop in the env's
+    `run()` — across the harness runtime axis. The task is prompt-less, so one
+    run covers the whole exchange shape: the caller opens (the user speaks first),
+    each later turn resumes the harness onto the conversation, and leaving the loop
+    ends the exchange (`user_closed`). The user runs in the eval process itself, so
+    there is no placement axis."""
     (trace,) = await run_v1(
         "echo-user-sim-v1",
         harness="null",
         harness_overrides={"runtime": {"type": harness_runtime}},
         output_dir=tmp_path,
         max_turns=6,
-        taskset_overrides={"task": {"user": user_runtime}},
     )
     assert trace.ok
     assert trace.num_turns >= 2  # genuinely multi-turn
+    assert trace.stop_condition == "user_closed"  # leaving the interaction ended it
     assert trace.reward == 1.0
 
 
+@pytest.mark.e2e
+async def test_interaction(live_ctx):
+    """Drive an agent turn-by-turn through `agent.interaction()` — the caller IS
+    the run's user. Runs on the tool-less `null` harness: nothing but the exchange
+    itself, yet a real rollout — trace, scoring, and the `user_closed` stop all
+    apply."""
+    import verifiers.v1 as vf
+    from verifiers.v1.harnesses.null import NullHarnessConfig
+
+    agent = vf.make_agent(
+        vf.AgentConfig(
+            harness=NullHarnessConfig(id="null"),
+            model=live_ctx.model,
+            sampling=live_ctx.sampling,
+        ),
+        client=live_ctx.client,
+    )
+    task = vf.Task(
+        vf.TaskData(
+            idx=0,
+            prompt=None,  # the interaction's caller opens the conversation
+            system_prompt="Repeat the user's message back exactly, no extra words.",
+        )
+    )
+    async with agent.interaction(task) as interaction:
+        first = await interaction.turn("hello world")
+        assert isinstance(first, vf.Segment)
+        assert not first.terminated
+        assert [message.role for message in first.messages] == ["assistant"]
+        assert "hello world" in first.last_reply.lower()
+        second = await interaction.turn("goodbye world")
+        assert not second.terminated
+        assert [message.role for message in second.messages] == ["assistant"]
+        assert "goodbye world" in second.last_reply.lower()
+    trace = interaction.trace
+    assert trace is not None and trace.errors == []
+    assert trace.stop_condition == "user_closed"  # closing the interaction ended it
+    assert trace.num_turns == 2
+
+
+@pytest.mark.e2e
+@pytest.mark.parametrize(
+    "harness,harness_runtime", ACP_RESUME_PLACEMENTS, indirect=True
+)
+async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
+    """Each ACP harness preserves context and MCP access across two segments."""
+    (trace,) = await run_v1(
+        "echo-acp-resume-v1",
+        harness=harness,
+        harness_overrides={"runtime": {"type": harness_runtime}},
+        output_dir=tmp_path,
+        max_turns=8,
+        max_tokens=8192,
+        rollout_timeout=600,
+    )
+    assert trace.ok, trace.errors
+    assert trace.stop_condition == "user_closed"
+    assert trace.rewards["resumed"] == 1.0
+    assert trace.tools  # ACP-native tools, or Pi's MCP adapter meta-tool
+    segments = trace.info["acp_segments"]
+    assert len(segments) == 2
+    assert segments[0]["terminated"] is False
+    assert segments[1]["terminated"] is False
+    assert "tool" in segments[1]["roles"]
+    assert segments[1]["tool_outputs"]
+
+
 @pytest.mark.e2e
 @pytest.mark.parametrize("harness_runtime,tool_runtime", TOOL_PLACEMENTS, indirect=True)
 async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
@@ -345,6 +424,102 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
     assert 0.0 <= solver.rewards["judge"] <= 1.0
 
 
+@pytest.mark.e2e
+async def test_env_id_user_sim(run_v1, tmp_path):
+    """The user-sim env over the echo taskset: a modeled user (null harness) opens
+    the conversation from the task's prompt-as-scenario; the assistant's trace is
+    judged by the task's own reward; both sides land agent-stamped on one episode."""
+    traces = await run_v1(
+        "echo-v1",
+        harness=None,  # multi-agent: each seat pins its own
+        env={"id": "user-sim", "assistant": {"harness": {"id": "null"}}},
+        output_dir=tmp_path,
+        max_turns=6,
+        rollout_timeout=300,
+    )
+    assert sorted(t.agent_name for t in traces) == ["assistant", "user"]
+    (assistant,) = [t for t in traces if t.agent_name == "assistant"]
+    (user,) = [t for t in traces if t.agent_name == "user"]
+    assert assistant.ok and user.ok
+    assert user.trainable is False
+    assert user.num_turns >= 1  # the modeled user actually spoke
+    assert assistant.metrics["user_turns"] >= 1
+    # `mask_prompt`: the scenario is hidden from the assistant's harness (the run's
+    # visible data) while the task's own rewards still scored the real row. The
+    # masked view is what persists (provenance is the row's idx); both sides land
+    # as ONE durable episode.
+    assert assistant.task.data.prompt is None
+    assert "echoed" in assistant.rewards
+    from verifiers.v1.cli.output import read_episodes
+    from verifiers.v1.trace import WireTrace
+
+    (record,) = read_episodes(tmp_path, WireTrace)
+    assert {t.agent_name for t in record.traces} == {"assistant", "user"}
+    assert record.id  # both traces are persisted under one durable episode identity
+
+
+@pytest.mark.e2e
+async def test_env_id_user_sim_with_tools(run_v1, tmp_path):
+    """THE tau-bench shape — a tool-using assistant composed with a modeled user.
+    The assistant's MCP tool loop runs entirely inside a harness segment and the
+    user exchange advances between segments, so the two can never race or amputate
+    each other (the failure mode of injecting user turns at the model boundary).
+    Reward 1.0 proves the tool actually ran mid-conversation: its token never
+    appears in any prompt."""
+    traces = await run_v1(
+        "echo-tool-v1",
+        harness=None,  # multi-agent: each seat pins its own
+        env={"id": "user-sim", "assistant": {"harness": {"id": "null"}}},
+        output_dir=tmp_path,
+        max_turns=8,
+        # Reasoning models can spend thousands of tokens on a turn; the default
+        # 2048 truncates the reply mid-relay, cutting the stamp out of the text.
+        max_tokens=8192,
+        rollout_timeout=300,
+    )
+    (assistant,) = [t for t in traces if t.agent_name == "assistant"]
+    (user,) = [t for t in traces if t.agent_name == "user"]
+    assert assistant.ok and user.ok
+    assert assistant.task.data.prompt is None  # the scenario stayed off the wire
+    assert user.num_turns >= 1  # the modeled user actually drove the exchange
+    assert assistant.rewards["echoed"] == 1.0  # the tool ran, mid-conversation
+    # The tool was advertised to the masked chat exactly as to any run.
+    assert assistant.tools is not None
+    assert any(tool.name == "echo_back" for tool in assistant.tools)
+
+
+@pytest.mark.e2e
+async def test_kuhn_poker_self_play(run_v1, tmp_path):
+    """The turn-coupled proof env: one Kuhn poker hand, both seats live interactions
+    of the run's own model (self-play), refereed host-side, paid out zero-sum."""
+    traces = await run_v1(
+        "kuhn-poker-v1",
+        harness=None,  # both seats pin the null harness themselves
+        output_dir=tmp_path,
+        max_turns=8,
+        # The Q decision (the one mixed-strategy spot in Kuhn) can cost a reasoning
+        # model thousands of tokens; the default 2048 truncates to an empty reply AND
+        # exhausts the rollout budget, so the invalid-move retry is never served.
+        max_tokens=8192,
+        rollout_timeout=300,
+    )
+    assert sorted(t.agent_name for t in traces) == ["player0", "player1"]
+    payoffs = {t.agent_name: t.rewards["payoff"] for t in traces}
+    assert payoffs["player0"] + payoffs["player1"] == 0  # zero-sum
+    assert abs(payoffs["player0"]) in (1.0, 2.0)
+    for trace in traces:
+        assert trace.ok
+        assert trace.info["kuhn"]["seat"] in (0, 1)
+    # A played-out hand has both seats speaking. A forfeit (the model never produced
+    # a legal move) still pays out zero-sum, but the hand dies mid-exchange, so only
+    # the seat that acted is guaranteed a turn — a hand where NOBODY spoke means the
+    # exchange machinery never ran at all.
+    if traces[0].info["kuhn"]["forfeited"] is None:
+        assert all(t.num_turns >= 1 for t in traces)
+    else:
+        assert any(t.num_turns >= 1 for t in traces)
+
+
 @pytest.mark.e2e
 async def test_multi_agent_env_server(run_v1_server, tmp_path):
     """The same env through the env-server pool: the worker rebuilds the role-typed

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev19}/tests/v1/test_envs.py

@@ -25,7 +25,10 @@ SKIP_EVAL: set[str] = set()
 # Per-run caps are seat fields; recipe envs name their own seats.
 SEATS: dict[str, tuple[str, ...]] = {
     "code_golf_v1": ("golfer",),
+    "kuhn_poker_v1": ("player0", "player1"),
+    "openenv_wordle_v1": ("player",),
     "proposer_solver_v1": ("proposer", "solver"),
+    "wordle_v1": ("player",),
 }