verifiers 0.4.1.dev42__tar.gz → 0.4.1.dev46__tar.gz

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev42
+Version: 0.4.1.dev46
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/conftest.py

@@ -29,7 +29,7 @@ CI_REASONING_EFFORT = "low"
 CI_JUDGE = {"model": CI_MODEL, "sampling": {"reasoning_effort": CI_REASONING_EFFORT}}
 CI_DECISION_JUDGE = {"model": "cloudflare/clef"}
 
-# Fixture tasksets (echo-v1, echo-agentic-v1, ...) live in tests/integration/fixtures,
+# Fixture tasksets (echo, echo-agentic, ...) live in tests/integration/fixtures,
 # added to the path via `pythonpath` in pyproject so the loader resolves them by id.
 
 LIVE_FIXTURES = {"run", "live_ctx"}
@@ -70,7 +70,7 @@ def harness(request) -> HarnessConfig:
 def pytest_configure(config) -> None:
     """Self-launching tool servers run `python -m <module>` in a fresh subprocess, which
     inherits `PYTHONPATH` but not pytest's in-process `pythonpath`. Put the fixture dir on
-    `PYTHONPATH` so a fixture server module (e.g. `tool_response_image_v1`)
+    `PYTHONPATH` so a fixture server module (e.g. `tool_response_image`)
     resolves there too — an installed example package (e.g. `glossary`) already would."""
     config.addinivalue_line(
         "markers", "opencode: integration cells on the OpenCode harness"

verifiers-0.4.1.dev42/tests/integration/fixtures/agent_user_v1.py → verifiers-0.4.1.dev46/tests/integration/fixtures/agent_user.py

@@ -1,4 +1,4 @@
-"""agent-user-v1: a scripted, model-free agent that runs as a non-root user and tries root actions.
+"""agent-user: a scripted, model-free agent that runs as a non-root user and tries root actions.
 
 Root setup creates the task's `agent_user`, plants a root-only secret, and prepares a uv
 script; the workdir stays root-owned, like a typical image WORKDIR. The scripted harness runs

verifiers-0.4.1.dev42/tests/integration/fixtures/agent_user_harness_v1.py → verifiers-0.4.1.dev46/tests/integration/fixtures/agent_user_harness.py

@@ -1,4 +1,4 @@
-"""agent-user-harness-v1: one turn of any harness running as a non-root agent user.
+"""agent-user-harness: one turn of any harness running as a non-root agent user.
 
 Root setup creates the task's `agent_user` (images normally ship one) and hands it the
 workdir; the harness then installs, launches as that user, and answers a one-line prompt.

verifiers-0.4.1.dev42/tests/integration/fixtures/duet_v1.py → verifiers-0.4.1.dev46/tests/integration/fixtures/duet.py

@@ -1,6 +1,6 @@
 """duet: the smallest multi-agent environment — two roles echo the same phrase.
 
-The multi-agent fixture for the v1 integration suite (resolved by id `duet-v1`): an
+The multi-agent fixture for the v1 integration suite (resolved by id `duet`): an
 `Env` subclass exported alongside its taskset, with two roles ("a", a trainable
 seat on the run's model; "b", pinned untrainable), a `run()` that fans both out on
 the task, and a `finalize()` that records a sibling-dependent signal. One episode
@@ -9,7 +9,7 @@ should land one record carrying two role-stamped traces.
 
 import asyncio
 
-from echo_v1 import EchoTaskset, lenient_match
+from echo import EchoTaskset, lenient_match
 
 import verifiers.v1 as vf
 

verifiers-0.4.1.dev42/tests/integration/fixtures/echo_v1.py → verifiers-0.4.1.dev46/tests/integration/fixtures/echo.py

@@ -3,7 +3,7 @@
 The smallest possible reward-1 taskset — no dataset, no tools, no reasoning required — so an
 end-to-end eval run is fast and deterministic. The reward is 1.0 when the phrase appears in
 the model's reply. It's a fixture taskset for the integration suite (in tests/integration/fixtures,
-resolved by id `echo-v1` via pytest's `pythonpath`).
+resolved by id `echo` via pytest's `pythonpath`).
 """
 
 from pydantic import Field

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/fixtures/echo_tool_compose/docker-compose.yaml

@@ -1,4 +1,4 @@
-# echo-tool-v1 with its tool server placed in a Compose service: the server runs in
+# echo-tool with its tool server placed in a Compose service: the server runs in
 # `tools`, beside the workspace (`main`, the runtime's image), and the harness reaches it
 # over the Compose network by the service's name.
 services:

verifiers-0.4.1.dev42/tests/integration/fixtures/network_policy_v1.py → verifiers-0.4.1.dev46/tests/integration/fixtures/network_policy.py

@@ -1,4 +1,4 @@
-"""network-policy-v1: the agent runs an egress probe under the task's network policy.
+"""network-policy: the agent runs an egress probe under the task's network policy.
 
 One task per policy the task data can declare: `open` (the default), `off` (an empty
 allowlist), `block` (blocks `example.com`) and `allow` (allows only `pypi.org`). Setup

verifiers-0.4.1.dev46/tests/integration/fixtures/parallel.py

@@ -0,0 +1,84 @@
+"""parallel: one rollout whose harness sends `n` chat programs at the model at once,
+so their requests overlap on one trace. With `same_prompt`, every copy sends the task
+prompt; otherwise copy `i` appends its index, so the copies diverge after the system
+prompt."""
+
+import asyncio
+
+from pydantic import PositiveInt
+
+import verifiers.v1 as vf
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.harness import Harness
+from verifiers.v1.harnesses.utils.launch import (
+    CHAT_PROGRAM_SOURCE,
+    launch_chat_program,
+)
+from verifiers.v1.runtimes import ProgramResult, Runtime
+
+__all__ = ["ParallelHarness", "ParallelTaskset"]
+
+
+class ParallelHarnessConfig(HarnessConfig):
+    n: PositiveInt = 4
+    same_prompt: bool = True
+
+
+class ParallelHarness(Harness[ParallelHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    EXECUTES_CODE = False
+    NEEDS_CONTAINER = False
+
+    async def setup(self, runtime: Runtime) -> None:
+        await runtime.prepare_uv_script(CHAT_PROGRAM_SOURCE, self.config.resolved_env)
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: vf.Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: vf.TaskData,
+        tool_interception_url: str | None = None,
+    ) -> ProgramResult:
+        system_prompt, prompt = self.resolve_prompt(data)
+
+        def copy_prompt(i: int) -> str:
+            return prompt if self.config.same_prompt else f"{prompt} ({i})"
+
+        results = await asyncio.gather(
+            *(
+                launch_chat_program(
+                    CHAT_PROGRAM_SOURCE,
+                    self.config,
+                    ctx,
+                    trace,
+                    runtime,
+                    endpoint,
+                    secret,
+                    mcp_urls,
+                    system_prompt,
+                    copy_prompt(i),
+                )
+                for i in range(self.config.n)
+            )
+        )
+        return next((r for r in results if r.exit_code != 0), results[0])
+
+
+class ParallelTask(vf.Task):
+    @vf.reward(weight=1.0)
+    async def answered(self, trace: vf.Trace) -> float:
+        return float(trace.num_turns >= 1)
+
+
+class ParallelTaskset(vf.Taskset[ParallelTask, vf.TasksetConfig]):
+    def load(self) -> list[ParallelTask]:
+        return [
+            ParallelTask(
+                vf.TaskData(idx=0, system_prompt="Be brief.", prompt="Reply with OK.")
+            )
+        ]

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/fixtures/pyproject.toml

@@ -10,15 +10,15 @@ requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 # Flat fixture modules whose tool servers self-launch in a sandbox (docker/prime) and so must
-# be importable there. The others (echo_v1, echo_user_sim_v1, echo_*_v0) have no server, so they
+# be importable there. The others (e.g. echo, echo_user_sim) have no server, so they
 # never get installed.
 [tool.hatch.build]
 include = [
-    "agent_user_v1.py",
-    "counter_tool_v1.py",
-    "echo_acp_resume_v1.py",
-    "echo_tool_v1.py",
-    "fetch_tool_v1.py",
-    "tool_response_image_v1.py",
+    "agent_user.py",
+    "counter_tool.py",
+    "echo_acp_resume.py",
+    "echo_tool.py",
+    "fetch_tool.py",
+    "tool_response_image.py",
     "pyproject.toml",
 ]

verifiers-0.4.1.dev42/tests/integration/fixtures/skill_codeword_v1.py → verifiers-0.4.1.dev46/tests/integration/fixtures/skill_codeword.py

@@ -1,4 +1,4 @@
-"""skill-codeword-v1: the answer lives only in a skill the task installs.
+"""skill-codeword: the answer lives only in a skill the task installs.
 
 The task ships a `project-codeword` skill (fixtures/skills) whose SKILL.md holds a
 codeword the prompt never reveals, so reward 1.0 proves the harness installed the skill

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_agent_user.py

@@ -1,6 +1,6 @@
 """The task's non-root agent user: a scripted agent is denied root-only actions
-(agent-user-v1), and every harness installs and runs as that user against a scripted
-upstream (agent-user-harness-v1), on the runtimes that can switch users."""
+(agent-user), and every harness installs and runs as that user against a scripted
+upstream (agent-user-harness), on the runtimes that can switch users."""
 
 import pytest
 from grid import grid, runtimes_grid
@@ -18,8 +18,8 @@ async def test_agent_user_cannot_act_as_root(run, harness_runtime, preinstalled_
     all act as the user. With `preinstalled_tools`, the tool's package was first installed
     by the default user, as on a reused runtime."""
     (episode,) = await run(
-        "agent-user-v1",
-        harness="agent-user-v1",
+        "agent-user",
+        harness="agent-user",
         runtime=harness_runtime,
         taskset_overrides={"preinstalled_tools": preinstalled_tools},
     )
@@ -46,7 +46,7 @@ async def test_agent_user_harness(run, harness, harness_runtime, scripted_model)
     """Each harness installs and runs a turn as a non-root agent user against a scripted
     upstream, so the run fails only if the harness can't set up or run without root."""
     (episode,) = await run(
-        "agent-user-harness-v1",
+        "agent-user-harness",
         harness=harness,
         runtime=harness_runtime,
         client=scripted_model,

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_agentic_judge.py

@@ -7,7 +7,7 @@ async def test_agentic_judge_shared_verifier(run, tmp_path):
     policy = tmp_path / "judge_policy.txt"
     policy.write_text("Check EMPIRICALLY that the agent echoed the word back.")
     (episode,) = await run(
-        "echo-v1",
+        "echo",
         harness=None,
         env={
             "id": "agentic-judge",
@@ -53,7 +53,7 @@ async def test_env_id_agentic_judge(run, tmp_path):
         "Check EMPIRICALLY that answer.txt contains exactly the requested phrase."
     )
     (episode,) = await run(
-        "echo-agentic-v1",
+        "echo-agentic",
         harness=None,
         env={
             "id": "agentic-judge",

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_limits.py

@@ -1,5 +1,5 @@
 """The seat's caps stop a rollout that would run on: the counter task wants three tool
-turns (counter-tool-v1), and each cap ends it after the first with its own stop
+turns (counter-tool), and each cap ends it after the first with its own stop
 condition. The turn cap also runs across the harness x runtime grid, since each program
 handles a refused request in its own way."""
 
@@ -17,7 +17,7 @@ CAPS = [
 @pytest.mark.parametrize("cap,stop_condition", CAPS)
 async def test_cap(run, cap, stop_condition):
     (episode,) = await run(
-        "counter-tool-v1",
+        "counter-tool",
         harness="null",
         runtime={"type": "subprocess"},
         env={"agent": cap},
@@ -31,7 +31,7 @@ async def test_cap(run, cap, stop_condition):
 
 async def test_rollout_timeout(run):
     (episode,) = await run(
-        "counter-tool-v1",
+        "counter-tool",
         harness="null",
         runtime={"type": "subprocess"},
         rollout_timeout=1,
@@ -44,7 +44,7 @@ async def test_rollout_timeout(run):
 @pytest.mark.parametrize("harness,harness_runtime", grid(needs=("mcp",)), indirect=True)
 async def test_max_turns(run, harness, harness_runtime):
     (episode,) = await run(
-        "counter-tool-v1",
+        "counter-tool",
         harness=harness,
         runtime=harness_runtime,
         max_turns=1,

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_multi_agent.py

@@ -1,15 +1,15 @@
-"""Multi-agent episodes: an `Env` with two roles (duet-v1), best-of-n over a plain
+"""Multi-agent episodes: an `Env` with two roles (duet), best-of-n over a plain
 taskset, and turn-coupled self-play (kuhn-poker)."""
 
 import json
 
 
 async def test_multi_agent_env(run):
-    """An `Env` subclass shipped with its taskset (duet-v1): two roles run the
+    """An `Env` subclass shipped with its taskset (duet): two roles run the
     task, `score()` episodes a sibling-dependent metric, and one episode carries
     two role-stamped traces."""
     (episode,) = await run(
-        "duet-v1",
+        "duet",
         harness=None,  # both duet seats pin their own harness
         max_turns=2,
     )
@@ -26,7 +26,7 @@ async def test_multi_agent_env(run):
     # The wire form (what traces.jsonl stores): one episode carrying both traces, each
     # self-stamped on its agent info (completion order — the seats land in either order).
     row = json.loads(episode.model_dump_json(exclude_none=True))
-    assert row["env"] == {"id": "duet-v1"}
+    assert row["env"] == {"id": "duet"}
     by_name = {t["agent"]["name"]: t for t in row["traces"]}
     assert set(by_name) == {"a", "b"}
     assert by_name["a"]["agent"]["trainable"] is True
@@ -37,7 +37,7 @@ async def test_env_id_best_of_n(run):
     """`--env.id` pairs a bundled env with an arbitrary taskset: best-of-n over the
     plain echo taskset — n solver attempts in one episode, sibling-scored."""
     (episode,) = await run(
-        "echo-v1",
+        "echo",
         harness=None,  # a multi-agent env refuses the run-level harness
         env={"id": "best-of-n", "n": 2, "agent": {"harness": {"id": "null"}}},
         max_turns=2,

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_multi_turn.py

@@ -1,4 +1,4 @@
-"""One task, many turns: the model completes a shell task (echo-agentic-v1: write a phrase
+"""One task, many turns: the model completes a shell task (echo-agentic: write a phrase
 to a file, checked in the runtime) across the harness x runtime grid, and on prime over
 every other `transport` a harness config can select. A turn the harness re-sends
 differently, beyond what its `replay_view` declares, forks the trace."""
@@ -21,7 +21,7 @@ MULTI_TURN = [
 @pytest.mark.parametrize("harness,harness_runtime", MULTI_TURN, indirect=True)
 async def test_multi_turn(run, harness, harness_runtime):
     (episode,) = await run(
-        "echo-agentic-v1",
+        "echo-agentic",
         harness=harness,
         runtime=harness_runtime,
         max_turns=10,

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_multi_turn_tool.py

@@ -1,5 +1,5 @@
 """One task, one MCP tool: the model calls a tool whose output carries a stamp the prompt
-never reveals (echo-tool-v1), so reward 1.0 proves the tool ran. Every MCP harness reaches
+never reveals (echo-tool), so reward 1.0 proves the tool ran. Every MCP harness reaches
 it colocated across the harness x runtime grid; the chat loop reaches it across its
 placements (colocated in the harness's box, in a Compose service beside it, or its own
 runtime) on the runtime x runtime grid."""
@@ -14,7 +14,7 @@ import verifiers.v1 as vf
 async def test_tool(run, harness, harness_runtime):
     """Every MCP-capable harness reaches a tool colocated in its runtime."""
     (episode,) = await run(
-        "echo-tool-v1",
+        "echo-tool",
         harness=harness,
         runtime=harness_runtime,
         max_turns=6,
@@ -46,7 +46,7 @@ async def test_tool_placement(run, harness_runtime, tool_runtime):
     on the chat loop. Eval-wide SHARED servers are a different scope (`Taskset.toolsets`),
     covered by `test_shared_tool`."""
     (episode,) = await run(
-        "echo-tool-v1",
+        "echo-tool",
         harness="null",
         runtime=harness_runtime,
         max_turns=6,

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_multimodal_tool.py

@@ -1,11 +1,11 @@
 """A tool that returns an image: MCP image content from a tool result survives into the
-trace (tool-response-image-v1; needs a vision model)."""
+trace (tool-response-image; needs a vision model)."""
 
 
 async def test_multimodal_tool(run):
     """MCP image content from a tool result survives into the v1 trace (needs a vision model)."""
     (episode,) = await run(
-        "tool-response-image-v1",
+        "tool-response-image",
         harness="null",
         runtime={"type": "subprocess"},
         reasoning_effort="none",

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_network_policy.py

@@ -1,6 +1,6 @@
 """A task's network policy holds during execution, and the model is told about it: the
 agent runs an egress probe under each policy the task data declares
-(network-policy-v1), with the agent's `network_notice` on or off, across every harness
+(network-policy), with the agent's `network_notice` on or off, across every harness
 on prime."""
 
 import pytest
@@ -42,7 +42,7 @@ def prompt_text(trace) -> str:
 )
 async def test_network_policy(run, harness, harness_runtime, notice):
     episodes = await run(
-        "network-policy-v1",
+        "network-policy",
         harness=harness,
         runtime=harness_runtime,
         num_tasks=len(NOTICES),

verifiers-0.4.1.dev46/tests/integration/test_parallel.py

@@ -0,0 +1,36 @@
+"""Overlapping requests on one trace: a harness sends `n` chat programs at a scripted
+upstream at once, and the trace graph keeps one branch per copy that shares exactly the
+prefix the copies have in common."""
+
+import pytest
+from parallel import ParallelHarnessConfig
+
+N = 4
+
+
+@pytest.mark.parametrize("harness_runtime", ["subprocess"], indirect=True)
+@pytest.mark.parametrize("same_prompt", [True, False], ids=["same", "distinct"])
+async def test_parallel_requests(run, harness_runtime, scripted_model, same_prompt):
+    (episode,) = await run(
+        "parallel",
+        harness=ParallelHarnessConfig(id="parallel", n=N, same_prompt=same_prompt),
+        runtime=harness_runtime,
+        client=scripted_model,
+    )
+    (trace,) = episode.traces
+    assert trace.ok, [error.message for error in trace.errors]
+    assert trace.num_turns == trace.num_branches == len(trace.calls) == N
+    assert len({call.node for call in trace.calls}) == N
+
+    roles = [node.message.role for node in trace.nodes]
+    assert roles.count("system") == 1
+    users = [i for i, role in enumerate(roles) if role == "user"]
+    replies = [node for node in trace.nodes if node.message.role == "assistant"]
+    # The same prompt shares the system and user nodes; distinct prompts share only
+    # the system node.
+    assert len(users) == (1 if same_prompt else N)
+    assert len(trace.nodes) == 1 + len(users) + N
+    assert {trace.nodes[i].parent for i in users} == {0}
+    assert sorted(reply.parent for reply in replies) == sorted(
+        users * N if same_prompt else users
+    )

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_resume_tool.py

@@ -1,5 +1,5 @@
 """One task, two segments: the harness preserves context and MCP access across a resumed
-exchange (echo-acp-resume-v1), across the harness x runtime grid."""
+exchange (echo-acp-resume), across the harness x runtime grid."""
 
 import pytest
 from grid import grid
@@ -11,7 +11,7 @@ from grid import grid
 async def test_resume_tool(run, harness, harness_runtime):
     """Each harness preserves context and MCP access across two segments."""
     (episode,) = await run(
-        "echo-acp-resume-v1",
+        "echo-acp-resume",
         harness=harness,
         runtime=harness_runtime,
         max_turns=8,

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_rubric_judge.py

@@ -24,7 +24,7 @@ async def test_rubric_judge(run, tmp_path, judge_id):
     if judge_id == "rubric":
         judge["path"] = str(rubric)
     (episode,) = await run(
-        "echo-v1",
+        "echo",
         harness="null",
         runtime={"type": "subprocess"},
         taskset_overrides={"task": {"judges": [judge]}},

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_sampling.py

@@ -13,7 +13,7 @@ SAMPLING = {"temperature": 0.3, "top_p": 0.9, "reasoning_effort": "high"}
 )
 async def test_sampling_overrides(run, harness, harness_runtime):
     (episode,) = await run(
-        "echo-v1",
+        "echo",
         harness=harness,
         runtime=harness_runtime,
         env={"agent": {"sampling": SAMPLING}},

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_single_turn.py

@@ -1,4 +1,4 @@
-"""One task, one reply: the model echoes a phrase back (echo-v1), across the harness x
+"""One task, one reply: the model echoes a phrase back (echo), across the harness x
 runtime grid."""
 
 import pytest
@@ -16,7 +16,7 @@ SINGLE_TURN = skip_harness(
 @pytest.mark.parametrize("harness,harness_runtime", SINGLE_TURN, indirect=True)
 async def test_single_turn(run, harness, harness_runtime):
     (episode,) = await run(
-        "echo-v1",
+        "echo",
         harness=harness,
         runtime=harness_runtime,
         env={"agent": {"sampling": {"temperature": 0.0}}},

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_skills.py

@@ -1,5 +1,5 @@
 """A task's skill reaches the program: the answer lives only in a SKILL.md the task
-installs (skill-codeword-v1), across the harness x runtime grid."""
+installs (skill-codeword), across the harness x runtime grid."""
 
 import pytest
 from grid import grid
@@ -10,7 +10,7 @@ from grid import grid
 )
 async def test_skills(run, harness, harness_runtime):
     (episode,) = await run(
-        "skill-codeword-v1",
+        "skill-codeword",
         harness=harness,
         runtime=harness_runtime,
         max_turns=8,

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_stateful_tool.py

@@ -1,5 +1,5 @@
 """A tool that writes the rollout's state: a `@vf.tool` increments the typed `trace.state`
-each call (counter-tool-v1) and the reward reads it back, across the tool server's
+each call (counter-tool) and the reward reads it back, across the tool server's
 placements on the runtime x runtime grid."""
 
 import pytest
@@ -13,7 +13,7 @@ async def test_tool_state(run, harness_runtime, tool_runtime):
     1.0 proves tool writes reach the host's `trace.state`, colocated and own-runtime (a
     SHARED server's per-rollout state channel is covered by `test_shared_tool`)."""
     (episode,) = await run(
-        "counter-tool-v1",
+        "counter-tool",
         harness="null",
         runtime=harness_runtime,
         max_turns=8,

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_tool_vms.py

@@ -6,7 +6,7 @@ import json
 import os
 
 import pytest
-from fetch_tool_v1 import FetchToolTask, FetchToolTaskConfig
+from fetch_tool import FetchToolTask, FetchToolTaskConfig
 
 import verifiers.v1 as vf
 from verifiers.v1.configs.agent import TimeoutConfig

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_user_echo.py

@@ -1,5 +1,5 @@
 """One task, a scripted user: the user speaks first and the harness resumes onto the
-conversation each turn (echo-user-sim-v1), across the harness x runtime grid; and the
+conversation each turn (echo-user-sim), across the harness x runtime grid; and the
 caller drives an agent turn by turn through `agent.interaction`."""
 
 import pytest
@@ -18,7 +18,7 @@ async def test_user(run, harness, harness_runtime):
     onto the conversation, and leaving the loop ends the exchange (`user_closed`).
     The user runs in the eval process itself, so there is no placement axis."""
     (episode,) = await run(
-        "echo-user-sim-v1",
+        "echo-user-sim",
         harness=harness,
         runtime=harness_runtime,
         max_turns=6,

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/tests/integration/test_user_sim.py

@@ -10,7 +10,7 @@ async def test_env_id_user_sim(run):
     the conversation from the task's prompt-as-scenario; the assistant's trace is
     judged by the task's own reward; both sides land agent-stamped on one episode."""
     (episode,) = await run(
-        "echo-v1",
+        "echo",
         harness=None,  # multi-agent: each seat pins its own
         env={"id": "user-sim", "assistant": {"harness": {"id": "null"}}},
         max_turns=6,
@@ -44,7 +44,7 @@ async def test_env_id_user_sim_with_tools(run):
     Reward 1.0 proves the tool actually ran mid-conversation: its token never
     appears in any prompt."""
     (episode,) = await run(
-        "echo-tool-v1",
+        "echo-tool",
         harness=None,  # multi-agent: each seat pins its own
         env={"id": "user-sim", "assistant": {"harness": {"id": "null"}}},
         max_turns=8,

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/verifiers/v1/clients/train.py

@@ -30,6 +30,7 @@ from verifiers.v1.types import (
     ToolCall,
     TurnTokens,
     Usage,
+    WeightVersionSpan,
 )
 
 logger = logging.getLogger(__name__)
@@ -181,6 +182,9 @@ def response_from_generate(
             sampling_mask=SamplingMask.from_sampling_mask(mask)
             if (mask := result.get("sampling_mask"))
             else None,
+            weight_versions=[WeightVersionSpan(**span) for span in spans]
+            if (spans := result.get("weight_versions"))
+            else None,
         ),
     )
 

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/verifiers/v1/graph.py

@@ -49,6 +49,7 @@ from verifiers.v1.types import (
     TextContentPart,
     Tool,
     ToolMessage,
+    WeightVersionSpan,
 )
 
 if TYPE_CHECKING:
@@ -164,6 +165,10 @@ class MessageNode(BaseModel):
     `ids` stores the flat token ids and `counts` stores each token's row size. Assistant
     nodes only. The arrays serialize as raw-byte `__nd__` dictionaries.
     """
+    weight_versions: list[WeightVersionSpan] | None = None
+    """Engine weight-version spans over this node's sampled tokens (indexes into the
+    completion, as vLLM returns them). Assistant nodes only; None when the engine labels
+    none."""
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
@@ -997,6 +1002,7 @@ def _commit_turn(turn: PendingTurn, response: Response) -> int:
             else [],
             # TurnTokens is discarded after commit, so transfer its logprobs without copying.
             logprobs=tokens.completion_logprobs if tokens else [],
+            weight_versions=tokens.weight_versions if tokens else None,
         )
     )
     assistant_id = len(trace.nodes) - 1

{verifiers-0.4.1.dev42 → verifiers-0.4.1.dev46}/verifiers/v1/types.py

@@ -201,6 +201,14 @@ class RoutedExperts(TypedDict):
     start: int
 
 
+class WeightVersionSpan(BaseModel):
+    """Completion tokens `[start, end)` were sampled by the engine weights labelled `version`."""
+
+    version: str
+    start: int
+    end: int
+
+
 @dataclass
 class SamplingMask:
     """Sampling masks stored as flat int32 `ids` and `counts` arrays.
@@ -276,6 +284,9 @@ class TurnTokens(BaseModel):
     # Transient carrier (excluded): per-completion-token sampling masks,
     # attributed to the assistant node by the turn's `commit`, then dropped.
     sampling_mask: SamplingMask | None = Field(default=None, exclude=True)
+    # Transient carrier (excluded): the engine's weight-version spans over the completion,
+    # attributed to the assistant node by the turn's `commit`, then dropped.
+    weight_versions: list[WeightVersionSpan] | None = Field(default=None, exclude=True)
 
 
 class Response(BaseModel):