verifiers 0.1.15.dev442__tar.gz → 0.1.15.dev443__tar.gz

{verifiers-0.1.15.dev442 → verifiers-0.1.15.dev443}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev442
+Version: 0.1.15.dev443
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev442 → verifiers-0.1.15.dev443}/pyproject.toml

@@ -247,7 +247,6 @@ markers = [
     "prime: v1 e2e cases touching the prime runtime (real sandbox + tunnel; local-only, excluded in CI)",
     "modal: v1 e2e cases touching the modal runtime (local-only setup; excluded in CI)",
     "colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
-    "shared: v1 e2e cases with a shared (one-per-eval) tool server",
     "null: v1 e2e cases on the null harness",
     "default: v1 e2e cases on the default harness",
     "rlm: v1 e2e cases on the rlm harness",

{verifiers-0.1.15.dev442 → verifiers-0.1.15.dev443}/tests/v1/conftest.py

@@ -19,7 +19,7 @@ Every matrix value carries a pytest mark, so subsets select with `-m`:
     uv run pytest tests/v1 -n auto -m prime                       # only prime (real sandboxes; local)
     uv run pytest tests/v1 -n auto -m modal                       # only modal (needs local setup)
 
-Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placements `colocated` / `shared`,
+Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placement `colocated`,
 harnesses `null` / `default` / `rlm` / `kimi_code` / `codex`. A mark is applied per axis, so it
 selects every case touching that value on ANY axis; for one exact combination use `-k` on the test
 id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). prime/modal provision real remote
@@ -75,19 +75,16 @@ USER_RUNTIMES = [
 
 @pytest.fixture(params=USER_RUNTIMES)
 def user_runtime(request) -> dict:
-    """A `taskset.user` override placing the user simulator: `colocated` (inside the harness's
+    """A `taskset.task.user` override placing the user simulator: `colocated` (inside the harness's
     runtime) or its own runtime, by type."""
     if request.param == "colocated":
         return {"colocated": True}
     return {"colocated": False, "runtime": {"type": request.param}}
 
 
-# The tool server's runtime: inside the harness's runtime (`colocated`), shared once per eval, or its
-# own runtime per rollout; this fans the tool test across all of them, each carrying its
-# placement/runtime mark (colocated/shared use the host subprocess runtime).
+# Task-scoped and shared tool tests both use these runtime placements.
 TOOL_RUNTIMES = [
     pytest.param("colocated", marks=pytest.mark.colocated, id="with-tool-colocated"),
-    pytest.param("shared", marks=pytest.mark.shared, id="with-tool-shared"),
     pytest.param(
         "subprocess", marks=pytest.mark.subprocess, id="with-tool-in-subprocess"
     ),
@@ -99,12 +96,10 @@ TOOL_RUNTIMES = [
 
 @pytest.fixture(params=TOOL_RUNTIMES)
 def tool_runtime(request) -> dict:
-    """A `taskset.tools` override placing the tool server: `colocated` (inside the harness's
-    runtime), `shared` (one instance for the whole eval), or its own runtime, by type."""
+    """A `taskset.task.tools` override placing the tool server: `colocated` (inside the harness's
+    runtime) or its own runtime, by type."""
     if request.param == "colocated":
         return {"colocated": True}
-    if request.param == "shared":
-        return {"shared": True}
     return {"runtime": {"type": request.param}}
 
 

verifiers-0.1.15.dev443/tests/v1/fixtures/counter_tool_v1.py

@@ -0,0 +1,59 @@
+"""Task-scoped counter tool with typed rollout state."""
+
+import verifiers.v1 as vf
+
+TARGET = 3
+
+
+class CounterState(vf.State):
+    count: int = 0
+
+
+class CounterToolset(vf.Toolset[vf.ToolsetConfig, CounterState]):
+    TOOL_PREFIX = "counter"
+
+    @vf.tool
+    def bump(self) -> str:
+        """Increment the shared counter and return its new value."""
+        self.state.count += 1
+        return f"count={self.state.count}"
+
+
+class CounterTaskConfig(vf.TaskConfig):
+    tools: vf.ToolsetConfig = vf.ToolsetConfig()
+
+
+class CounterTask(vf.Task[vf.TaskData, CounterState, CounterTaskConfig]):
+    tools = (CounterToolset,)
+
+    @vf.reward(weight=1.0)
+    async def counted(self, trace: vf.Trace) -> float:
+        return float(trace.state.count >= 2)
+
+
+class CounterConfig(vf.TasksetConfig):
+    task: CounterTaskConfig = CounterTaskConfig()
+
+
+class CounterTaskset(vf.Taskset[CounterTask, CounterConfig]):
+    def load(self) -> list[CounterTask]:
+        return [
+            CounterTask(
+                vf.TaskData(
+                    idx=0,
+                    prompt=(
+                        f"Call the `counter_bump` tool {TARGET} times, one call per turn — wait "
+                        "for each result before the next. After the last result, reply with "
+                        "<answer>done</answer>."
+                    ),
+                ),
+                self.config.task,
+            )
+        ]
+
+
+__all__ = ["CounterTaskset"]
+
+
+if __name__ == "__main__":
+    CounterToolset.run()

{verifiers-0.1.15.dev442 → verifiers-0.1.15.dev443}/tests/v1/fixtures/echo_agentic_v1.py

@@ -25,39 +25,42 @@ def lenient_match(answer: str, text: str) -> bool:
     return _key(answer) in _key(text)
 
 
-class EchoAgenticTask(vf.Task):
+class EchoAgenticData(vf.TaskData):
     answer: str
     """The phrase the model should write into the file."""
 
 
+class EchoAgenticTask(vf.Task[EchoAgenticData]):
+    @vf.reward(weight=1.0)
+    async def wrote_phrase(self, runtime: Runtime) -> float:
+        try:
+            content = (await runtime.read(TARGET)).decode(errors="replace")
+        except (SandboxError, OSError, ValueError):
+            return 0.0  # the model never wrote the file
+        return float(lenient_match(self.data.answer, content))
+
+
 class EchoAgenticConfig(vf.TasksetConfig):
     phrase: str = "hello world"
 
 
 class EchoAgenticTaskset(vf.Taskset[EchoAgenticTask, EchoAgenticConfig]):
-    def load_tasks(self) -> list[EchoAgenticTask]:
+    def load(self) -> list[EchoAgenticTask]:
         phrase = self.config.phrase
         return [
             EchoAgenticTask(
-                idx=0,
-                prompt=(
-                    f"Use the bash tool to write exactly the text '{phrase}' to a file named "
-                    f"{TARGET} in the current directory, then finish."
+                EchoAgenticData(
+                    idx=0,
+                    prompt=(
+                        f"Use the bash tool to write exactly the text '{phrase}' to a file "
+                        f"named {TARGET} in the current directory, then finish."
+                    ),
+                    system_prompt=SYSTEM,
+                    answer=phrase,
                 ),
-                system_prompt=SYSTEM,
-                answer=phrase,
+                self.config.task,
             )
         ]
 
-    @vf.reward(weight=1.0)
-    async def wrote_phrase(
-        self, task: EchoAgenticTask, trace: vf.Trace, runtime: Runtime
-    ) -> float:
-        try:
-            content = (await runtime.read(TARGET)).decode(errors="replace")
-        except (SandboxError, OSError, ValueError):
-            return 0.0  # the model never wrote the file
-        return float(lenient_match(task.answer, content))
-
 
 __all__ = ["EchoAgenticTaskset"]

{verifiers-0.1.15.dev442 → verifiers-0.1.15.dev443}/tests/v1/fixtures/echo_tool_v1.py

@@ -1,12 +1,12 @@
 """echo (v1, MCP tool): retrieve a stamped echo from a `vf.Toolset`, then report it.
 
-The v1 tool fixture for the e2e matrix. The taskset declares an `EchoToolset` (`vf.Toolset`)
-with one `@vf.tool` method whose placement is CLI-tunable (`--taskset.tools.colocated`,
-`--taskset.tools.shared`, `--taskset.tools.runtime.type`): it runs colocated in the harness's
-runtime, shared once per eval, or in its own runtime, and the harness must reach it wherever it
-lives. The tool stamps its output with a token the prompt never reveals, so the reward is
-1.0 only if the model actually called the tool — trivial when the infra works, impossible when
-it doesn't. The tool is task-agnostic, so it works in `shared` placement too.
+The v1 tool fixture for the e2e matrix. The task declares an `EchoToolset` (`vf.Toolset`)
+with one `@vf.tool` method whose placement is CLI-tunable (`--taskset.task.tools.colocated`,
+`--taskset.task.tools.runtime.type`): it runs colocated in the harness's runtime or in its
+own runtime, and the harness must reach it wherever it lives. The tool stamps its output
+with a token the prompt never reveals, so the reward is 1.0 only if the model actually
+called the tool — trivial when the infra works, impossible when it doesn't. The tool is
+task-agnostic, so it would also serve taskset-scoped (`Taskset.tools`).
 """
 
 import verifiers.v1 as vf
@@ -24,38 +24,40 @@ class EchoToolset(vf.Toolset[vf.ToolsetConfig]):
         return f"{message} [{ECHO_TOKEN}]"
 
 
-class EchoToolTask(vf.Task):
-    pass
+class EchoToolTaskConfig(vf.TaskConfig):
+    tools: vf.ToolsetConfig = vf.ToolsetConfig()
+
+
+class EchoToolTask(vf.Task[vf.TaskData, vf.State, EchoToolTaskConfig]):
+    tools = (EchoToolset,)
+
+    @vf.reward(weight=1.0)
+    async def echoed(self, trace: vf.Trace) -> float:
+        # The stamped token reaches the answer only if the model called the MCP tool.
+        last = trace.assistant_messages[-1].content if trace.assistant_messages else ""
+        last = (last or "").lower()
+        return float(PHRASE in last and ECHO_TOKEN in last)
 
 
 class EchoToolConfig(vf.TasksetConfig):
-    tools: vf.ToolsetConfig = vf.ToolsetConfig()
+    task: EchoToolTaskConfig = EchoToolTaskConfig()
 
 
 class EchoToolTaskset(vf.Taskset[EchoToolTask, EchoToolConfig]):
-    def load_tasks(self) -> list[EchoToolTask]:
+    def load(self) -> list[EchoToolTask]:
         return [
             EchoToolTask(
-                idx=0,
-                prompt=(
-                    f'Call the `echo_back` tool with the message "{PHRASE}", then reply '
-                    "with exactly what it returns inside <answer></answer> tags."
+                vf.TaskData(
+                    idx=0,
+                    prompt=(
+                        f'Call the `echo_back` tool with the message "{PHRASE}", then reply '
+                        "with exactly what it returns inside <answer></answer> tags."
+                    ),
                 ),
+                self.config.task,
             )
         ]
 
-    def tools(self, task: EchoToolTask) -> list[vf.Toolset]:
-        return [EchoToolset(self.config.tools)]
-
-    @vf.reward(weight=1.0)
-    async def echoed(
-        self, task: EchoToolTask, trace: vf.Trace, runtime: vf.Runtime
-    ) -> float:
-        # The stamped token reaches the answer only if the model called the MCP tool.
-        last = trace.assistant_messages[-1].content if trace.assistant_messages else ""
-        last = (last or "").lower()
-        return float(PHRASE in last and ECHO_TOKEN in last)
-
 
 __all__ = ["EchoToolTaskset"]
 

{verifiers-0.1.15.dev442 → verifiers-0.1.15.dev443}/tests/v1/fixtures/echo_user_sim_v1.py

@@ -1,11 +1,4 @@
-"""echo (v1, user simulator): echo a phrase per turn, driven by a `vf.User` simulator.
-
-The v1 user-sim fixture for the e2e matrix. The user simulator is a `vf.User` class whose
-placement is CLI-tunable (`--taskset.user.colocated`, `--taskset.user.runtime.type`): it runs
-either inside the harness's runtime (`colocated`) or in its own runtime (the default), and
-either way the framework drives it and reaches it from wherever the harness runs. A user-sim is
-a task tool, so this needs a tool-supporting harness.
-"""
+"""Multi-turn echo task driven by a `vf.User` simulator."""
 
 import verifiers.v1 as vf
 
@@ -17,13 +10,13 @@ def _key(text: str) -> str:
     return "".join(c for c in text.casefold() if c.isalnum())
 
 
-class EchoUserSimTask(vf.Task):
-    phrases: list[str]
+class EchoUserSimTaskConfig(vf.TaskConfig):
+    user: vf.UserConfig = vf.UserConfig()
 
 
 class EchoUserSimConfig(vf.TasksetConfig):
     phrases: list[str] = PHRASES
-    user: vf.UserConfig = vf.UserConfig()
+    task: EchoUserSimTaskConfig = EchoUserSimTaskConfig()
 
 
 class EchoUserSimState(vf.State):
@@ -45,36 +38,44 @@ class EchoUserSimUser(vf.User[vf.UserConfig, EchoUserSimState]):
         return [{"role": "user", "content": self.phrases[self.turns]}]
 
 
-class EchoUserSimTaskset(
-    vf.Taskset[EchoUserSimTask, EchoUserSimConfig, EchoUserSimState]
+class EchoUserSimData(vf.TaskData):
+    phrases: list[str]
+
+
+class EchoUserSimTask(
+    vf.Task[EchoUserSimData, EchoUserSimState, EchoUserSimTaskConfig]
 ):
+    user = EchoUserSimUser
+
     @vf.stop
     async def user_finished(self, trace: vf.Trace) -> bool:
         return trace.state.user_finished
 
-    def load_tasks(self) -> list[EchoUserSimTask]:
-        return [
-            EchoUserSimTask(
-                idx=0,
-                prompt=self.config.phrases[0],
-                system_prompt=SYSTEM,
-                phrases=self.config.phrases,
-            )
-        ]
-
-    def user(self, task: EchoUserSimTask) -> vf.User:
-        return EchoUserSimUser(self.config.user)
-
     @vf.reward(weight=1.0)
-    async def echoed(self, task: EchoUserSimTask, trace: vf.Trace) -> float:
+    async def echoed(self, trace: vf.Trace) -> float:
         replies = [m.content for m in trace.assistant_messages]
-        phrases = task.phrases
+        phrases = self.data.phrases
         if len(replies) < len(phrases):
             return 0.0
         matched = sum(_key(p) in _key(r or "") for r, p in zip(replies, phrases))
         return matched / len(phrases)
 
 
+class EchoUserSimTaskset(vf.Taskset[EchoUserSimTask, EchoUserSimConfig]):
+    def load(self) -> list[EchoUserSimTask]:
+        return [
+            EchoUserSimTask(
+                EchoUserSimData(
+                    idx=0,
+                    prompt=self.config.phrases[0],
+                    system_prompt=SYSTEM,
+                    phrases=self.config.phrases,
+                ),
+                self.config.task,
+            )
+        ]
+
+
 __all__ = ["EchoUserSimTaskset"]
 
 

{verifiers-0.1.15.dev442 → verifiers-0.1.15.dev443}/tests/v1/fixtures/echo_v1.py

@@ -22,40 +22,45 @@ def lenient_match(answer: str, text: str) -> bool:
     return _key(answer) in _key(text)
 
 
-class EchoTask(vf.Task):
+class EchoData(vf.TaskData):
     answer: str
     """The phrase the model should echo back."""
 
 
+class EchoTask(vf.Task[EchoData]):
+    @vf.stop
+    async def single_turn(self, trace: vf.Trace) -> bool:
+        return trace.num_turns >= 1
+
+    @vf.reward(weight=1.0)
+    async def echoed(self, trace: vf.Trace) -> float:
+        reply = trace.assistant_messages[-1].content if trace.assistant_messages else ""
+        return float(lenient_match(self.data.answer, reply or ""))
+
+
 class EchoConfig(vf.TasksetConfig):
     phrases: list[str] = ["hello world", "ping", "verifiers"]
 
 
 class EchoTaskset(vf.Taskset[EchoTask, EchoConfig]):
-    def load_tasks(self) -> list[EchoTask]:
+    def load(self) -> list[EchoTask]:
         return [
             # Keep coding-agent harnesses on the direct-response path instead of
             # spending this single-turn smoke task on a tool call.
             EchoTask(
-                idx=i,
-                prompt=(
-                    "Do not call tools or execute code. Reply immediately and include "
-                    f"this exact phrase in your final response: {phrase}"
+                EchoData(
+                    idx=i,
+                    prompt=(
+                        "Do not call tools or execute code. Reply immediately and include "
+                        f"this exact phrase in your final response: {phrase}"
+                    ),
+                    system_prompt=SYSTEM,
+                    answer=phrase,
                 ),
-                system_prompt=SYSTEM,
-                answer=phrase,
+                self.config.task,
             )
             for i, phrase in enumerate(self.config.phrases)
         ]
 
-    @vf.stop
-    async def single_turn(self, trace: vf.Trace) -> bool:
-        return trace.num_turns >= 1
-
-    @vf.reward(weight=1.0)
-    async def echoed(self, task: EchoTask, trace: vf.Trace) -> float:
-        reply = trace.assistant_messages[-1].content if trace.assistant_messages else ""
-        return float(lenient_match(task.answer, reply or ""))
-
 
 __all__ = ["EchoTaskset"]

{verifiers-0.1.15.dev442 → verifiers-0.1.15.dev443}/tests/v1/fixtures/tool_response_image_v1.py

@@ -27,24 +27,7 @@ class VisionToolset(vf.Toolset[vf.ToolsetConfig]):
 
 
 class ToolResponseImageTask(vf.Task):
-    pass
-
-
-class ToolResponseImageTaskset(vf.Taskset[ToolResponseImageTask, vf.TasksetConfig]):
-    def load_tasks(self) -> list[ToolResponseImageTask]:
-        return [
-            ToolResponseImageTask(
-                idx=0,
-                prompt=(
-                    "Call the `vision_snapshot` tool exactly once. After it returns, "
-                    "reply with exactly `done`."
-                ),
-                system_prompt=SYSTEM,
-            )
-        ]
-
-    def tools(self, task: ToolResponseImageTask) -> list[vf.Toolset]:
-        return [VisionToolset(vf.ToolsetConfig())]
+    tools = (VisionToolset,)
 
     @vf.reward(weight=1.0)
     async def preserved_image_tool_result(self, trace: vf.Trace) -> float:
@@ -56,6 +39,23 @@ class ToolResponseImageTaskset(vf.Taskset[ToolResponseImageTask, vf.TasksetConfi
         return 0.0
 
 
+class ToolResponseImageTaskset(vf.Taskset[ToolResponseImageTask, vf.TasksetConfig]):
+    def load(self) -> list[ToolResponseImageTask]:
+        return [
+            ToolResponseImageTask(
+                vf.TaskData(
+                    idx=0,
+                    prompt=(
+                        "Call the `vision_snapshot` tool exactly once. After it returns, "
+                        "reply with exactly `done`."
+                    ),
+                    system_prompt=SYSTEM,
+                ),
+                self.config.task,
+            )
+        ]
+
+
 __all__ = ["ToolResponseImageTaskset"]
 
 

{verifiers-0.1.15.dev442 → verifiers-0.1.15.dev443}/tests/v1/test_color_codeword_tasks.py

@@ -12,7 +12,7 @@ def test_color_images_are_rendered_once(monkeypatch) -> None:
     monkeypatch.setattr(taskset, "color_data_url", counted)
     tasks = taskset.ColorCodewordTaskset(
         taskset.ColorCodewordConfig(num_examples=100)
-    ).load_tasks()
+    ).load()
 
     assert len(tasks) == 100
     assert calls == list(taskset.COLOR_MAP)