verifiers 0.2.2.dev56__tar.gz → 0.2.2.dev58__tar.gz

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev56
+Version: 0.2.2.dev58
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/tests/v1/fixtures/counter_tool_v1.py

@@ -24,7 +24,9 @@ class CounterTaskConfig(vf.TaskConfig):
 
 
 class CounterTask(vf.Task[vf.TaskData, CounterState, CounterTaskConfig]):
-    tools = (CounterToolset,)
+    @classmethod
+    def toolsets(cls, config: CounterTaskConfig) -> list[vf.Toolset]:
+        return [CounterToolset(config.tools)]
 
     @vf.reward(weight=1.0)
     async def counted(self, trace: vf.Trace) -> float:

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/tests/v1/fixtures/echo_acp_resume_v1.py

@@ -28,7 +28,9 @@ class ACPResumeConfig(vf.TasksetConfig):
 
 
 class ACPResumeTask(vf.Task[vf.TaskData, vf.State, ACPResumeTaskConfig]):
-    tools = (ResumeToolset,)
+    @classmethod
+    def toolsets(cls, config: ACPResumeTaskConfig) -> list[vf.Toolset]:
+        return [ResumeToolset(config.tools)]
 
     @vf.reward(weight=1.0)
     async def resumed(self, trace: vf.Trace) -> float:

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/tests/v1/fixtures/echo_tool_v1.py

@@ -1,12 +1,13 @@
 """echo (v1, MCP tool): retrieve a stamped echo from a `vf.Toolset`, then report it.
 
-The v1 tool fixture for the e2e matrix. The task declares an `EchoToolset` (`vf.Toolset`)
-with one `@vf.tool` method whose placement is CLI-tunable (`--taskset.task.tools.colocated`,
-`--taskset.task.tools.runtime.type`): it runs colocated in the harness's runtime or in its
-own runtime, and the harness must reach it wherever it lives. The tool stamps its output
-with a token the prompt never reveals, so the reward is 1.0 only if the model actually
-called the tool — trivial when the infra works, impossible when it doesn't. The tool is
-task-agnostic, so it would also serve taskset-scoped (`Taskset.tools`).
+The v1 tool fixture for the e2e matrix. The task constructs an `EchoToolset`
+(`vf.Toolset`) in `Task.toolsets`, with one `@vf.tool` method whose placement is
+CLI-tunable (`--taskset.task.tools.colocated`, `--taskset.task.tools.runtime.type`):
+it runs colocated in the harness's runtime or in its own runtime, and the harness
+must reach it wherever it lives. The tool stamps its output with a token the prompt
+never reveals, so the reward is 1.0 only if the model actually called the tool —
+trivial when the infra works, impossible when it doesn't. The tool is task-agnostic,
+so it would also serve taskset-scoped (`Taskset.toolsets`).
 """
 
 import verifiers.v1 as vf
@@ -29,7 +30,9 @@ class EchoToolTaskConfig(vf.TaskConfig):
 
 
 class EchoToolTask(vf.Task[vf.TaskData, vf.State, EchoToolTaskConfig]):
-    tools = (EchoToolset,)
+    @classmethod
+    def toolsets(cls, config: EchoToolTaskConfig) -> list[vf.Toolset]:
+        return [EchoToolset(config.tools)]
 
     @vf.reward(weight=1.0)
     async def echoed(self, trace: vf.Trace) -> float:

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/tests/v1/fixtures/tool_response_image_v1.py

@@ -26,8 +26,16 @@ class VisionToolset(vf.Toolset[vf.ToolsetConfig]):
         ]
 
 
-class ToolResponseImageTask(vf.Task):
-    tools = (VisionToolset,)
+class ToolResponseImageTaskConfig(vf.TaskConfig):
+    tools: vf.ToolsetConfig = vf.ToolsetConfig()
+
+
+class ToolResponseImageTask(
+    vf.Task[vf.TaskData, vf.State, ToolResponseImageTaskConfig]
+):
+    @classmethod
+    def toolsets(cls, config: ToolResponseImageTaskConfig) -> list[vf.Toolset]:
+        return [VisionToolset(config.tools)]
 
     @vf.reward(weight=1.0)
     async def preserved_image_tool_result(self, trace: vf.Trace) -> float:
@@ -39,7 +47,13 @@ class ToolResponseImageTask(vf.Task):
         return 0.0
 
 
-class ToolResponseImageTaskset(vf.Taskset[ToolResponseImageTask, vf.TasksetConfig]):
+class ToolResponseImageConfig(vf.TasksetConfig):
+    task: ToolResponseImageTaskConfig = ToolResponseImageTaskConfig()
+
+
+class ToolResponseImageTaskset(
+    vf.Taskset[ToolResponseImageTask, ToolResponseImageConfig]
+):
     def load(self) -> list[ToolResponseImageTask]:
         return [
             ToolResponseImageTask(

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/tests/v1/test_e2e.py

@@ -234,7 +234,7 @@ async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
     the harness's runtime, or its own runtime) x the harness `runtime`. The tool stamps
     its output with a token the prompt never reveals, so reward 1.0 proves the tool was
     reachable from wherever the harness runs and actually ran. Eval-wide SHARED servers
-    are a different scope (`Taskset.tools`) with their own env-server-path coverage:
+    are a different scope (`Taskset.toolsets`) with their own env-server-path coverage:
     `test_shared_tool_isolation`."""
     (trace,) = await run_v1(
         "echo-tool-v1",

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/verifiers/v1/agent.py

@@ -335,7 +335,7 @@ class Agent:
         if self._server is None:
             return None
         if self._server.tunnel is not None or (
-            run_is_local and not shared_tools and not type(task).tools
+            run_is_local and not shared_tools and not task.toolsets(task.config)
         ):
             return self._server
         return None
@@ -517,7 +517,10 @@ class Agent:
             )
             run_is_local = runtime_is_local(runtime_config)
         validate_pairing(
-            self.harness, type(task), runtime_config, shared_tools=shared_tools
+            self.harness,
+            type(task),
+            runtime_config,
+            tools=[*task.toolsets(task.config), *shared_tools.values()],
         )
         # Timeout precedence: agent-level wins, else the task's, else no limit.
         agent_timeout = (

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/verifiers/v1/cli/init.py

@@ -71,7 +71,12 @@ def _taskset_py(pkg: str, prefix: str, *, add_tool: bool) -> str:
     if add_tool:
         local_imports.append(f"from {pkg}.servers.tool import {prefix}Toolset")
         task_config_fields += "\n    tools: vf.ToolsetConfig = vf.ToolsetConfig()"
-        task_decls += f"\n    tools = ({prefix}Toolset,)"
+        task_decls += (
+            "\n\n    @classmethod"
+            f"\n    def toolsets(cls, config: {prefix}TaskConfig) -> list[vf.Toolset]:"
+            f"\n        return [{prefix}Toolset(config.tools)]"
+            "\n"
+        )
     if local_imports:
         imports += "\n\n" + "\n".join(local_imports)
     methods_block = ""
@@ -178,7 +183,7 @@ def _readme(dash: str, pkg: str, *, add_tool: bool, add_harness: bool) -> str:
     ]
     if add_tool:
         layout.append(
-            f"- `{pkg}/servers/tool.py` — a `vf.Toolset` tool server, declared on `Task.tools`."
+            f"- `{pkg}/servers/tool.py` — a `vf.Toolset` tool server, constructed in `Task.toolsets`."
         )
     if add_harness:
         layout.append(

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/verifiers/v1/env.py

@@ -31,7 +31,7 @@ from verifiers.v1.interception import (
 from verifiers.v1.mcp import SharedToolServer, serve_shared
 from verifiers.v1.retries import run_episode_with_retry
 from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local
-from verifiers.v1.task import Task, resolve_server_config
+from verifiers.v1.task import Task
 from verifiers.v1.trace import Error, Trace
 from verifiers.v1.utils.generic import concrete_type
 from verifiers.v1.utils.memory import trim_memory_periodically
@@ -378,24 +378,17 @@ class Env(ABC, Generic[ConfigT]):
 
     def _requires_tunnel(self, shared: dict[str, SharedToolServer]) -> bool:
         """`requires_tunnel` over the consumers known before any rollout: role
-        runtimes, live `shared` servers, and the task class's tool servers;
-        a class overriding `server_config` conservatively counts as remote."""
+        runtimes, live `shared` servers, and the task class's tool servers
+        (`Task.toolsets` is a classmethod, so no task instance is needed)."""
         task_cls = type(self.taskset).task_type()
-        server_classes = [*task_cls.tools]
-        if server_classes and task_cls.server_config is not Task.server_config:
-            return True
-        sole = len({*task_cls.tools}) == 1
         configs = [
-            resolve_server_config(
-                task_cls.__name__, self.taskset.config.task, server_cls, sole=sole
-            )
-            for server_cls in server_classes
+            server.config for server in task_cls.toolsets(self.taskset.config.task)
         ]
         return requires_tunnel(self._runs_local(), configs, shared.values())
 
     @contextlib.asynccontextmanager
     async def shared_tools(self):
-        servers = self.taskset.tool_servers()
+        servers = self.taskset.toolsets(self.taskset.config)
         if not servers:
             yield {}
             return

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/verifiers/v1/mcp/launch.py

@@ -343,7 +343,7 @@ async def serve_shared(toolsets: list[Toolset], harness_is_local: bool = True):
             name = toolset.server_name
             if name in servers:
                 raise ToolsetError(
-                    f"duplicate shared tool server name '{name}' in Taskset.tools — "
+                    f"duplicate shared tool server name '{name}' in Taskset.toolsets — "
                     f"give one a distinct TOOL_PREFIX"
                 )
             if type(toolset).setup_task is not ServerBase.setup_task:
@@ -351,7 +351,7 @@ async def serve_shared(toolsets: list[Toolset], harness_is_local: bool = True):
                     "shared server %r overrides `setup_task`, but `setup_task` is NEVER "
                     "called for a taskset-scoped server (it's built once, task-agnostic) — "
                     "its per-task logic will not run. Move task-agnostic work into `setup`, "
-                    "or declare it on `Task.tools` to run it per-rollout.",
+                    "or construct it in `Task.toolsets` to run it per-rollout.",
                     name,
                 )
             if cfg.url:  # already running remotely; nothing launched, nothing to bridge

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/verifiers/v1/rollout.py

@@ -258,7 +258,7 @@ class RolloutRun:
             ):
                 await self.harness.setup(runtime)
             async with boundary(ToolsetError, "building tool servers"):
-                tool_servers = self.task.tool_servers()
+                toolsets = self.task.toolsets(self.task.config)
             # `base_url` is the interception server's reachable URL for this rollout.
             # The harness reaches the model at `{base_url}/v1`; tool servers reach this
             # rollout's `/state` + `/task` at `base_url` — it's universally reachable
@@ -268,7 +268,7 @@ class RolloutRun:
                     self._interception,
                     runtime,
                     self._session,
-                    tool_servers,
+                    toolsets,
                     self._shared_tools,
                 )
             )
@@ -276,7 +276,7 @@ class RolloutRun:
             self._secret = secret
             self._urls = await self._stack.enter_async_context(
                 serve_tools(
-                    tool_servers,
+                    toolsets,
                     runtime,
                     shared=self._shared_tools,
                     state_secret=secret,

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/verifiers/v1/task.py

@@ -3,8 +3,8 @@
 `TaskData` is the wire half: a frozen pydantic model carrying the data which
 initializes a task instance. Rides on `trace.task.data` in `traces.jsonl`.
 
-`Task` is the behavior half: runtime prep (`setup`/`finalize`), tool declarations
-(`tools`), and scoring (`@reward`/`@metric`) methods.
+`Task` is the behavior half: runtime prep (`setup`/`finalize`), tool servers
+(`toolsets`), and scoring (`@reward`/`@metric`) methods.
 """
 
 from __future__ import annotations
@@ -16,7 +16,6 @@ from collections.abc import Mapping
 from typing import TYPE_CHECKING, ClassVar, Generic, Self
 
 from pydantic import BaseModel, ConfigDict, Field
-from pydantic_config import BaseConfig
 from typing_extensions import TypeVar
 
 from verifiers.v1.artifacts import Artifact
@@ -122,37 +121,10 @@ DataT = TypeVar("DataT", bound=TaskData)
 ConfigT = TypeVar("ConfigT", bound=TaskConfig, default=TaskConfig)
 
 
-def resolve_server_config(
-    owner: str, config: BaseConfig, server_cls: type, *, sole: bool = True
-) -> BaseConfig:
-    """The config a declared server class is built with, resolved off `config`'s
-    fields: exact type match, else — only for a `sole` declared server — the unique
-    isinstance match, else a default-constructed one. Two matching fields raise; the
-    `server_config` methods are the explicit-pairing override. The isinstance
-    fallback is `sole`-gated because with several servers a subclass-typed field
-    could silently pair with the wrong one."""
-    cfg_cls = server_cls._config_cls()
-    values = {name: getattr(config, name) for name in type(config).model_fields}
-    matched = [name for name, v in values.items() if type(v) is cfg_cls]
-    if not matched and sole:
-        matched = [name for name, v in values.items() if isinstance(v, cfg_cls)]
-    if len(matched) > 1:
-        raise TaskError(
-            f"{owner}: ambiguous config for {server_cls.__name__} — config fields "
-            f"{matched} all match {cfg_cls.__name__}; override `server_config` to pair "
-            f"them explicitly"
-        )
-    if matched:
-        return values[matched[0]]
-    return cfg_cls()
-
-
 class Task(Generic[DataT, StateT, ConfigT]):
     NEEDS_CONTAINER: ClassVar[bool] = False
     """Whether the task needs a containerized environment (isolated filesystem, ...)."""
 
-    tools: ClassVar[tuple[type[Toolset], ...]] = ()
-
     def __init__(self, data: DataT, config: ConfigT | None = None) -> None:
         self.data = data
         self.config = config if config is not None else self.config_type()()
@@ -249,16 +221,18 @@ class Task(Generic[DataT, StateT, ConfigT]):
 
         return [load_judge(config) for config in self.config.judges]
 
-    def server_config(self, server_cls: type) -> BaseConfig:
-        return resolve_server_config(
-            type(self).__name__,
-            self.config,
-            server_cls,
-            sole=len(set(type(self).tools)) == 1,
-        )
+    @classmethod
+    def toolsets(cls, config: ConfigT) -> list[Toolset]:
+        """Tool servers launched per rollout, each constructed with its config off
+        `config` — override and wire explicitly:
+
+            @classmethod
+            def toolsets(cls, config: MyTaskConfig) -> list[vf.Toolset]:
+                return [SearchToolset(config.tools)]
 
-    def tool_servers(self) -> list[Toolset]:
-        return [cls(self.server_config(cls)) for cls in type(self).tools]
+        A classmethod so consumers can size placement (tunnels) off the class
+        before any task instance exists."""
+        return []
 
 
 TaskT = TypeVar("TaskT", bound=Task)

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/verifiers/v1/taskset.py

@@ -19,13 +19,12 @@ import itertools
 import random
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Iterable, Iterator
-from typing import TYPE_CHECKING, ClassVar, Generic, Self
+from typing import TYPE_CHECKING, Generic, Self
 
-from pydantic_config import BaseConfig
 from typing_extensions import TypeVar
 
 from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.task import Task, TaskT, resolve_server_config
+from verifiers.v1.task import Task, TaskT
 from verifiers.v1.utils.generic import concrete_type
 from verifiers.v1.utils.sampling import SEED
 
@@ -40,10 +39,6 @@ class Taskset(ABC, Generic[TaskT, TasksetConfigT]):
     """Whether the taskset is infinite (yields tasks forever). Class-declared;
     a `head(n)` view shadows it per instance (bounded by construction)."""
 
-    tools: ClassVar[tuple[type[Toolset], ...]] = ()
-    """Tool servers shared by all tasks in the taskset. The environment will
-    spawn a single, global instance, reused across tasks."""
-
     def __init__(self, config: TasksetConfigT) -> None:
         self.config = config
         override = config.system_prompt
@@ -102,13 +97,14 @@ class Taskset(ABC, Generic[TaskT, TasksetConfigT]):
     def task_type(cls) -> type[Task]:
         return concrete_type(cls, Task, origin=Taskset) or Task
 
-    def server_config(self, server_cls: type) -> BaseConfig:
-        return resolve_server_config(
-            type(self).__name__,
-            self.config,
-            server_cls,
-            sole=len(set(type(self).tools)) == 1,
-        )
-
-    def tool_servers(self) -> list[Toolset]:
-        return [cls(self.server_config(cls)) for cls in type(self).tools]
+    @classmethod
+    def toolsets(cls, config: TasksetConfigT) -> list[Toolset]:
+        """Tool servers shared by all tasks in the taskset (one global instance
+        per server, reused across an environment worker's rollouts), each
+        constructed with its config off `config` — override and wire explicitly:
+
+            @classmethod
+            def toolsets(cls, config: MyConfig) -> list[vf.Toolset]:
+                return [SearchToolset(config.tools)]
+        """
+        return []

{verifiers-0.2.2.dev56 → verifiers-0.2.2.dev58}/verifiers/v1/utils/compile.py

@@ -77,18 +77,19 @@ def validate_pairing(
     task_cls: type[Task],
     runtime_config: RuntimeConfig,
     *,
-    shared_tools: Collection = (),
+    tools: Collection = (),
 ) -> None:
     """Reject an impossible harness/task/runtime combination before any work happens.
-    Every check reads class-level facts, so a failure holds for every row the task
-    class can carry. For `shared_tools` only emptiness matters — declarations and
-    live servers alike mean MCP is in play. (Hosting a user is interaction-scoped,
-    not task-scoped — `Agent.interaction` checks the harness can resume an exchange.)"""
-    if not harness.SUPPORTS_MCP and (task_cls.tools or shared_tools):
+    A failure holds for every row the task class can carry. For `tools` only
+    emptiness matters — task-declared and shared servers alike mean MCP is in play.
+    (Hosting a user is interaction-scoped, not task-scoped — `Agent.interaction`
+    checks the harness can resume an exchange.)"""
+    if not harness.SUPPORTS_MCP and tools:
         raise ValueError(
-            f"Harness {harness.config.id!r} does not support MCP tools, but "
-            f"{task_cls.__name__} exposes tool servers (MCP). Run it with a harness that "
-            f"supports MCP (e.g. --env.agent.harness.id bash), or use tasks without tools."
+            f"Harness {harness.config.id!r} does not support MCP tools, but the run "
+            f"serves some ({task_cls.__name__}'s or the taskset's shared servers). Run "
+            f"it with a harness that supports MCP (e.g. --env.agent.harness.id bash), "
+            f"or use tasks without tools."
         )
     if not harness.SUPPORTS_SKILLS and harness.config.skills:
         raise ValueError(