verifiers 0.2.2.dev55__py3-none-any.whl → 0.2.2.dev57__py3-none-any.whl

verifiers/v1/agent.py

@@ -335,7 +335,7 @@ class Agent:
         if self._server is None:
             return None
         if self._server.tunnel is not None or (
-            run_is_local and not shared_tools and not type(task).tools
+            run_is_local and not shared_tools and not task.toolsets(task.config)
         ):
             return self._server
         return None
@@ -517,7 +517,10 @@ class Agent:
             )
             run_is_local = runtime_is_local(runtime_config)
         validate_pairing(
-            self.harness, type(task), runtime_config, shared_tools=shared_tools
+            self.harness,
+            type(task),
+            runtime_config,
+            tools=[*task.toolsets(task.config), *shared_tools.values()],
         )
         # Timeout precedence: agent-level wins, else the task's, else no limit.
         agent_timeout = (

verifiers/v1/cli/init.py

@@ -71,7 +71,12 @@ def _taskset_py(pkg: str, prefix: str, *, add_tool: bool) -> str:
     if add_tool:
         local_imports.append(f"from {pkg}.servers.tool import {prefix}Toolset")
         task_config_fields += "\n    tools: vf.ToolsetConfig = vf.ToolsetConfig()"
-        task_decls += f"\n    tools = ({prefix}Toolset,)"
+        task_decls += (
+            "\n\n    @classmethod"
+            f"\n    def toolsets(cls, config: {prefix}TaskConfig) -> list[vf.Toolset]:"
+            f"\n        return [{prefix}Toolset(config.tools)]"
+            "\n"
+        )
     if local_imports:
         imports += "\n\n" + "\n".join(local_imports)
     methods_block = ""
@@ -178,7 +183,7 @@ def _readme(dash: str, pkg: str, *, add_tool: bool, add_harness: bool) -> str:
     ]
     if add_tool:
         layout.append(
-            f"- `{pkg}/servers/tool.py` — a `vf.Toolset` tool server, declared on `Task.tools`."
+            f"- `{pkg}/servers/tool.py` — a `vf.Toolset` tool server, constructed in `Task.toolsets`."
         )
     if add_harness:
         layout.append(

verifiers/v1/env.py

@@ -31,7 +31,7 @@ from verifiers.v1.interception import (
 from verifiers.v1.mcp import SharedToolServer, serve_shared
 from verifiers.v1.retries import run_episode_with_retry
 from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local
-from verifiers.v1.task import Task, resolve_server_config
+from verifiers.v1.task import Task
 from verifiers.v1.trace import Error, Trace
 from verifiers.v1.utils.generic import concrete_type
 from verifiers.v1.utils.memory import trim_memory_periodically
@@ -378,24 +378,17 @@ class Env(ABC, Generic[ConfigT]):
 
     def _requires_tunnel(self, shared: dict[str, SharedToolServer]) -> bool:
         """`requires_tunnel` over the consumers known before any rollout: role
-        runtimes, live `shared` servers, and the task class's tool servers;
-        a class overriding `server_config` conservatively counts as remote."""
+        runtimes, live `shared` servers, and the task class's tool servers
+        (their configs read off the declared `tools` fields, no task needed)."""
         task_cls = type(self.taskset).task_type()
-        server_classes = [*task_cls.tools]
-        if server_classes and task_cls.server_config is not Task.server_config:
-            return True
-        sole = len({*task_cls.tools}) == 1
         configs = [
-            resolve_server_config(
-                task_cls.__name__, self.taskset.config.task, server_cls, sole=sole
-            )
-            for server_cls in server_classes
+            server.config for server in task_cls.toolsets(self.taskset.config.task)
         ]
         return requires_tunnel(self._runs_local(), configs, shared.values())
 
     @contextlib.asynccontextmanager
     async def shared_tools(self):
-        servers = self.taskset.tool_servers()
+        servers = self.taskset.toolsets(self.taskset.config)
         if not servers:
             yield {}
             return

verifiers/v1/harnesses/__init__.py

@@ -1,4 +1,8 @@
 from verifiers.v1.harnesses.bash import BashHarness, BashHarnessConfig
+from verifiers.v1.harnesses.browser_use import (
+    BrowserUseHarness,
+    BrowserUseHarnessConfig,
+)
 from verifiers.v1.harnesses.claude_code import (
     ClaudeCodeHarness,
     ClaudeCodeHarnessConfig,
@@ -18,6 +22,8 @@ from verifiers.v1.harnesses.terminus_2 import Terminus2Harness, Terminus2Harness
 __all__ = [
     "BashHarness",
     "BashHarnessConfig",
+    "BrowserUseHarness",
+    "BrowserUseHarnessConfig",
     "ClaudeCodeHarness",
     "ClaudeCodeHarnessConfig",
     "CodexHarness",

verifiers/v1/harnesses/browser_use/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.harnesses.browser_use.harness import (
+    BrowserUseHarness,
+    BrowserUseHarnessConfig,
+)
+
+__all__ = ["BrowserUseHarness", "BrowserUseHarnessConfig"]

verifiers/v1/harnesses/browser_use/harness.py

@@ -0,0 +1,125 @@
+import json
+from pathlib import Path
+from typing import Literal
+
+from pydantic import model_validator
+
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.dialects.chat import message_to_wire
+from verifiers.v1.harness import Harness
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+
+PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
+
+# The helper names and persistence rules the model needs to use the local tool.
+BROWSER_SYSTEM_PROMPT = """You are a browser automation agent. Your `browser` tool executes Python code that controls a real Chromium over CDP through browser-harness; its helpers are pre-imported.
+
+Rules of the tool:
+- Each call runs in a fresh Python process: variables do NOT persist between calls. The browser does persist — tabs, cookies, and page state carry over.
+- Use print() for anything you want to see. Filter in Python before printing; a raw AX tree or DOM dump is huge.
+- The first navigation is new_tab(url), not goto_url(url). After navigating, call wait_for_load().
+
+Core helpers: new_tab(url), goto_url(url), page_info(), js(expression), click_at_xy(x, y), type_text(text), press_key(key), fill_input(selector, text), scroll(x, y, dy), wait_for_load(), wait_for_element(selector), wait_for_network_idle(), list_tabs(), switch_tab(target), close_tab(), ensure_real_tab(), capture_screenshot(path), upload_file(selector, path), and raw cdp("Domain.method", ...).
+
+Finding elements: prefer the accessibility tree over screenshots. cdp("Accessibility.getFullAXTree")["nodes"] has every element's role, name, and backendDOMNodeId — filter in Python before printing. For coordinates: q = cdp("DOM.getBoxModel", backendNodeId=n)["model"]["content"]; x, y = sum(q[0::2])/4, sum(q[1::2])/4, then click_at_xy(x, y) and verify with a targeted js(...) or page_info() check. Fall back to js(...) over the DOM when the AX tree lacks the element."""
+
+
+class BrowserUseHarnessConfig(HarnessConfig):
+    browser: Literal["chromium", "cdp"] = "chromium"
+    """`chromium` launches locally; `cdp` attaches to `cdp_url` without owning it."""
+
+    cdp_url: str | None = None
+    """Rollout-scoped HTTP or WebSocket endpoint used with `browser = "cdp"`."""
+
+    @model_validator(mode="after")
+    def _require_cdp_url_iff_cdp(self) -> "BrowserUseHarnessConfig":
+        if self.browser == "cdp" and not self.cdp_url:
+            raise ValueError(
+                "browser='cdp' needs cdp_url set to a CDP endpoint to attach to"
+            )
+        if self.browser != "cdp" and self.cdp_url:
+            raise ValueError(
+                f"cdp_url is only valid with browser='cdp', not browser={self.browser!r}"
+            )
+        return self
+
+
+class BrowserUseHarness(Harness[BrowserUseHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_RESUME = True
+    # The browser tool executes model-authored Python through a third-party daemon.
+    NEEDS_CONTAINER = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env)
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+    ) -> ProgramResult:
+        system_prompt, prompt = self.resolve_prompt(data)
+        system_prompt = "\n\n".join(
+            p for p in (BROWSER_SYSTEM_PROMPT, system_prompt) if p
+        )
+        # Default resume replays the transcript. If there was no task system
+        # prompt, that transcript already contains this harness prompt.
+        replaying_browser_prompt = (
+            data.system_prompt is None
+            and prompt is not None
+            and not isinstance(prompt, str)
+            and any(
+                message.role == "system" and message.content == BROWSER_SYSTEM_PROMPT
+                for message in prompt
+            )
+        )
+        env = {**self.config.resolved_env}
+        state = f".vf-browser-{trace.id}"
+        args = [
+            f"--base-url={endpoint}",
+            f"--api-key={secret}",
+            f"--model={ctx.model}",
+            f"--browser={self.config.browser}",
+            # A resumed segment reuses this trace's browser and profile.
+            f"--state-dir={state}",
+        ]
+        if not replaying_browser_prompt:
+            args.append(f"--system-prompt={system_prompt}")
+        if self.config.cdp_url:
+            args.append(f"--cdp-url={self.config.cdp_url}")
+        if mcp_urls:
+            # The program connects to the tool servers over HTTP; hand it a standard
+            # `mcpServers` URL config (the `mcp` client itself comes from the uv deps).
+            args.append(
+                "--mcp-config="
+                + json.dumps(
+                    {
+                        "mcpServers": {
+                            name: {"url": url} for name, url in mcp_urls.items()
+                        }
+                    }
+                )
+            )
+        if isinstance(prompt, str):
+            args.append(f"--prompt={prompt}")
+        elif prompt is not None:
+            # Base64 images can exceed exec limits, so hand Messages off through a file.
+            path = f".vf-initial-messages-{trace.id}.json"
+            await runtime.write(
+                path,
+                json.dumps([message_to_wire(m) for m in prompt]).encode(),
+            )
+            args.append(f"--initial-messages-file={path}")
+        program = await runtime.prepare_uv_script(
+            PROGRAM_SOURCE, self.config.resolved_env
+        )
+        return await runtime.run_program([*program, *args], env)

verifiers/v1/harnesses/browser_use/program.py

@@ -0,0 +1,402 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "browser-harness==0.1.8",
+#     "openai",
+#     "mcp>=1.24.0,<2",
+#     "httpx",
+#     "tenacity",
+# ]
+# ///
+"""A chat loop whose one local tool drives a real Chromium over CDP.
+
+Each tool call pipes the model's code to browser-harness, whose daemon holds the
+CDP connection and pre-imports its page helpers. `chromium` launches and owns a
+local browser; `cdp` attaches to an HTTP or WebSocket endpoint it does not own.
+Trace-scoped state preserves tabs across resume and records owned process IDs.
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import time
+import urllib.request
+from contextlib import AsyncExitStack, asynccontextmanager, suppress
+from pathlib import Path
+from typing import Any, cast
+from urllib.parse import urlsplit
+
+import httpx
+from openai import AsyncOpenAI
+from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter
+
+MCP_CALL_ATTEMPTS = 6
+MCP_TIMEOUT = httpx.Timeout(600.0, connect=5.0)  # the OpenAI SDK client defaults
+
+BROWSER_TOOL_TIMEOUT = 3600
+"""Matches the bash harness's command timeout."""
+
+BROWSER_READY_TIMEOUT = 60
+"""The documented Playwright image announced CDP in under one second from a
+cold container; one minute leaves ample runtime startup headroom."""
+
+_DEVTOOLS_LINE = re.compile(r"DevTools listening on ws://127\.0\.0\.1:(\d+)/")
+
+BROWSER_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "browser",
+        "description": (
+            "Execute Python code that drives the browser through browser-harness. "
+            "Helpers are pre-imported; use print() to see values. Each call runs in "
+            "a fresh process: Python variables do not persist between calls, but the "
+            "browser (tabs, cookies, page state) does."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "code": {
+                    "type": "string",
+                    "description": "The Python code to execute.",
+                }
+            },
+            "required": ["code"],
+        },
+    },
+}
+
+
+def find_browser() -> str:
+    """The Chromium binary to launch, where a machine keeps one.
+
+    `PLAYWRIGHT_BROWSERS_PATH` before PATH: an image that installs browsers
+    through Playwright puts nothing on PATH.
+    """
+    for key in ("BH_CHROME_PATH", "CHROME_PATH"):
+        candidate = os.environ.get(key)
+        if candidate and os.access(candidate, os.X_OK):
+            return candidate
+    registry = os.environ.get("PLAYWRIGHT_BROWSERS_PATH")
+    if registry:
+        builds = sorted(Path(registry).glob("chromium-*/chrome-linux*/chrome"))
+        if builds:
+            return str(builds[-1])
+    for name in (
+        "google-chrome-stable",
+        "google-chrome",
+        "chromium",
+        "chromium-browser",
+    ):
+        found = shutil.which(name)
+        if found:
+            return found
+    raise SystemExit(
+        "no Chromium/Chrome found; run on a browser-capable image "
+        "(e.g. mcr.microsoft.com/playwright/python) or set BH_CHROME_PATH"
+    )
+
+
+def _endpoint_alive(endpoint: str) -> bool:
+    try:
+        # browser-harness uses five seconds for this same DevTools HTTP probe.
+        urllib.request.urlopen(f"{endpoint}/json/version", timeout=5).close()
+        return True
+    except OSError:
+        return False
+
+
+def ensure_chromium(state_dir: Path) -> str:
+    """Reuse this trace's live Chromium or launch it."""
+    endpoint_file = state_dir / "cdp-endpoint"
+    if endpoint_file.exists():
+        endpoint = endpoint_file.read_text().strip()
+        if endpoint and _endpoint_alive(endpoint):
+            return endpoint
+
+    profile = state_dir / "profile"
+    profile.mkdir(parents=True, exist_ok=True)
+    log = state_dir / "browser.log"
+    with open(log, "wb") as stderr:
+        browser = subprocess.Popen(
+            [
+                find_browser(),
+                "--remote-debugging-port=0",
+                f"--user-data-dir={profile}",
+                "--headless",
+                "--no-first-run",
+                "--no-default-browser-check",
+                "--disable-dev-shm-usage",
+                "--no-sandbox",
+            ],
+            stdout=subprocess.DEVNULL,
+            stderr=stderr,
+            stdin=subprocess.DEVNULL,
+            start_new_session=True,
+        )
+        deadline = time.monotonic() + BROWSER_READY_TIMEOUT
+        while time.monotonic() < deadline and browser.poll() is None:
+            output = log.read_text(errors="replace")
+            if match := _DEVTOOLS_LINE.search(output):
+                endpoint = f"http://127.0.0.1:{match.group(1)}"
+                endpoint_file.write_text(endpoint)
+                return endpoint
+            time.sleep(0.1)
+        if browser.poll() is None:
+            browser.kill()
+        browser.wait()
+    tail = log.read_text(errors="replace")[-2000:]
+    raise SystemExit(
+        f"browser did not announce a DevTools port within {BROWSER_READY_TIMEOUT}s: "
+        f"{tail or '<no browser output>'}"
+    )
+
+
+def run_browser(code: str, env: dict[str, str]) -> str:
+    """Run model code through the pinned browser-harness CLI."""
+    try:
+        result = subprocess.run(
+            [sys.executable, "-m", "browser_harness.run"],
+            input=code,
+            capture_output=True,
+            text=True,
+            timeout=BROWSER_TOOL_TIMEOUT,
+            env=env,
+            check=False,
+        )
+        return (result.stdout + result.stderr) or "(no output)"
+    except Exception as e:  # noqa: BLE001 - tool failures are returned to the model
+        return f"error: {e}"
+
+
+async def chat(
+    client: AsyncOpenAI,
+    model: str,
+    messages: list[dict[str, Any]],
+    tools: list[dict[str, Any]],
+):
+    completion = await client.chat.completions.create(
+        model=model,
+        messages=cast(Any, messages),
+        tools=cast(Any, tools or None),
+    )
+    return completion.choices[0].message
+
+
+@asynccontextmanager
+async def mcp_session(spec: dict):
+    """One fresh streamable-HTTP session to an MCP server, opened and closed within the caller's
+    task so AnyIO cancellation scopes stay correctly nested. A teardown failure after the body
+    completed is swallowed — the result is already in hand, and closing noise must not fail (or
+    replay) an already-answered call."""
+    from mcp import ClientSession
+    from mcp.client.streamable_http import (
+        create_mcp_http_client,
+        streamable_http_client,
+    )
+
+    stack = AsyncExitStack()
+    try:
+        http_client = await stack.enter_async_context(
+            create_mcp_http_client(
+                headers=spec.get("headers") or None, timeout=MCP_TIMEOUT
+            )
+        )
+        read, write, *_ = await stack.enter_async_context(
+            streamable_http_client(spec["url"], http_client=http_client)
+        )
+        session = await stack.enter_async_context(ClientSession(read, write))
+        await session.initialize()
+        yield session
+    finally:
+        with suppress(Exception):
+            await stack.aclose()
+
+
+async def with_retry(call):
+    """Run one session-scoped operation, retrying transient failures with backoff. A call whose
+    response was lost may be replayed — MCP has no idempotency key, so tools should tolerate
+    at-least-once delivery (a tool that fails reports through its result, not an exception)."""
+    async for attempt in AsyncRetrying(
+        stop=stop_after_attempt(MCP_CALL_ATTEMPTS),
+        wait=wait_exponential_jitter(initial=0.5, max=30),
+        reraise=True,
+    ):
+        with attempt:
+            return await call()
+
+
+async def connect_mcp(
+    config: dict, reserved: set[str]
+) -> tuple[list[dict], dict, dict]:
+    """Enumerate each configured MCP server's tools (a streamable-HTTP `url`); return (tool schemas,
+    dispatch mapping `<server>_<tool>` -> (server name, raw tool name), servers mapping name -> spec).
+    No session is held — a stateless-HTTP server is reconnected per call."""
+    tool_schemas: list[dict] = []
+    dispatch: dict[str, tuple] = {}
+    servers: dict[str, dict] = {}
+    for name, spec in config.get("mcpServers", {}).items():
+        servers[name] = spec
+
+        async def list_tools(spec: dict = spec):
+            async with mcp_session(spec) as session:
+                return (await session.list_tools()).tools
+
+        for tool in await with_retry(list_tools):
+            # A server named "" (TOOL_PREFIX = None) advertises its tools bare.
+            full = f"{name}_{tool.name}" if name else tool.name
+            if full in reserved or full in dispatch:
+                raise ValueError(
+                    f"duplicate tool name {full!r}; keep MCP tool names qualified"
+                )
+            tool_schemas.append(
+                {
+                    "type": "function",
+                    "function": {
+                        "name": full,
+                        "description": tool.description or "",
+                        "parameters": tool.inputSchema,
+                    },
+                }
+            )
+            dispatch[full] = (name, tool.name)
+    return tool_schemas, dispatch, servers
+
+
+def mcp_content_to_chat_content(blocks) -> str | list[dict]:
+    parts = []
+    for block in blocks:
+        if block.type == "text":
+            parts.append({"type": "text", "text": block.text})
+        elif block.type == "image":
+            url = f"data:{block.mimeType};base64,{block.data}"
+            parts.append({"type": "image_url", "image_url": {"url": url}})
+        else:
+            parts.append({"type": "text", "text": str(block)})
+    if not parts:
+        return str(blocks)
+    if all(part["type"] == "text" for part in parts):
+        return "\n".join(str(part["text"]) for part in parts)
+    return parts
+
+
+async def call_mcp(
+    servers: dict, dispatch: dict, name: str, arguments: dict
+) -> str | list[dict]:
+    """Call a tool on a fresh session per attempt — see `with_retry` for the replay semantics.
+    The result is converted outside the retry so a conversion failure fails once."""
+    server_name, raw = dispatch[name]
+
+    async def call():
+        async with mcp_session(servers[server_name]) as session:
+            return await session.call_tool(raw, arguments)
+
+    result = await with_retry(call)
+    return mcp_content_to_chat_content(result.content)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", required=True)
+    parser.add_argument("--api-key", required=True)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--state-dir", required=True)
+    parser.add_argument("--browser", default="chromium")
+    parser.add_argument("--cdp-url", default="")
+    parser.add_argument("--system-prompt", default="")
+    parser.add_argument("--prompt", default="")
+    parser.add_argument("--initial-messages-file", default="")
+    parser.add_argument("--mcp-config", default="")
+    return parser.parse_args()
+
+
+async def main() -> None:
+    args = parse_args()
+    initial = []
+    if args.initial_messages_file:
+        path = Path(args.initial_messages_file)
+        payload = path.read_bytes()
+        path.unlink()
+        initial = json.loads(payload)
+    state_dir = Path(args.state_dir)
+    state_dir.mkdir(parents=True, exist_ok=True)
+    endpoint = (
+        ensure_chromium(state_dir) if args.browser == "chromium" else args.cdp_url
+    )
+    browser_env = {
+        **os.environ,
+        "BH_HOME": str(state_dir / "bh-home"),
+        "BH_TELEMETRY": "0",
+        (
+            "BU_CDP_WS" if urlsplit(endpoint).scheme in {"ws", "wss"} else "BU_CDP_URL"
+        ): endpoint,
+    }
+    client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key)
+    config = json.loads(args.mcp_config or "{}")
+    tools = [BROWSER_TOOL]
+    reserved = {"browser"}
+    mcp_tools, dispatch, servers = (
+        await connect_mcp(config, reserved)
+        if config.get("mcpServers")
+        else ([], {}, {})
+    )
+    tools += mcp_tools
+    messages = (
+        [{"role": "system", "content": args.system_prompt}]
+        if args.system_prompt
+        else []
+    )
+    if initial:
+        messages.extend(initial)
+    elif args.prompt:
+        messages.append({"role": "user", "content": args.prompt})
+    while True:
+        message = await chat(client, args.model, messages, tools)
+        messages.append(message.model_dump(exclude_none=True))
+        if not message.tool_calls:
+            break
+        for call in message.tool_calls:
+            name = call.function.name
+            try:
+                tool_args = json.loads(call.function.arguments or "{}")
+            except json.JSONDecodeError as e:
+                messages.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": call.id,
+                        "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
+                    }
+                )
+                continue
+            # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls
+            # below assume a dict, so reject anything else as a tool error rather than crashing.
+            if not isinstance(tool_args, dict):
+                messages.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": call.id,
+                        "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
+                    }
+                )
+                continue
+            if name in dispatch:
+                content = await call_mcp(servers, dispatch, name, tool_args)
+            elif name == "browser":
+                content = await asyncio.to_thread(
+                    run_browser,
+                    tool_args.get("code", ""),
+                    browser_env,
+                )
+            else:
+                content = f"error: unknown tool {name!r}"
+            messages.append(
+                {"role": "tool", "tool_call_id": call.id, "content": content}
+            )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

verifiers/v1/mcp/launch.py

@@ -343,7 +343,7 @@ async def serve_shared(toolsets: list[Toolset], harness_is_local: bool = True):
             name = toolset.server_name
             if name in servers:
                 raise ToolsetError(
-                    f"duplicate shared tool server name '{name}' in Taskset.tools — "
+                    f"duplicate shared tool server name '{name}' in Taskset.toolsets — "
                     f"give one a distinct TOOL_PREFIX"
                 )
             if type(toolset).setup_task is not ServerBase.setup_task:
@@ -351,7 +351,7 @@ async def serve_shared(toolsets: list[Toolset], harness_is_local: bool = True):
                     "shared server %r overrides `setup_task`, but `setup_task` is NEVER "
                     "called for a taskset-scoped server (it's built once, task-agnostic) — "
                     "its per-task logic will not run. Move task-agnostic work into `setup`, "
-                    "or declare it on `Task.tools` to run it per-rollout.",
+                    "or construct it in `Task.toolsets` to run it per-rollout.",
                     name,
                 )
             if cfg.url:  # already running remotely; nothing launched, nothing to bridge

verifiers/v1/rollout.py

@@ -258,7 +258,7 @@ class RolloutRun:
             ):
                 await self.harness.setup(runtime)
             async with boundary(ToolsetError, "building tool servers"):
-                tool_servers = self.task.tool_servers()
+                toolsets = self.task.toolsets(self.task.config)
             # `base_url` is the interception server's reachable URL for this rollout.
             # The harness reaches the model at `{base_url}/v1`; tool servers reach this
             # rollout's `/state` + `/task` at `base_url` — it's universally reachable
@@ -268,7 +268,7 @@ class RolloutRun:
                     self._interception,
                     runtime,
                     self._session,
-                    tool_servers,
+                    toolsets,
                     self._shared_tools,
                 )
             )
@@ -276,7 +276,7 @@ class RolloutRun:
             self._secret = secret
             self._urls = await self._stack.enter_async_context(
                 serve_tools(
-                    tool_servers,
+                    toolsets,
                     runtime,
                     shared=self._shared_tools,
                     state_secret=secret,

verifiers/v1/task.py

@@ -16,7 +16,6 @@ from collections.abc import Mapping
 from typing import TYPE_CHECKING, ClassVar, Generic, Self
 
 from pydantic import BaseModel, ConfigDict, Field
-from pydantic_config import BaseConfig
 from typing_extensions import TypeVar
 
 from verifiers.v1.artifacts import Artifact
@@ -122,37 +121,10 @@ DataT = TypeVar("DataT", bound=TaskData)
 ConfigT = TypeVar("ConfigT", bound=TaskConfig, default=TaskConfig)
 
 
-def resolve_server_config(
-    owner: str, config: BaseConfig, server_cls: type, *, sole: bool = True
-) -> BaseConfig:
-    """The config a declared server class is built with, resolved off `config`'s
-    fields: exact type match, else — only for a `sole` declared server — the unique
-    isinstance match, else a default-constructed one. Two matching fields raise; the
-    `server_config` methods are the explicit-pairing override. The isinstance
-    fallback is `sole`-gated because with several servers a subclass-typed field
-    could silently pair with the wrong one."""
-    cfg_cls = server_cls._config_cls()
-    values = {name: getattr(config, name) for name in type(config).model_fields}
-    matched = [name for name, v in values.items() if type(v) is cfg_cls]
-    if not matched and sole:
-        matched = [name for name, v in values.items() if isinstance(v, cfg_cls)]
-    if len(matched) > 1:
-        raise TaskError(
-            f"{owner}: ambiguous config for {server_cls.__name__} — config fields "
-            f"{matched} all match {cfg_cls.__name__}; override `server_config` to pair "
-            f"them explicitly"
-        )
-    if matched:
-        return values[matched[0]]
-    return cfg_cls()
-
-
 class Task(Generic[DataT, StateT, ConfigT]):
     NEEDS_CONTAINER: ClassVar[bool] = False
     """Whether the task needs a containerized environment (isolated filesystem, ...)."""
 
-    tools: ClassVar[tuple[type[Toolset], ...]] = ()
-
     def __init__(self, data: DataT, config: ConfigT | None = None) -> None:
         self.data = data
         self.config = config if config is not None else self.config_type()()
@@ -249,16 +221,18 @@ class Task(Generic[DataT, StateT, ConfigT]):
 
         return [load_judge(config) for config in self.config.judges]
 
-    def server_config(self, server_cls: type) -> BaseConfig:
-        return resolve_server_config(
-            type(self).__name__,
-            self.config,
-            server_cls,
-            sole=len(set(type(self).tools)) == 1,
-        )
+    @classmethod
+    def toolsets(cls, config: ConfigT) -> list[Toolset]:
+        """Tool servers launched per rollout, each constructed with its config off
+        `config` — override and wire explicitly:
+
+            @classmethod
+            def toolsets(cls, config: MyTaskConfig) -> list[vf.Toolset]:
+                return [SearchToolset(config.tools)]
 
-    def tool_servers(self) -> list[Toolset]:
-        return [cls(self.server_config(cls)) for cls in type(self).tools]
+        A classmethod so consumers can size placement (tunnels) off the class
+        before any task instance exists."""
+        return []
 
 
 TaskT = TypeVar("TaskT", bound=Task)

verifiers/v1/taskset.py

@@ -19,13 +19,12 @@ import itertools
 import random
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Iterable, Iterator
-from typing import TYPE_CHECKING, ClassVar, Generic, Self
+from typing import TYPE_CHECKING, Generic, Self
 
-from pydantic_config import BaseConfig
 from typing_extensions import TypeVar
 
 from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.task import Task, TaskT, resolve_server_config
+from verifiers.v1.task import Task, TaskT
 from verifiers.v1.utils.generic import concrete_type
 from verifiers.v1.utils.sampling import SEED
 
@@ -40,10 +39,6 @@ class Taskset(ABC, Generic[TaskT, TasksetConfigT]):
     """Whether the taskset is infinite (yields tasks forever). Class-declared;
     a `head(n)` view shadows it per instance (bounded by construction)."""
 
-    tools: ClassVar[tuple[type[Toolset], ...]] = ()
-    """Tool servers shared by all tasks in the taskset. The environment will
-    spawn a single, global instance, reused across tasks."""
-
     def __init__(self, config: TasksetConfigT) -> None:
         self.config = config
         override = config.system_prompt
@@ -102,13 +97,14 @@ class Taskset(ABC, Generic[TaskT, TasksetConfigT]):
     def task_type(cls) -> type[Task]:
         return concrete_type(cls, Task, origin=Taskset) or Task
 
-    def server_config(self, server_cls: type) -> BaseConfig:
-        return resolve_server_config(
-            type(self).__name__,
-            self.config,
-            server_cls,
-            sole=len(set(type(self).tools)) == 1,
-        )
-
-    def tool_servers(self) -> list[Toolset]:
-        return [cls(self.server_config(cls)) for cls in type(self).tools]
+    @classmethod
+    def toolsets(cls, config: TasksetConfigT) -> list[Toolset]:
+        """Tool servers shared by all tasks in the taskset (one global instance
+        per server, reused across an environment worker's rollouts), each
+        constructed with its config off `config` — override and wire explicitly:
+
+            @classmethod
+            def toolsets(cls, config: MyConfig) -> list[vf.Toolset]:
+                return [SearchToolset(config.tools)]
+        """
+        return []

verifiers/v1/utils/compile.py

@@ -77,18 +77,19 @@ def validate_pairing(
     task_cls: type[Task],
     runtime_config: RuntimeConfig,
     *,
-    shared_tools: Collection = (),
+    tools: Collection = (),
 ) -> None:
     """Reject an impossible harness/task/runtime combination before any work happens.
-    Every check reads class-level facts, so a failure holds for every row the task
-    class can carry. For `shared_tools` only emptiness matters — declarations and
-    live servers alike mean MCP is in play. (Hosting a user is interaction-scoped,
-    not task-scoped — `Agent.interaction` checks the harness can resume an exchange.)"""
-    if not harness.SUPPORTS_MCP and (task_cls.tools or shared_tools):
+    A failure holds for every row the task class can carry. For `tools` only
+    emptiness matters — task-declared and shared servers alike mean MCP is in play.
+    (Hosting a user is interaction-scoped, not task-scoped — `Agent.interaction`
+    checks the harness can resume an exchange.)"""
+    if not harness.SUPPORTS_MCP and tools:
         raise ValueError(
-            f"Harness {harness.config.id!r} does not support MCP tools, but "
-            f"{task_cls.__name__} exposes tool servers (MCP). Run it with a harness that "
-            f"supports MCP (e.g. --env.agent.harness.id bash), or use tasks without tools."
+            f"Harness {harness.config.id!r} does not support MCP tools, but the run "
+            f"serves some ({task_cls.__name__}'s or the taskset's shared servers). Run "
+            f"it with a harness that supports MCP (e.g. --env.agent.harness.id bash), "
+            f"or use tasks without tools."
         )
     if not harness.SUPPORTS_SKILLS and harness.config.skills:
         raise ValueError(

{verifiers-0.2.2.dev55.dist-info → verifiers-0.2.2.dev57.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev55
+Version: 0.2.2.dev57
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev55.dist-info → verifiers-0.2.2.dev57.dist-info}/RECORD

@@ -168,10 +168,10 @@ verifiers/utils/tool_utils.py,sha256=gInWZODWQUZN4TyEMuuITyOUm9qlHJxtcvr1zZl_zJs
 verifiers/utils/usage_utils.py,sha256=Mw6WC78r6EsLyRFdP9ICveqptkjizbSkAC86ZH0geVU,3918
 verifiers/utils/version_utils.py,sha256=39gV8-mNF9GspAgKloLo8oy7_MVluMrwGPrntQv2hx4,2656
 verifiers/v1/__init__.py,sha256=Im5qTZy9mH6PRTUi-1V6W7EWKZzJyTWLGKMYALbWz_w,7467
-verifiers/v1/agent.py,sha256=dj4GabQSs6Wzwbph5Rlx-efsVQsSuWV2YI5hfvq5mGA,31990
+verifiers/v1/agent.py,sha256=t5Z_iI6Zyt0ieMDI7pfHUVYIKViuSXF-CI_rIApx3f4,32071
 verifiers/v1/artifacts.py,sha256=5eKOjDSUWwj7GrhxILMHdz2ngEONWMdcTGlBcuGgrLU,5834
 verifiers/v1/decorators.py,sha256=9CcCwgZlRjVGPqT5G1CFJUHP4OIMuBGkUJZkcI6dnbk,3865
-verifiers/v1/env.py,sha256=a7cKeecsm4d1lZPhRJy-m7FFWHrHF2msOdqma7o5I6M,18468
+verifiers/v1/env.py,sha256=wJp9i49_LKVzMFXkwtt_SaRMUJ84i5r1t3QfNvnt4Ks,18178
 verifiers/v1/episode.py,sha256=YtuRTV_PpwDglXFeN6UNg8IswpP1HseFLCYtascGMNI,3218
 verifiers/v1/errors.py,sha256=slZnrtqMo_BgXQV46wJJhV6vvAAnVsuCbVGp2vBVWfs,6888
 verifiers/v1/graph.py,sha256=aBAzh1Ibp3klKNYjvzNyJSbjV8Oi5TJ791dN7o6ex1E,29070
@@ -181,12 +181,12 @@ verifiers/v1/legacy.py,sha256=ISk3Bix9Hy5EjZtyV9QO0LNyDNDklBHVjFokOrw_3U8,22628
 verifiers/v1/loaders.py,sha256=NSIBdNU3JVeyddL-dM4dJt6khCHyjCgzJuFtLrgiwD0,9989
 verifiers/v1/push.py,sha256=MWZmvyj0iyzNIO0xRhlkCXtFF1plq6EJi8rWlvrFtvI,11363
 verifiers/v1/retries.py,sha256=quUGVruUkZx-9WQcFiocMfL_M056voU8gwDqsuLR3f4,5412
-verifiers/v1/rollout.py,sha256=pmF4sgyHDRUUTnOxOWgBsPP8x6L_KFGaeR_dDKziANs,20470
+verifiers/v1/rollout.py,sha256=ehPoN6EnrIg_lvF1-pdf9pS8j47K3z6jDHc66MixzX0,20470
 verifiers/v1/scoring.py,sha256=-R5Cog14r_tJ6Q_oOfGr5VPAm2CCUhofYZB6B9lm4wM,5787
 verifiers/v1/session.py,sha256=J6ZPuvsg6vMTJAs9dMUMtfdjCuvp6Hom877DVdKFKlY,6912
 verifiers/v1/state.py,sha256=pcpN2V6tX7rIO5XdtdUYay1j0ktEwrMtq7w1ApfsPdE,675
-verifiers/v1/task.py,sha256=pM2S4YU56jTuvvwHe00r1pBupQ42pvMUzpQhUXqpAPk,10232
-verifiers/v1/taskset.py,sha256=5ffTlnmiN2GDePhonGSJ1a7iTseLPOC0DkwaVwf7uY4,4465
+verifiers/v1/task.py,sha256=skqOc-j2AxA6Wz2ikDD03o3cu1cVvr1G6gRJZrpOgfY,9059
+verifiers/v1/taskset.py,sha256=s9T7v3REU1EKVCu2zORMA3gWinRK6LJk4GUvoKhAmEQ,4337
 verifiers/v1/trace.py,sha256=_bGHOrE6fRLGm0JYc0PrrrO3lyr-qqOF5Y1bHfViwR0,19347
 verifiers/v1/types.py,sha256=1PxamJspmoTc3OlFZAwH6o_2i_Tg9ZNvHxttIHWIxq8,8948
 verifiers/v1/acp/__init__.py,sha256=9dwH6fLopNndRmcpRSYC8ghzzMquaLfgh645QM-Dwic,2189
@@ -194,7 +194,7 @@ verifiers/v1/acp/_runner.py,sha256=FqEnHR0Q_YrCt_EuaJqiqTpd2_kFa41FqcEunwOhvTI,6
 verifiers/v1/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/cli/debug.py,sha256=-jeEV_Q5tV8SOkWc1GdPZcXZWON_jU5gjeO4YoZm79o,11482
 verifiers/v1/cli/gepa.py,sha256=hLIbS_4jBx0vvy4PXoePR5riFXcNvzQCMixZ-g2OR_U,4026
-verifiers/v1/cli/init.py,sha256=dVpX1YMz7DFmHgskAsPjP-HmVTGoZfC0ZUyE-T4I6l0,8322
+verifiers/v1/cli/init.py,sha256=Qj96Kwzfhaejhrc0yYqHNRl3ncoID028RGlHvS1C98Q,8508
 verifiers/v1/cli/output.py,sha256=pMS5Z0EBgyAVShNCAk9jVURNeLOaTfeRWpt0J8hHjq8,5890
 verifiers/v1/cli/replay.py,sha256=DAh9kB5PB-ppZLGhxdJwrCPch_Yky5HKhplwGFhKEz0,10164
 verifiers/v1/cli/resolve.py,sha256=OsNw4C3r9xOVFa-mminJFcex_bAsJ6EQ8K7A_H8u2kQ,4370
@@ -252,10 +252,13 @@ verifiers/v1/gepa/config.py,sha256=gFn8Re4GoeHJs2PZ-iw4jCEZ72AiM1lNgwaA0uvJlYw,4
 verifiers/v1/gepa/dataset.py,sha256=N2QYqUljHVuAdCbUuydEIlarvAWa6LFdGKu9qsOvjro,2186
 verifiers/v1/gepa/reflection.py,sha256=ptHhx0lcDLcWhWhXuslh9T8gPVNpnX5XLPtXoneW1yA,1054
 verifiers/v1/gepa/runner.py,sha256=-LZEt0lod6NKfyxg2dvOXZjfzTv3AWwendexd1De26c,6260
-verifiers/v1/harnesses/__init__.py,sha256=3fgfAMzFrNgu7RIO_evZPxipPdiskiHun5EKsdS4y6k,1301
+verifiers/v1/harnesses/__init__.py,sha256=706ES37vrh_kCbctnbgkXIb1jBtO2ozZY5klBSg8ss0,1460
 verifiers/v1/harnesses/bash/__init__.py,sha256=IAV4eMMqDHSl-9ANb0Q3kxmBZ22SeHZFBM448X4gQio,140
 verifiers/v1/harnesses/bash/harness.py,sha256=ZtjhsANuK3vXkf9mRCi8-sBMyPebJMBNHa58vTd2Fm8,5284
 verifiers/v1/harnesses/bash/program.py,sha256=hGv3k9jwoabL045AGiZWiX76Cca3Juk58AaMYeJTEGo,15334
+verifiers/v1/harnesses/browser_use/__init__.py,sha256=yxm2IG5TDLadxZEkX8SL26XWbcPcODXcrJfPfm1GqwU,171
+verifiers/v1/harnesses/browser_use/harness.py,sha256=-P7CwMMkR7rLANAb6T4aOjAS4YuxzMPbm1MGvbIzbNI,5835
+verifiers/v1/harnesses/browser_use/program.py,sha256=uk_nwTX_68bZckSFMkgtgwSkUK5HWBGqLTCDwxbt8P8,14287
 verifiers/v1/harnesses/claude_code/__init__.py,sha256=JkZQylMSA3o1fvBx_NM41y_umuahmOSqFjTrJ2hryTw,171
 verifiers/v1/harnesses/claude_code/harness.py,sha256=98k5T8d8nFQmxIU-ytkrkrrP6UNR5NxoGhVJRLVa-fI,4015
 verifiers/v1/harnesses/codex/__init__.py,sha256=ocyBvlpSO8c3pT6D1ebrQohoXYESQcCyPT5kN3j1-N4,132
@@ -291,7 +294,7 @@ verifiers/v1/judges/reference.txt,sha256=Ej35kGXiT2uJ0LcHIeez49rCS_9uHmCQKoVdn93
 verifiers/v1/judges/rubric.py,sha256=PglrJhMQR6scFyfz20jVlLQEzeqXKV2P6r0K0K7jjBE,11916
 verifiers/v1/judges/rubric.txt,sha256=3KA2ZeYYdcDweJhUGopxu9EFgARhu2aWft5S5cj8Bc8,323
 verifiers/v1/mcp/__init__.py,sha256=pm8ZWBiL1q1tY4wHvJzM_QrZ9oR8aGm-bIRWiRJsgHI,408
-verifiers/v1/mcp/launch.py,sha256=lRtcl98BO7imC44bsxO2yljwRQR_KCvJ6P-bd_hS5L8,18444
+verifiers/v1/mcp/launch.py,sha256=KtRr59FMQkp4_m8ImUw-pIrxMUbDXd1RvzLJXw_HQ3Q,18452
 verifiers/v1/mcp/server.py,sha256=zS3-m-0IX3z8nVzUhQj391f8_OieE1nKr5S4Nj0dSSU,10987
 verifiers/v1/mcp/toolset.py,sha256=tSPXgTNREZyaeCnoFVnDfXxFAj_pStZMPUIgyPrY_Dk,996
 verifiers/v1/runtimes/__init__.py,sha256=TqNtJVahzQuLS4fgCVeT9xdOzTkp0hRAPOkl7eFVDUg,2011
@@ -319,7 +322,7 @@ verifiers/v1/tasksets/textarena/__init__.py,sha256=Os2OlBY_pSH0B1DP32RitumgSLpq2
 verifiers/v1/tasksets/textarena/taskset.py,sha256=9fa_unlFiFZyuN4I02rW3x4qyRnFWXqwTUZTloQefu0,4433
 verifiers/v1/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/utils/aio.py,sha256=0yNFOqB2oO6Qktc6NI_ZSAGuo1aHsIK_wuT1BtmoZ3I,1532
-verifiers/v1/utils/compile.py,sha256=plpZYt-UD8q7Yx5pD7Rfy1w7Jer5TCBk2CJGHiNcPio,5696
+verifiers/v1/utils/compile.py,sha256=Oh4hDVRNsql_BSOMh4kDEuze9_ZF17Q7uzBhhU70UZA,5663
 verifiers/v1/utils/format.py,sha256=NfQo9M5KMzWCZpQaet5YtsJx56vCKAZPfbjZZJLJhhM,2187
 verifiers/v1/utils/generic.py,sha256=2VIN9RapMO9pVkFHB1UKUHck1vlTJ1pjfad2NzxuuXA,2055
 verifiers/v1/utils/git.py,sha256=MTRinOvG4EddfjdahHkFZUihHmhJ37n5ROtgjzjE1NA,8366
@@ -330,8 +333,8 @@ verifiers/v1/utils/logging.py,sha256=OcMHA6NsYux3oIzjPuI95rDWmFBHNcHDjZeNIhXTX-Y
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
 verifiers/v1/utils/sampling.py,sha256=52TJ5-Hmpvp4YSgKNR-LmAPVwm4RDcH-o2-BnbVTmYQ,965
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.2.2.dev55.dist-info/METADATA,sha256=Jcl9dMFMROAPjPXzDCMNCz_r43jDq4Xp2QjEZZ0Z9b4,4545
-verifiers-0.2.2.dev55.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev55.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
-verifiers-0.2.2.dev55.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev55.dist-info/RECORD,,
+verifiers-0.2.2.dev57.dist-info/METADATA,sha256=4FbkVpDnb3GnGhrNC-YaPVwF8qfnOQgOMoHLmWkC57E,4545
+verifiers-0.2.2.dev57.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev57.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
+verifiers-0.2.2.dev57.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev57.dist-info/RECORD,,