verifiers 0.1.15.dev410__tar.gz → 0.1.15.dev412__tar.gz

{verifiers-0.1.15.dev410 → verifiers-0.1.15.dev412}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev410
+Version: 0.1.15.dev412
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev410 → verifiers-0.1.15.dev412}/verifiers/v1/GUIDE.md

@@ -552,7 +552,7 @@ Otherwise pick a built-in, selected with `--harness.id`:
 
 | id | what it is |
 | --- | --- |
-| `default` | a `bash` + `edit` coding agent (the `edit` tool is on by default — `--harness.edit false` for bash-only) — the fallback when no harness is given |
+| `default` | a `bash` + `edit` coding agent (`edit` on by default — `--harness.edit false` for bash-only; `--harness.search true` adds a serper.dev `search` tool) — the fallback when no harness is given |
 | `null` | a tiny OpenAI chat loop (MCP tools only, no tools of its own) |
 | `rlm` | the RLM CLI agent |
 | `codex` | the Codex CLI (Responses dialect + SSE relay) |

{verifiers-0.1.15.dev410 → verifiers-0.1.15.dev412}/verifiers/v1/cli/validate.py

@@ -29,9 +29,12 @@ from verifiers.v1.cli.resolve import (
     with_positional_taskset,
 )
 from verifiers.v1.configs.validate import ValidateConfig
+from verifiers.v1.decorators import invoke
 from verifiers.v1.env import resolve_runtime_config
 from verifiers.v1.runtimes import make_runtime
+from verifiers.v1.state import state_cls
 from verifiers.v1.taskset import Taskset
+from verifiers.v1.trace import Trace
 
 logger = logging.getLogger(__name__)
 
@@ -81,8 +84,12 @@ async def _validate_task(taskset: Taskset, task, config: ValidateConfig) -> dict
     )
     valid, exc = False, None
     try:
+        trace = Trace(task=task, state=state_cls(type(taskset))())
         await runtime.start()
-        await asyncio.wait_for(taskset.setup(task, runtime), setup_timeout)
+        await asyncio.wait_for(
+            invoke(taskset.setup, {"task": task, "trace": trace, "runtime": runtime}),
+            setup_timeout,
+        )
         valid = await asyncio.wait_for(
             taskset.validate(task, runtime), config.validate_timeout
         )

{verifiers-0.1.15.dev410 → verifiers-0.1.15.dev412}/verifiers/v1/harnesses/default/harness.py

@@ -1,14 +1,16 @@
-"""The built-in default harness: a chat loop with a local `bash` tool and an optional `edit` tool.
+"""The built-in default harness: a chat loop with a local `bash` tool, plus optional `edit`/`search`.
 
 A growing-message-list chat loop with a local `bash` tool that runs shell commands in the runtime,
-the taskset's MCP tools, and — on by default, gated by the `edit` config flag — a local `edit`
-tool for single-occurrence string replacement in a file (ported from the rlm `edit` skill; a model
-handles it more reliably than hand-built `sed`/heredoc shell). This is the fallback harness when no
-`--harness.id` is given. Its uv script (deps: openai, mcp) is prepared during setup, then launched
-as the harness program. For a pure chat loop with no local tools, use the `null` harness.
+the taskset's MCP tools, and two optional local tools: `edit` (single-occurrence string replacement
+in a file, ported from the rlm `edit` skill; on by default — a model handles it more reliably than
+hand-built `sed`/heredoc shell) and `search` (Google results via serper.dev; off by default, needs
+`SERPER_API_KEY`). This is the fallback harness when no `--harness.id` is given. Its uv script
+(deps: openai, mcp) is prepared during setup, then launched as the harness program. For a pure chat
+loop with no local tools, use the `null` harness.
 """
 
 import json
+import os
 from pathlib import Path
 
 from verifiers.v1.harness import Harness, HarnessConfig
@@ -27,6 +29,11 @@ BASH_SYSTEM_PROMPT = (
 EDIT_SYSTEM_PROMPT = (
     "You also have an edit tool for single-occurrence string replacement in a file."
 )
+# Appended when search is enabled, so the model knows the extra tool exists.
+SEARCH_PROMPT = (
+    "You also have a search tool that returns Google results (title, URL, snippet) for a query; "
+    "use it to research, and use bash (e.g. curl) to read result pages in full when needed."
+)
 
 
 class DefaultHarnessConfig(HarnessConfig):
@@ -37,6 +44,11 @@ class DefaultHarnessConfig(HarnessConfig):
     """Offer the local `edit` tool (single-occurrence string replacement in a file) alongside
     `bash`. On by default; set `--harness.edit false` for a bash-only agent."""
 
+    search: bool = False
+    """Offer a `search` tool (Google web results via serper.dev). Requires `SERPER_API_KEY` in the
+    eval environment; the key is handed to the program over argv (like the interception secret) so
+    the agent's `bash` subprocesses don't inherit it."""
+
 
 class DefaultHarness(Harness[DefaultHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = True
@@ -57,10 +69,14 @@ class DefaultHarness(Harness[DefaultHarnessConfig]):
         mcp_urls: dict[str, str],
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(trace.task)
-        harness_prompt = BASH_SYSTEM_PROMPT
+        fragments = [BASH_SYSTEM_PROMPT]
         if self.config.edit:
-            harness_prompt = f"{BASH_SYSTEM_PROMPT} {EDIT_SYSTEM_PROMPT}"
-        system_prompt = "\n\n".join(p for p in (harness_prompt, system_prompt) if p)
+            fragments.append(EDIT_SYSTEM_PROMPT)
+        if self.config.search:
+            fragments.append(SEARCH_PROMPT)
+        system_prompt = "\n\n".join(
+            p for p in (" ".join(fragments), system_prompt) if p
+        )
         env = {**self.config.resolved_env}
         args = [
             f"--base-url={endpoint}",
@@ -70,6 +86,24 @@ class DefaultHarness(Harness[DefaultHarnessConfig]):
         ]
         if self.config.edit:
             args.append("--edit")
+        if self.config.search:
+            # Resolve the key and keep it OUT of the program env: it's handed to the program over
+            # argv (--serper-key), so popping it here stops the agent's `bash` subprocesses from
+            # inheriting it via $SERPER_API_KEY / /proc/self/environ. Prefer a key set in the harness
+            # env (--harness.env / forward_env); fall back to the host env only when the key is
+            # *absent* (None), not present-but-empty — a rollout setting SERPER_API_KEY="" is
+            # deliberately masking the host secret, so honor that (the check below then fails loudly
+            # rather than leaking the host key). The pop is scoped to search=true, so an unrelated
+            # key forwarded for the agent's own bash-side use is left untouched.
+            serper_key = env.pop("SERPER_API_KEY", None)
+            if serper_key is None:
+                serper_key = os.environ.get("SERPER_API_KEY")
+            if not serper_key:
+                raise ValueError(
+                    "default search=true requires SERPER_API_KEY in the eval environment "
+                    "(the host env or --harness.env)"
+                )
+            args += ["--search", f"--serper-key={serper_key}"]
         if mcp_urls:
             # The program connects to the tool servers over HTTP; hand it a standard
             # `mcpServers` URL config (the `mcp` client itself comes from the uv deps).

{verifiers-0.1.15.dev410 → verifiers-0.1.15.dev412}/verifiers/v1/harnesses/default/program.py

@@ -1,18 +1,19 @@
 # /// script
 # requires-python = ">=3.10"
-# dependencies = ["openai", "mcp"]
+# dependencies = ["openai", "mcp", "httpx"]
 # ///
-"""The default harness's program: a chat loop with a local `bash` tool (+ optional `edit`, MCP).
+"""The default harness's program: a chat loop with a local `bash` tool (+ optional `edit`, `search`, MCP).
 
 A growing-message-list chat loop. It always offers a local `bash` tool that runs shell commands in
 the runtime; with `--edit` it also offers a local `edit` tool that replaces a unique string in a
-file. When the harness sets MCP_CONFIG (a standard `mcpServers` URL map) it also connects to those
-servers over streamable HTTP, exposes their tools to the model as `<server>_<tool>`, and routes
-those calls to the server. The loop runs until the model answers without a tool call.
+file, and with `--search` a `search` tool (Serper Google search). When the harness sets
+MCP_CONFIG (a standard `mcpServers` URL map) it also connects to those servers over streamable
+HTTP, exposes their tools to the model as `<server>_<tool>`, and routes those calls to the server.
+The loop runs until the model answers without a tool call.
 
-It runs as a uv script (deps: openai, mcp), so the chat + tool plumbing is just the SDKs — the
-harness bootstraps `uv` in the runtime. The interception endpoint, per-rollout secret, and model
-arrive as argv (not env), so the local tools' subprocesses never inherit them.
+It runs as a uv script (deps: openai, mcp, httpx), so the chat + tool plumbing is just the SDKs —
+the harness bootstraps `uv` in the runtime. The interception endpoint, per-rollout secret, model,
+and serper key arrive as argv (not env), so the local tools' subprocesses never inherit them.
 """
 
 import argparse
@@ -23,8 +24,11 @@ import subprocess
 from contextlib import AsyncExitStack
 from pathlib import Path
 
+import httpx
 from openai import AsyncOpenAI
 
+SERPER_URL = "https://google.serper.dev/search"
+
 BASH_TOOL = {
     "type": "function",
     "function": {
@@ -66,6 +70,77 @@ EDIT_TOOL = {
 }
 
 
+SEARCH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "search",
+        "description": (
+            "Run a web search via Serper (Google) and return the top organic results as title, "
+            "URL, and snippet. Issue focused queries and call it several times to cover different "
+            "angles; use the bash tool (e.g. curl) to read a result page in full."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "query": {"type": "string", "description": "The search query."},
+                "num_results": {
+                    "type": "integer",
+                    "description": "Number of results to return (default 5).",
+                },
+            },
+            "required": ["query"],
+        },
+    },
+}
+
+
+def format_results(results, query: str) -> str:
+    """Format Serper organic results as title/URL/snippet blocks."""
+    sections = []
+    for i, result in enumerate(results, 1):
+        title = (result.get("title") or "").strip() or "Untitled"
+        lines = [f"Result {i}: {title}"]
+        link = (result.get("link") or "").strip()
+        if link:
+            lines.append(f"URL: {link}")
+        snippet = (result.get("snippet") or "").strip()
+        if snippet:
+            lines.append(f"  - {snippet}")
+        sections.append("\n".join(lines))
+    if not sections:
+        return f"No results returned for query: {query}"
+    return "\n\n---\n\n".join(sections)
+
+
+def run_search(query: str, api_key: str, num_results: int = 5) -> str:
+    """Serper Google web search -> formatted organic results.
+
+    The key arrives as an argument (handed in by the harness over argv, like the interception
+    secret) instead of from `$SERPER_API_KEY`, so the agent's `bash` subprocesses never inherit it.
+    The whole call is wrapped so a bad query or malformed payload becomes a tool error rather than
+    raising out of the chat loop and killing the rollout."""
+    if not api_key:
+        return "Error: no Serper API key (SERPER_API_KEY was not set in the eval environment)"
+    # num_results comes straight from model tool JSON, so it may be a non-int (e.g. "ten"); coerce
+    # defensively — `organic[:num_results]` would otherwise raise on a bad slice.
+    try:
+        num_results = max(1, int(num_results))
+    except (TypeError, ValueError):
+        num_results = 5
+    try:
+        response = httpx.post(
+            SERPER_URL,
+            json={"q": query},
+            headers={"X-API-KEY": api_key, "Content-Type": "application/json"},
+            timeout=45,
+        )
+        response.raise_for_status()
+        organic = response.json().get("organic") or []
+        return format_results(organic[:num_results], query)
+    except Exception as e:
+        return f"search failed ({e}). Try again or rephrase the query."
+
+
 def run_bash(command: str) -> str:
     try:
         result = subprocess.run(
@@ -183,12 +258,19 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--prompt", default="")
     parser.add_argument("--mcp-config", default="")
     parser.add_argument("--edit", action="store_true")
+    parser.add_argument("--search", action="store_true")
+    parser.add_argument("--serper-key", default="")
     return parser.parse_args()
 
 
 async def main() -> None:
     args = parse_args()
-    client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key)
+    # No client-side retries: re-sending a request whose turn the interception already committed
+    # re-samples it and forks the trace into an extra dead-end branch. A generous timeout lets a
+    # slow turn finish instead of timing out and re-sending; genuine failures surface as error rows.
+    client = AsyncOpenAI(
+        base_url=args.base_url, api_key=args.api_key, timeout=1800.0, max_retries=0
+    )
     config = json.loads(args.mcp_config or "{}")
     async with AsyncExitStack() as stack:
         mcp_tools, dispatch = (
@@ -197,6 +279,8 @@ async def main() -> None:
         tools = [BASH_TOOL]
         if args.edit:
             tools.append(EDIT_TOOL)
+        if args.search:
+            tools.append(SEARCH_TOOL)
         tools += mcp_tools
         messages = (
             [{"role": "system", "content": args.system_prompt}]
@@ -254,6 +338,13 @@ async def main() -> None:
                         tool_args.get("old_str"),
                         tool_args.get("new_str"),
                     )
+                elif name == "search" and args.search:
+                    content = await asyncio.to_thread(
+                        run_search,
+                        tool_args.get("query", ""),
+                        args.serper_key,
+                        tool_args.get("num_results", 5),
+                    )
                 else:
                     content = f"error: unknown tool {name!r}"
                 messages.append(

{verifiers-0.1.15.dev410 → verifiers-0.1.15.dev412}/verifiers/v1/rollout.py

@@ -20,7 +20,7 @@ from enum import StrEnum
 
 from verifiers.v1.harness import Harness
 from verifiers.v1.clients import RolloutContext
-from verifiers.v1.decorators import discover_decorated
+from verifiers.v1.decorators import discover_decorated, invoke
 from verifiers.v1.errors import (
     HarnessError,
     RolloutError,
@@ -165,7 +165,10 @@ class Rollout:
                 boundary(TasksetError, "taskset setup"),
                 asyncio.timeout_at(setup_deadline),
             ):
-                await self.taskset.setup(self.task, runtime)
+                await invoke(
+                    self.taskset.setup,
+                    {"task": self.task, "trace": trace, "runtime": runtime},
+                )
             async with (
                 boundary(HarnessError, "harness setup"),
                 asyncio.timeout_at(setup_deadline),

{verifiers-0.1.15.dev410 → verifiers-0.1.15.dev412}/verifiers/v1/taskset.py

@@ -80,10 +80,16 @@ class Taskset(Generic[TaskT, ConfigT, StateT]):
         multi-turn conversation (e.g. a TextArena game)."""
         return None
 
-    async def setup(self, task: TaskT, runtime: Runtime) -> None:
+    async def setup(self, task: TaskT, trace: Trace, runtime: Runtime) -> None:
         """Prepare the live runtime for this task, after `runtime.start()` and before the
         harness runs. No-op by default; override to run per-task setup in the runtime (e.g.
-        a SWE row checking out its base commit). Errors propagate and fail the rollout."""
+        a SWE row checking out its base commit). Errors propagate and fail the rollout.
+
+        Like the scoring hooks, `setup` declares the inputs it needs *by parameter name* and
+        the framework injects them: any subset of `task`, `trace`, `runtime`. The trace (and
+        its per-rollout `trace.state`) already exists when `setup` runs, so an override may
+        stash per-rollout state there — e.g. `setup(self, task, trace, runtime)` or the
+        legacy `setup(self, task, runtime)` both work."""
         return None
 
     async def finalize(self, task: TaskT, trace: Trace, runtime: Runtime) -> None: