verifiers 0.1.15.dev387__tar.gz → 0.1.15.dev389__tar.gz

{verifiers-0.1.15.dev387 → verifiers-0.1.15.dev389}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev387
+Version: 0.1.15.dev389
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.1.15.dev389/verifiers/v1/harnesses/bash_edit/__init__.py

@@ -0,0 +1,10 @@
+"""bash_edit — v1's built-in agentic harness: the `bash` chat loop plus a local `edit` tool
+(single-occurrence string replacement, ported from rlm). Its `harness.py` (class + config) and
+the `program.py` script staged into the runtime. Resolved by id via `load_harness`."""
+
+from verifiers.v1.harnesses.bash_edit.harness import (
+    BashEditHarness,
+    BashEditHarnessConfig,
+)
+
+__all__ = ["BashEditHarness", "BashEditHarnessConfig"]

verifiers-0.1.15.dev389/verifiers/v1/harnesses/bash_edit/harness.py

@@ -0,0 +1,84 @@
+"""The built-in bash_edit harness: the `bash` chat loop plus a local `edit` tool.
+
+Same growing-message-list chat loop as the `bash` harness — a local `bash` tool that runs shell
+commands in the runtime, plus the taskset's MCP tools — with one extra local tool: `edit`, a
+single-occurrence string replacement in a file (ported from the rlm `edit` skill). For agentic
+tasks where targeted file edits are common and a model handles `edit` more reliably than
+hand-built `sed`/heredoc shell. Its uv script (deps: openai, mcp) is prepared during setup, then
+launched as the harness program.
+"""
+
+import json
+from pathlib import Path
+
+from verifiers.v1.harness import Harness, HarnessConfig
+from verifiers.v1.clients import RolloutContext
+from verifiers.v1.dialects.chat import message_to_wire
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.trace import Trace
+
+PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
+
+# Frames the model as a coding agent and names its local tools (a pure-text chat loop gets no
+# harness-injected prompt).
+BASH_EDIT_SYSTEM_PROMPT = (
+    "You are a coding agent. You have access to a bash tool for running shell commands and "
+    "an edit tool for single-occurrence string replacement in a file."
+)
+
+
+class BashEditHarnessConfig(HarnessConfig):
+    """The built-in bash_edit harness. A uv script (deps: openai, mcp), so it runs in any runtime
+    that has `uv` (the harness bootstraps it) with no other setup."""
+
+
+class BashEditHarness(Harness[BashEditHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_USER_SIM = True
+    SUPPORTS_MESSAGE_PROMPT = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+
+    async def launch(
+        self,
+        ctx: RolloutContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+    ) -> ProgramResult:
+        system_prompt, prompt = self.resolve_prompt(trace.task)
+        system_prompt = "\n\n".join(
+            p for p in (BASH_EDIT_SYSTEM_PROMPT, system_prompt) if p
+        )
+        env = {**self.config.env}
+        args = [
+            f"--base-url={endpoint}",
+            f"--api-key={secret}",
+            f"--model={ctx.model}",
+            f"--system-prompt={system_prompt}",
+        ]
+        if mcp_urls:
+            # The program connects to the tool servers over HTTP; hand it a standard
+            # `mcpServers` URL config (the `mcp` client itself comes from the uv deps).
+            args.append(
+                "--mcp-config="
+                + json.dumps(
+                    {
+                        "mcpServers": {
+                            name: {"url": url} for name, url in mcp_urls.items()
+                        }
+                    }
+                )
+            )
+        # A Messages prompt (e.g. an image-bearing prompt) seeds the chat loop directly via env
+        # (it can be large multimodal content that overflows argv); a plain string is the single
+        # first user message; None means the task has no prompt and the user simulator opens it.
+        if isinstance(prompt, str):
+            args.append(f"--prompt={prompt}")
+        elif prompt is not None:
+            env["INITIAL_MESSAGES"] = json.dumps([message_to_wire(m) for m in prompt])
+        program = await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        return await runtime.run_program([*program, *args], env)

verifiers-0.1.15.dev389/verifiers/v1/harnesses/bash_edit/program.py

@@ -0,0 +1,250 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = ["openai", "mcp"]
+# ///
+"""The bash_edit harness's program: a chat loop with local `bash` + `edit` tools (+ optional MCP).
+
+A growing-message-list chat loop. It always offers a local `bash` tool that runs shell commands in
+the runtime and a local `edit` tool that replaces a unique string in a file; when the harness sets
+MCP_CONFIG (a standard `mcpServers` URL map) it also connects to those servers over streamable
+HTTP, exposes their tools to the model as `<server>_<tool>`, and routes those calls to the server.
+The loop runs until the model answers without a tool call.
+
+It runs as a uv script (deps: openai, mcp), so the chat + tool plumbing is just the SDKs — the
+harness bootstraps `uv` in the runtime. The interception endpoint, per-rollout secret, and model
+arrive as argv (not env), so the local tools' subprocesses never inherit them.
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import subprocess
+from contextlib import AsyncExitStack
+from pathlib import Path
+
+from openai import AsyncOpenAI
+
+BASH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "bash",
+        "description": "Run a bash command and return its combined stdout and stderr.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "command": {"type": "string", "description": "The bash command to run."}
+            },
+            "required": ["command"],
+        },
+    },
+}
+
+EDIT_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "edit",
+        "description": (
+            "Replace a unique string in a file. old_str must appear exactly once in the file."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "path": {
+                    "type": "string",
+                    "description": "File path (relative to cwd or absolute).",
+                },
+                "old_str": {
+                    "type": "string",
+                    "description": "Exact string to find (must appear exactly once).",
+                },
+                "new_str": {"type": "string", "description": "Replacement string."},
+            },
+            "required": ["path", "old_str", "new_str"],
+        },
+    },
+}
+
+
+def run_bash(command: str) -> str:
+    try:
+        result = subprocess.run(
+            ["bash", "-c", command], capture_output=True, text=True, timeout=3600
+        )
+        return result.stdout + result.stderr
+    except Exception as e:
+        return f"error: {e}"
+
+
+def run_edit(path: str, old_str: str, new_str: str) -> str:
+    if not isinstance(path, str) or not path:
+        return "error: 'path' is required"
+    if not isinstance(old_str, str) or not isinstance(new_str, str):
+        return "error: 'old_str' and 'new_str' must be strings"
+    if not old_str:
+        # '' matches everywhere (''.count('') == 1 on an empty file), so it would insert
+        # rather than replace — reject it to keep the "exactly once" contract honest.
+        return "error: 'old_str' must be a non-empty string"
+    filepath = Path(path)
+    if not filepath.is_absolute():
+        filepath = Path.cwd() / filepath
+    if not filepath.exists():
+        return f"error: {path} not found"
+    # Reading/writing can fail on a directory, permissions, or non-text content; return the
+    # error as a tool result instead of letting it abort the chat loop.
+    try:
+        content = filepath.read_text()
+    except Exception as e:
+        return f"error: could not read {path}: {e}"
+    count = content.count(old_str)
+    if count != 1:
+        return f"error: old_str must appear exactly once in {path} (found {count})"
+    try:
+        filepath.write_text(content.replace(old_str, new_str, 1))
+    except Exception as e:
+        return f"error: could not write {path}: {e}"
+    return f"Edited {path}"
+
+
+async def chat(
+    client: AsyncOpenAI, model: str, messages: list[dict], tools: list[dict]
+):
+    completion = await client.chat.completions.create(
+        model=model, messages=messages, tools=tools or None
+    )
+    return completion.choices[0].message
+
+
+async def connect_mcp(stack: AsyncExitStack, config: dict) -> tuple[list[dict], dict]:
+    """Connect to each configured MCP server (a streamable-HTTP `url`); return
+    (tool schemas, dispatch mapping `<server>_<tool>` -> (session, raw tool name))."""
+    from mcp import ClientSession
+    from mcp.client.streamable_http import (
+        create_mcp_http_client,
+        streamable_http_client,
+    )
+
+    tool_schemas: list[dict] = []
+    dispatch: dict[str, tuple] = {}
+    for name, spec in config.get("mcpServers", {}).items():
+        http_client = await stack.enter_async_context(
+            create_mcp_http_client(headers=spec.get("headers") or None)
+        )
+        read, write, *_ = await stack.enter_async_context(
+            streamable_http_client(spec["url"], http_client=http_client)
+        )
+        session = await stack.enter_async_context(ClientSession(read, write))
+        await session.initialize()
+        for tool in (await session.list_tools()).tools:
+            full = f"{name}_{tool.name}"
+            tool_schemas.append(
+                {
+                    "type": "function",
+                    "function": {
+                        "name": full,
+                        "description": tool.description or "",
+                        "parameters": tool.inputSchema,
+                    },
+                }
+            )
+            dispatch[full] = (session, tool.name)
+    return tool_schemas, dispatch
+
+
+def mcp_content_to_chat_content(blocks) -> str | list[dict]:
+    parts = []
+    for block in blocks:
+        if block.type == "text":
+            parts.append({"type": "text", "text": block.text})
+        elif block.type == "image":
+            url = f"data:{block.mimeType};base64,{block.data}"
+            parts.append({"type": "image_url", "image_url": {"url": url}})
+        else:
+            parts.append({"type": "text", "text": str(block)})
+    if not parts:
+        return str(blocks)
+    if all(part["type"] == "text" for part in parts):
+        return "\n".join(part["text"] for part in parts)
+    return parts
+
+
+async def call_mcp(dispatch: dict, name: str, arguments: dict) -> str | list[dict]:
+    session, raw = dispatch[name]
+    result = await session.call_tool(raw, arguments)
+    return mcp_content_to_chat_content(result.content)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", required=True)
+    parser.add_argument("--api-key", required=True)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--system-prompt", default="")
+    parser.add_argument("--prompt", default="")
+    parser.add_argument("--mcp-config", default="")
+    return parser.parse_args()
+
+
+async def main() -> None:
+    args = parse_args()
+    client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key)
+    config = json.loads(args.mcp_config or "{}")
+    async with AsyncExitStack() as stack:
+        mcp_tools, dispatch = (
+            await connect_mcp(stack, config) if config.get("mcpServers") else ([], {})
+        )
+        tools = [BASH_TOOL, EDIT_TOOL] + mcp_tools
+        messages = (
+            [{"role": "system", "content": args.system_prompt}]
+            if args.system_prompt
+            else []
+        )
+        # A Messages prompt (e.g. an image-bearing prompt) arrives pre-built as OpenAI wire dicts
+        # via INITIAL_MESSAGES (kept in env: it can be large multimodal content that overflows
+        # argv, and it's prompt content, not a credential); otherwise --prompt is the opening
+        # message. Both empty means the task has no prompt — the user simulator seeds the opening.
+        initial = json.loads(os.environ.get("INITIAL_MESSAGES", "[]"))
+        if initial:
+            messages.extend(initial)
+        elif args.prompt:
+            messages.append({"role": "user", "content": args.prompt})
+        while True:
+            message = await chat(client, args.model, messages, tools)
+            messages.append(message.model_dump(exclude_none=True))
+            if not message.tool_calls:
+                break
+            for call in message.tool_calls:
+                name = call.function.name
+                try:
+                    tool_args = json.loads(call.function.arguments or "{}")
+                except json.JSONDecodeError as e:
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "tool_call_id": call.id,
+                            "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
+                        }
+                    )
+                    continue
+                if name in dispatch:
+                    content = await call_mcp(dispatch, name, tool_args)
+                elif name == "bash":
+                    content = await asyncio.to_thread(
+                        run_bash, tool_args.get("command", "")
+                    )
+                elif name == "edit":
+                    content = await asyncio.to_thread(
+                        run_edit,
+                        tool_args.get("path"),
+                        tool_args.get("old_str"),
+                        tool_args.get("new_str"),
+                    )
+                else:
+                    content = f"error: unknown tool {name!r}"
+                messages.append(
+                    {"role": "tool", "tool_call_id": call.id, "content": content}
+                )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

{verifiers-0.1.15.dev387 → verifiers-0.1.15.dev389}/verifiers/v1/harnesses/rlm/harness.py

@@ -11,6 +11,7 @@ programmatically (`await tools_<name>(...)`), rather than via a native MCP clien
 import json
 import logging
 import shlex
+from typing import Literal
 
 from pydantic import model_validator
 
@@ -22,6 +23,8 @@ from verifiers.v1.trace import Trace
 
 logger = logging.getLogger(__name__)
 
+BuiltinSkill = Literal["edit", "search"]
+
 RLM_REPO = "github.com/PrimeIntellect-ai/rlm.git"
 # rlm writes its session under $RLM_HOME/sessions/<id>/; point it at a workdir-
 # relative dir so it stays in the runtime (and is cleaned up with the workdir).
@@ -37,8 +40,8 @@ class RLMHarnessConfig(HarnessConfig):
     """Git ref (branch, tag, or commit) of rlm to install."""
     max_depth: int = 0
     """Recursion depth rlm may spawn sub-harnesses to (RLM_MAX_DEPTH)."""
-    skills: list[str] | None = None
-    """Built-in rlm skills to enable (RLM_SKILLS), e.g. `["edit"]`; None enables none.
+    skills: list[BuiltinSkill] = []
+    """Built-in rlm skills to enable (RLM_SKILLS), e.g. `["edit"]`; empty enables none.
     The tool set is fixed (ipython); only built-in skills are selectable."""
 
     @model_validator(mode="after")