verifiers 0.1.15.dev386__tar.gz → 0.1.15.dev388__tar.gz

{verifiers-0.1.15.dev386 → verifiers-0.1.15.dev388}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev386
+Version: 0.1.15.dev388
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev386 → verifiers-0.1.15.dev388}/verifiers/v1/README.md

@@ -228,9 +228,8 @@ uv run eval gsm8k-v1 -n 1 --retries.rollout.max-retries 3 --retries.rollout.incl
 
 Some tasksets wrap a whole benchmark family rather than a single task — shipped, installed by
 default. For example, `textarena` (TextArena games) and `harbor` (the agentic-
-benchmark registry). Harbor is the showcase: it pulls tasks straight from the Harbor registry
-via the `harbor` CLI (`uv tool install harbor`), each in its own declared, pullable container
-image — e.g. Terminal-Bench 2:
+benchmark registry). Harbor is the showcase: it pulls tasks straight from the Harbor registry,
+each in its own declared, pullable container image — e.g. Terminal-Bench 2:
 
 ```bash
 uv run eval harbor --taskset.dataset terminal-bench/terminal-bench-2 -n 10 --harness.id rlm

verifiers-0.1.15.dev388/verifiers/v1/harnesses/bash_edit/__init__.py

@@ -0,0 +1,10 @@
+"""bash_edit — v1's built-in agentic harness: the `bash` chat loop plus a local `edit` tool
+(single-occurrence string replacement, ported from rlm). Its `harness.py` (class + config) and
+the `program.py` script staged into the runtime. Resolved by id via `load_harness`."""
+
+from verifiers.v1.harnesses.bash_edit.harness import (
+    BashEditHarness,
+    BashEditHarnessConfig,
+)
+
+__all__ = ["BashEditHarness", "BashEditHarnessConfig"]

verifiers-0.1.15.dev388/verifiers/v1/harnesses/bash_edit/harness.py

@@ -0,0 +1,84 @@
+"""The built-in bash_edit harness: the `bash` chat loop plus a local `edit` tool.
+
+Same growing-message-list chat loop as the `bash` harness — a local `bash` tool that runs shell
+commands in the runtime, plus the taskset's MCP tools — with one extra local tool: `edit`, a
+single-occurrence string replacement in a file (ported from the rlm `edit` skill). For agentic
+tasks where targeted file edits are common and a model handles `edit` more reliably than
+hand-built `sed`/heredoc shell. Its uv script (deps: openai, mcp) is prepared during setup, then
+launched as the harness program.
+"""
+
+import json
+from pathlib import Path
+
+from verifiers.v1.harness import Harness, HarnessConfig
+from verifiers.v1.clients import RolloutContext
+from verifiers.v1.dialects.chat import message_to_wire
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.trace import Trace
+
+PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
+
+# Frames the model as a coding agent and names its local tools (a pure-text chat loop gets no
+# harness-injected prompt).
+BASH_EDIT_SYSTEM_PROMPT = (
+    "You are a coding agent. You have access to a bash tool for running shell commands and "
+    "an edit tool for single-occurrence string replacement in a file."
+)
+
+
+class BashEditHarnessConfig(HarnessConfig):
+    """The built-in bash_edit harness. A uv script (deps: openai, mcp), so it runs in any runtime
+    that has `uv` (the harness bootstraps it) with no other setup."""
+
+
+class BashEditHarness(Harness[BashEditHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_USER_SIM = True
+    SUPPORTS_MESSAGE_PROMPT = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+
+    async def launch(
+        self,
+        ctx: RolloutContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+    ) -> ProgramResult:
+        system_prompt, prompt = self.resolve_prompt(trace.task)
+        system_prompt = "\n\n".join(
+            p for p in (BASH_EDIT_SYSTEM_PROMPT, system_prompt) if p
+        )
+        env = {**self.config.env}
+        args = [
+            f"--base-url={endpoint}",
+            f"--api-key={secret}",
+            f"--model={ctx.model}",
+            f"--system-prompt={system_prompt}",
+        ]
+        if mcp_urls:
+            # The program connects to the tool servers over HTTP; hand it a standard
+            # `mcpServers` URL config (the `mcp` client itself comes from the uv deps).
+            args.append(
+                "--mcp-config="
+                + json.dumps(
+                    {
+                        "mcpServers": {
+                            name: {"url": url} for name, url in mcp_urls.items()
+                        }
+                    }
+                )
+            )
+        # A Messages prompt (e.g. an image-bearing prompt) seeds the chat loop directly via env
+        # (it can be large multimodal content that overflows argv); a plain string is the single
+        # first user message; None means the task has no prompt and the user simulator opens it.
+        if isinstance(prompt, str):
+            args.append(f"--prompt={prompt}")
+        elif prompt is not None:
+            env["INITIAL_MESSAGES"] = json.dumps([message_to_wire(m) for m in prompt])
+        program = await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.env)
+        return await runtime.run_program([*program, *args], env)

verifiers-0.1.15.dev388/verifiers/v1/harnesses/bash_edit/program.py

@@ -0,0 +1,250 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = ["openai", "mcp"]
+# ///
+"""The bash_edit harness's program: a chat loop with local `bash` + `edit` tools (+ optional MCP).
+
+A growing-message-list chat loop. It always offers a local `bash` tool that runs shell commands in
+the runtime and a local `edit` tool that replaces a unique string in a file; when the harness sets
+MCP_CONFIG (a standard `mcpServers` URL map) it also connects to those servers over streamable
+HTTP, exposes their tools to the model as `<server>_<tool>`, and routes those calls to the server.
+The loop runs until the model answers without a tool call.
+
+It runs as a uv script (deps: openai, mcp), so the chat + tool plumbing is just the SDKs — the
+harness bootstraps `uv` in the runtime. The interception endpoint, per-rollout secret, and model
+arrive as argv (not env), so the local tools' subprocesses never inherit them.
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import subprocess
+from contextlib import AsyncExitStack
+from pathlib import Path
+
+from openai import AsyncOpenAI
+
+BASH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "bash",
+        "description": "Run a bash command and return its combined stdout and stderr.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "command": {"type": "string", "description": "The bash command to run."}
+            },
+            "required": ["command"],
+        },
+    },
+}
+
+EDIT_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "edit",
+        "description": (
+            "Replace a unique string in a file. old_str must appear exactly once in the file."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "path": {
+                    "type": "string",
+                    "description": "File path (relative to cwd or absolute).",
+                },
+                "old_str": {
+                    "type": "string",
+                    "description": "Exact string to find (must appear exactly once).",
+                },
+                "new_str": {"type": "string", "description": "Replacement string."},
+            },
+            "required": ["path", "old_str", "new_str"],
+        },
+    },
+}
+
+
+def run_bash(command: str) -> str:
+    try:
+        result = subprocess.run(
+            ["bash", "-c", command], capture_output=True, text=True, timeout=3600
+        )
+        return result.stdout + result.stderr
+    except Exception as e:
+        return f"error: {e}"
+
+
+def run_edit(path: str, old_str: str, new_str: str) -> str:
+    if not isinstance(path, str) or not path:
+        return "error: 'path' is required"
+    if not isinstance(old_str, str) or not isinstance(new_str, str):
+        return "error: 'old_str' and 'new_str' must be strings"
+    if not old_str:
+        # '' matches everywhere (''.count('') == 1 on an empty file), so it would insert
+        # rather than replace — reject it to keep the "exactly once" contract honest.
+        return "error: 'old_str' must be a non-empty string"
+    filepath = Path(path)
+    if not filepath.is_absolute():
+        filepath = Path.cwd() / filepath
+    if not filepath.exists():
+        return f"error: {path} not found"
+    # Reading/writing can fail on a directory, permissions, or non-text content; return the
+    # error as a tool result instead of letting it abort the chat loop.
+    try:
+        content = filepath.read_text()
+    except Exception as e:
+        return f"error: could not read {path}: {e}"
+    count = content.count(old_str)
+    if count != 1:
+        return f"error: old_str must appear exactly once in {path} (found {count})"
+    try:
+        filepath.write_text(content.replace(old_str, new_str, 1))
+    except Exception as e:
+        return f"error: could not write {path}: {e}"
+    return f"Edited {path}"
+
+
+async def chat(
+    client: AsyncOpenAI, model: str, messages: list[dict], tools: list[dict]
+):
+    completion = await client.chat.completions.create(
+        model=model, messages=messages, tools=tools or None
+    )
+    return completion.choices[0].message
+
+
+async def connect_mcp(stack: AsyncExitStack, config: dict) -> tuple[list[dict], dict]:
+    """Connect to each configured MCP server (a streamable-HTTP `url`); return
+    (tool schemas, dispatch mapping `<server>_<tool>` -> (session, raw tool name))."""
+    from mcp import ClientSession
+    from mcp.client.streamable_http import (
+        create_mcp_http_client,
+        streamable_http_client,
+    )
+
+    tool_schemas: list[dict] = []
+    dispatch: dict[str, tuple] = {}
+    for name, spec in config.get("mcpServers", {}).items():
+        http_client = await stack.enter_async_context(
+            create_mcp_http_client(headers=spec.get("headers") or None)
+        )
+        read, write, *_ = await stack.enter_async_context(
+            streamable_http_client(spec["url"], http_client=http_client)
+        )
+        session = await stack.enter_async_context(ClientSession(read, write))
+        await session.initialize()
+        for tool in (await session.list_tools()).tools:
+            full = f"{name}_{tool.name}"
+            tool_schemas.append(
+                {
+                    "type": "function",
+                    "function": {
+                        "name": full,
+                        "description": tool.description or "",
+                        "parameters": tool.inputSchema,
+                    },
+                }
+            )
+            dispatch[full] = (session, tool.name)
+    return tool_schemas, dispatch
+
+
+def mcp_content_to_chat_content(blocks) -> str | list[dict]:
+    parts = []
+    for block in blocks:
+        if block.type == "text":
+            parts.append({"type": "text", "text": block.text})
+        elif block.type == "image":
+            url = f"data:{block.mimeType};base64,{block.data}"
+            parts.append({"type": "image_url", "image_url": {"url": url}})
+        else:
+            parts.append({"type": "text", "text": str(block)})
+    if not parts:
+        return str(blocks)
+    if all(part["type"] == "text" for part in parts):
+        return "\n".join(part["text"] for part in parts)
+    return parts
+
+
+async def call_mcp(dispatch: dict, name: str, arguments: dict) -> str | list[dict]:
+    session, raw = dispatch[name]
+    result = await session.call_tool(raw, arguments)
+    return mcp_content_to_chat_content(result.content)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", required=True)
+    parser.add_argument("--api-key", required=True)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--system-prompt", default="")
+    parser.add_argument("--prompt", default="")
+    parser.add_argument("--mcp-config", default="")
+    return parser.parse_args()
+
+
+async def main() -> None:
+    args = parse_args()
+    client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key)
+    config = json.loads(args.mcp_config or "{}")
+    async with AsyncExitStack() as stack:
+        mcp_tools, dispatch = (
+            await connect_mcp(stack, config) if config.get("mcpServers") else ([], {})
+        )
+        tools = [BASH_TOOL, EDIT_TOOL] + mcp_tools
+        messages = (
+            [{"role": "system", "content": args.system_prompt}]
+            if args.system_prompt
+            else []
+        )
+        # A Messages prompt (e.g. an image-bearing prompt) arrives pre-built as OpenAI wire dicts
+        # via INITIAL_MESSAGES (kept in env: it can be large multimodal content that overflows
+        # argv, and it's prompt content, not a credential); otherwise --prompt is the opening
+        # message. Both empty means the task has no prompt — the user simulator seeds the opening.
+        initial = json.loads(os.environ.get("INITIAL_MESSAGES", "[]"))
+        if initial:
+            messages.extend(initial)
+        elif args.prompt:
+            messages.append({"role": "user", "content": args.prompt})
+        while True:
+            message = await chat(client, args.model, messages, tools)
+            messages.append(message.model_dump(exclude_none=True))
+            if not message.tool_calls:
+                break
+            for call in message.tool_calls:
+                name = call.function.name
+                try:
+                    tool_args = json.loads(call.function.arguments or "{}")
+                except json.JSONDecodeError as e:
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "tool_call_id": call.id,
+                            "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
+                        }
+                    )
+                    continue
+                if name in dispatch:
+                    content = await call_mcp(dispatch, name, tool_args)
+                elif name == "bash":
+                    content = await asyncio.to_thread(
+                        run_bash, tool_args.get("command", "")
+                    )
+                elif name == "edit":
+                    content = await asyncio.to_thread(
+                        run_edit,
+                        tool_args.get("path"),
+                        tool_args.get("old_str"),
+                        tool_args.get("new_str"),
+                    )
+                else:
+                    content = f"error: unknown tool {name!r}"
+                messages.append(
+                    {"role": "tool", "tool_call_id": call.id, "content": content}
+                )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

{verifiers-0.1.15.dev386 → verifiers-0.1.15.dev388}/verifiers/v1/tasksets/harbor/taskset.py

@@ -1,10 +1,9 @@
 """harbor: run a Harbor (Terminal-Bench) dataset.
 
-`dataset` is a Harbor Hub registry id (e.g. "name", "name@version",
-"org/name@ref"), downloaded + cached on first use via the `harbor` CLI
-(fetched on demand with `uv` when needed). Each task dir ships task.toml +
-instruction.md (+ tests/, solution/, environment/). Defaults to the registry
-`hello-world` task.
+`dataset` is a Harbor Hub package id (e.g. "org/name" or "org/name@ref"),
+downloaded directly from the registry and cached on first use. Each task dir
+ships task.toml + instruction.md (+ tests/, solution/, environment/). Defaults
+to the registry `harbor/hello-world` dataset.
 
 The harness runs in a container and edits /app; then the task's verifier
 (tests/test.sh) runs in the SAME container and the reward it writes to
@@ -20,14 +19,16 @@ no environment at all also runs on that image, unless `require_image`.
 """
 
 import io
-import logging
-import shutil
-import subprocess
-import sys
+import json
+import os
 import tarfile
+import tempfile
 import tomllib
+from concurrent.futures import ThreadPoolExecutor
 from functools import lru_cache
 from pathlib import Path
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
 
 from pydantic import Field
 
@@ -40,14 +41,14 @@ from verifiers.v1.trace import Trace
 from verifiers.v1.types import StrictBaseModel
 
 CACHE = Path.home() / ".cache" / "harbor"
-HARBOR_PACKAGE = "harbor==0.14.0"
-logger = logging.getLogger("verifiers.v1.tasksets.harbor")
+HARBOR_SUPABASE_URL = "https://ofhuhcpkvzjlejydnvyd.supabase.co"
+HARBOR_SUPABASE_PUBLISHABLE_KEY = "sb_publishable_Z-vuQbpvpG-PStjbh4yE0Q_e-d3MTIH"
 
 
 class HarborConfig(TasksetConfig):
-    dataset: str = "hello-world"
-    """A Harbor Hub registry id ("name", "name@version", "org/name@ref"),
-    downloaded + cached via the `harbor` CLI."""
+    dataset: str = "harbor/hello-world"
+    """A Harbor Hub package id ("org/name" or "org/name@ref"), where ref is a
+    tag, integer revision, or sha256 digest. Downloaded directly and cached."""
     tasks: list[str] | None = None
     """Optional subset of task names to load (None = all)."""
     timeout_multiplier: float = Field(1.0, gt=0)
@@ -87,30 +88,130 @@ class HarborTask(Task):
 
 
 def dataset_dir(dataset: str) -> Path:
-    """Download a Harbor Hub `dataset` to a directory of task dirs via the `harbor`
-    CLI, cached on first use."""
+    """Download a Harbor Hub task or dataset package, cached on first use.
+
+    Harbor refs are tags, integer revisions, or ``sha256:`` digests.
+    """
     out = CACHE / dataset.replace("/", "_").replace("@", "_")
     if out.is_dir():
         return out
 
-    harbor_bin = shutil.which("harbor")
-    if harbor_bin is not None:
-        command = [harbor_bin]
+    package, _, ref = dataset.partition("@")
+    org, name = package.split("/", 1)
+    ref = ref or "latest"
+    url = os.getenv("HARBOR_SUPABASE_URL", HARBOR_SUPABASE_URL).rstrip("/")
+    key = os.getenv("HARBOR_SUPABASE_PUBLISHABLE_KEY", HARBOR_SUPABASE_PUBLISHABLE_KEY)
+    headers = {
+        "apikey": key,
+        "Authorization": f"Bearer {key}",
+        "Content-Type": "application/json",
+    }
+    # Harbor stores datasets in version tables; standalone tasks use the RPC fallback below.
+    version_query = {
+        "package.name": f"eq.{name}",
+        "package.type": "eq.dataset",
+        "package.org.name": f"eq.{org}",
+        "limit": 1,
+    }
+    if ref.isdigit():
+        table = "dataset_version"
+        version_query |= {
+            "select": "id,package:package_id!inner(name,type,org:org_id!inner(name))",
+            "revision": f"eq.{ref}",
+        }
+    elif ref.startswith("sha256:"):
+        table = "dataset_version"
+        version_query |= {
+            "select": "id,package:package_id!inner(name,type,org:org_id!inner(name))",
+            "content_hash": f"eq.{ref.removeprefix('sha256:')}",
+        }
     else:
-        uv_bin = shutil.which("uv")
-        if uv_bin is None:
-            raise RuntimeError(
-                "Harbor datasets require an installed `harbor` CLI or `uv` for automatic installation"
-            )
-        logger.info("harbor: installing %s with uv", HARBOR_PACKAGE)
-        command = [uv_bin, "tool", "run"]
-        # Harbor requires Python 3.12, while Verifiers also supports Python 3.11.
-        if sys.version_info[:2] == (3, 11):
-            command.extend(["--python", "3.12"])
-        command.extend(["--from", HARBOR_PACKAGE, "harbor"])
-    subprocess.run(
-        [*command, "download", dataset, "--export", "-o", str(out)], check=True
+        table = "dataset_version_tag"
+        version_query |= {
+            "select": "dataset_version:dataset_version_id(id),package:package_id!inner(name,type,org:org_id!inner(name))",
+            "tag": f"eq.{ref}",
+        }
+    request = Request(
+        f"{url}/rest/v1/{table}?{urlencode(version_query)}", headers=headers
     )
+    with urlopen(request) as response:
+        versions = json.load(response)
+
+    if versions:
+        version = versions[0]
+        version_id = version.get("id") or version["dataset_version"]["id"]
+        task_query = {
+            "select": "task_version:task_version_id(archive_path,package:package_id(name))",
+            "dataset_version_id": f"eq.{version_id}",
+            "order": "task_version_id",
+            "limit": 1000,
+            "offset": 0,
+        }
+        request = Request(
+            f"{url}/rest/v1/dataset_version_task?{urlencode(task_query)}",
+            headers={**headers, "Prefer": "count=exact"},
+        )
+        with urlopen(request) as response:
+            pages = [json.load(response)]
+            total = int(response.headers["Content-Range"].rsplit("/", 1)[1])
+
+        for offset in range(1000, total, 1000):
+            task_query["offset"] = offset
+            request = Request(
+                f"{url}/rest/v1/dataset_version_task?{urlencode(task_query)}",
+                headers=headers,
+            )
+            with urlopen(request) as response:
+                pages.append(json.load(response))
+        downloads = [
+            (
+                Path(name) / row["task_version"]["package"]["name"],
+                row["task_version"]["archive_path"],
+            )
+            for page in pages
+            for row in page
+        ]
+    else:
+        body = json.dumps({"p_org": org, "p_name": name, "p_ref": ref}).encode()
+        request = Request(
+            f"{url}/rest/v1/rpc/resolve_task_version", body, headers=headers
+        )
+        with urlopen(request) as response:
+            task = json.load(response)
+        if task is None:
+            raise ValueError(f"Harbor package not found: {dataset}")
+        downloads = [(Path(name), task["archive_path"])]
+
+    if not downloads:
+        raise ValueError(f"Harbor dataset has no tasks: {dataset}")
+
+    CACHE.mkdir(parents=True, exist_ok=True)
+    # Stream archives concurrently into a temporary directory; publish only a complete cache.
+    with tempfile.TemporaryDirectory(dir=CACHE) as temp:
+
+        def extract(item: tuple[Path, str]) -> None:
+            relative_path, archive_path = item
+            target = Path(temp) / relative_path
+            target.mkdir(parents=True, exist_ok=True)
+            request = Request(
+                f"{url}/storage/v1/object/authenticated/packages/{archive_path}",
+                headers=headers,
+            )
+            with urlopen(request) as response:
+                with tarfile.open(fileobj=response, mode="r|gz") as tar:
+                    kwargs = (
+                        {"filter": "data"} if hasattr(tarfile, "data_filter") else {}
+                    )
+                    tar.extractall(target, **kwargs)
+
+        with ThreadPoolExecutor(max_workers=100) as pool:
+            list(pool.map(extract, downloads))
+        try:
+            Path(temp).rename(out)
+        except OSError:
+            if out.is_dir():
+                return out
+            raise
     return out