verifiers 0.1.15.dev408__tar.gz → 0.1.15.dev409__tar.gz

{verifiers-0.1.15.dev408 → verifiers-0.1.15.dev409}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev408
+Version: 0.1.15.dev409
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev408 → verifiers-0.1.15.dev409}/pyproject.toml

@@ -249,8 +249,8 @@ markers = [
     "modal: v1 e2e cases touching the modal runtime (local-only setup; excluded in CI)",
     "colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
     "shared: v1 e2e cases with a shared (one-per-eval) tool server",
+    "null: v1 e2e cases on the null harness",
     "default: v1 e2e cases on the default harness",
-    "bash: v1 e2e cases on the bash harness",
     "rlm: v1 e2e cases on the rlm harness",
     "kimi_code: v1 e2e cases on the kimi-code harness",
     "codex: v1 e2e cases on the codex harness",

{verifiers-0.1.15.dev408 → verifiers-0.1.15.dev409}/tests/v1/conftest.py

@@ -15,12 +15,12 @@ Every matrix value carries a pytest mark, so subsets select with `-m`:
     uv run pytest tests/v1 -n auto                                # the whole matrix (needs modal setup)
     uv run pytest tests/v1 -n auto -m "not prime and not modal"  # the CI matrix (host + docker only)
     uv run pytest tests/v1 -n auto -m docker                      # any case touching the docker runtime
-    uv run pytest tests/v1 -n auto -m bash                        # only the bash harness
+    uv run pytest tests/v1 -n auto -m default                     # only the default harness
     uv run pytest tests/v1 -n auto -m prime                       # only prime (real sandboxes; local)
     uv run pytest tests/v1 -n auto -m modal                       # only modal (needs local setup)
 
 Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placements `colocated` / `shared`,
-harnesses `default` / `bash` / `rlm` / `kimi_code` / `codex`. A mark is applied per axis, so it
+harnesses `null` / `default` / `rlm` / `kimi_code` / `codex`. A mark is applied per axis, so it
 selects every case touching that value on ANY axis; for one exact combination use `-k` on the test
 id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). prime/modal provision real remote
 sandboxes (slow, infra-flaky, need setup), so they're local-only — CI runs `-m "not prime and not modal"`.
@@ -108,15 +108,15 @@ def tool_runtime(request) -> dict:
     return {"runtime": {"type": request.param}}
 
 
-# Harnesses, composed with the runtime fixtures, each carrying its harness mark (`-m bash`, ...).
+# Harnesses, composed with the runtime fixtures, each carrying its harness mark (`-m default`, ...).
 # Built-ins are bundled in the `harnesses` package; the agent CLIs (`rlm` / `kimi-code` / `codex`)
 # install their dependencies at rollout. `compact` (an example harness) and `terminus-2` (drives
-# the host tmux) are excluded. `test_agentic` skips `default` (a chat loop with no shell);
+# the host tmux) are excluded. `test_agentic` skips `null` (a chat loop with no shell);
 # `test_single_turn` skips `codex` (a coding agent, unreliable on a no-op echo).
 @pytest.fixture(
     params=[
+        pytest.param("null", marks=pytest.mark.null, id="null"),
         pytest.param("default", marks=pytest.mark.default, id="default"),
-        pytest.param("bash", marks=pytest.mark.bash, id="bash"),
         pytest.param("rlm", marks=pytest.mark.rlm, id="rlm"),
         pytest.param("kimi-code", marks=pytest.mark.kimi_code, id="kimi-code"),
         pytest.param("codex", marks=pytest.mark.codex, id="codex"),
@@ -167,7 +167,7 @@ def _eval_config(
     taskset: str,
     *,
     output_dir: Path,
-    harness: str = "default",
+    harness: str = "null",
     n: int = 1,
     num_tasks: int = 1,
     max_tokens: int = 2048,

{verifiers-0.1.15.dev408 → verifiers-0.1.15.dev409}/tests/v1/test_e2e.py

@@ -51,7 +51,7 @@ async def test_user(run_v1, harness_runtime, user_runtime, tmp_path):
         )
     (trace,) = await run_v1(
         "echo-user-sim-v1",
-        harness="default",
+        harness="null",
         harness_overrides={"runtime": {"type": harness_runtime}},
         output_dir=tmp_path,
         max_turns=6,
@@ -83,7 +83,7 @@ async def test_tool(run_v1, run_v1_server, harness_runtime, tool_runtime, tmp_pa
     run = run_v1_server if tool_runtime.get("shared") else run_v1
     (trace,) = await run(
         "echo-tool-v1",
-        harness="default",
+        harness="null",
         harness_overrides={"runtime": {"type": harness_runtime}},
         output_dir=tmp_path,
         max_turns=6,
@@ -114,7 +114,7 @@ async def test_tool_state(run_v1, harness_runtime, tool_runtime, tmp_path):
         )
     (trace,) = await run_v1(
         "counter-tool-v1",
-        harness="default",
+        harness="null",
         harness_overrides={"runtime": {"type": harness_runtime}},
         output_dir=tmp_path,
         max_turns=8,
@@ -167,7 +167,7 @@ async def test_shared_tool_isolation(
         )
     traces = await run_v1_server(
         "scratchpad-v1",
-        harness="default",
+        harness="null",
         harness_overrides={"runtime": {"type": harness_runtime}},
         output_dir=tmp_path,
         num_tasks=2,  # two distinct words, run concurrently against the one shared server
@@ -195,7 +195,7 @@ async def test_tool_response_image(run_v1, tmp_path):
     """MCP image content from a tool result survives into the v1 trace (needs a vision model)."""
     (trace,) = await run_v1(
         "tool-response-image-v1",
-        harness="default",
+        harness="null",
         harness_overrides={"runtime": {"type": "subprocess"}},
         model="qwen/qwen3-vl-8b-instruct",
         output_dir=tmp_path,
@@ -209,9 +209,9 @@ async def test_tool_response_image(run_v1, tmp_path):
 @pytest.mark.e2e
 async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
     """Agentic: write a phrase to a file with the agent's shell, checked in the runtime."""
-    if harness == "default":
+    if harness == "null":
         pytest.skip(
-            "default is a chat loop with no shell — it can't do the file-write task"
+            "null is a chat loop with no shell — it can't do the file-write task"
         )
     (trace,) = await run_v1(
         "echo-agentic-v1",

{verifiers-0.1.15.dev408 → verifiers-0.1.15.dev409}/verifiers/v1/GUIDE.md

@@ -552,15 +552,16 @@ Otherwise pick a built-in, selected with `--harness.id`:
 
 | id | what it is |
 | --- | --- |
-| `default` | a tiny OpenAI chat loop (MCP tools only, no tools of its own) |
-| `bash` | the `default` chat loop plus a local `bash` tool, for shell-driving agents |
+| `default` | a `bash` + `edit` coding agent (the `edit` tool is on by default — `--harness.edit false` for bash-only) — the fallback when no harness is given |
+| `null` | a tiny OpenAI chat loop (MCP tools only, no tools of its own) |
 | `rlm` | the RLM CLI agent |
 | `codex` | the Codex CLI (Responses dialect + SSE relay) |
 | `mini-swe-agent` | the mini-swe-agent CLI (a minimal SWE agent) |
 | `kimi-code` | the Kimi Code CLI agent |
 
 ```bash
-uv run eval gsm8k-v1 -n 1                    # default harness
+uv run eval gsm8k-v1 -n 1                    # default harness (bash + edit; the fallback)
+uv run eval gsm8k-v1 -n 1 --harness.id null  # bare chat loop, no local tools
 uv run eval gsm8k-v1 -n 1 --harness.id rlm   # same taskset, different driver
 ```
 
@@ -792,7 +793,7 @@ form. In a prime-rl config:
 [[orchestrator.train.env]]
 name    = "gsm8k"
 taskset = { id = "gsm8k-v1", dataset_name = "..." }                 # any v1 taskset id
-harness = { id = "default", runtime = { type = "subprocess" } }
+harness = { id = "null", runtime = { type = "subprocess" } }
 timeout = { scoring = 10 }                                          # per-stage cap (default: no limit)
 # pool  = { type = "elastic", max_workers = 8, multiplex = 128 }    # env-server pool (default elastic, self-sizing)
 ```

{verifiers-0.1.15.dev408 → verifiers-0.1.15.dev409}/verifiers/v1/README.md

@@ -96,7 +96,8 @@ Harness examples (under `environments/`):
 The program that drives the rollout — same taskset, different driver:
 
 ```bash
-uv run eval gsm8k-v1 -n 1                     # default: bare agent (MCP tools only)
+uv run eval gsm8k-v1 -n 1                     # default harness (fallback): a bash + edit agent
+uv run eval gsm8k-v1 -n 1 --harness.id null   # bare chat loop, no local tools (MCP tools only)
 uv run eval gsm8k-v1 -n 1 --harness.id rlm    # the rlm harness
 uv run eval gsm8k-v1 -n 1 --harness.id codex  # the codex harness
 ```
@@ -106,7 +107,7 @@ filesystem, so run it under a containerized runtime: `docker` locally, or a remo
 `modal` sandbox (not the default `subprocess`). 
 
 ```bash
-uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id bash            # bash-only agent
+uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id default         # the bash + edit agent
 uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id mini-swe-agent  # the mini-swe-agent CLI
 uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id rlm             # the rlm CLI agent
 uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id codex           # the codex CLI agent

{verifiers-0.1.15.dev408 → verifiers-0.1.15.dev409}/verifiers/v1/cli/dashboard/eval.py

@@ -115,9 +115,9 @@ def _aligned(rows: list[list[str]]) -> list[str]:
 
 
 def _warning(config: EvalConfig) -> Text | None:
-    """A local-runtime caution for a code-running harness (none for the tool-less `default`),
+    """A local-runtime caution for a code-running harness (none for the tool-less `null`),
     shown above the overview rather than as a row in it."""
-    if config.harness.id != "default" and config.harness.runtime.type == "subprocess":
+    if config.harness.id != "null" and config.harness.runtime.type == "subprocess":
         return Text(
             "warning  Runs on the local system; local files and settings may affect this "
             "evaluation. Use subprocess only for debugging, or use docker or prime for an "
@@ -254,10 +254,12 @@ def _breakdown(done: list[Trace]) -> Table | None:
             names.extend(n for n in getattr(trace, source) if n not in names)
         if not names:
             continue
-        segments = [
-            f"{name} {format_mean(done, lambda t, n=name, s=source: getattr(t, s).get(n, 0.0))}"
-            for name in names
-        ]
+        segments = []
+        for name in names:
+            mean = format_mean(
+                done, lambda t, n=name, s=source: getattr(t, s).get(n, 0.0)
+            )
+            segments.append(f"{name} {mean}")
         grid.add_row(label, "  ·  ".join(segments))
 
     # Resource use over every completed rollout (errored ones still spent tokens/time): tokens and

{verifiers-0.1.15.dev408 → verifiers-0.1.15.dev409}/verifiers/v1/env.py

@@ -262,7 +262,11 @@ class Environment:
                 "(NEEDS_CONTAINER), but the harness runs on the subprocess runtime; "
                 "use --harness.runtime.type docker or prime."
             )
-        if self.harness.config.id != "default" and isinstance(
+        # The warning is about the *agent* running arbitrary code on the host: every harness hands
+        # it local execution (bash/edit, or a CLI agent) except the tool-less `null` chat loop,
+        # whose program only relays the model and remote MCP tools — so exempt `null`, warn for the
+        # rest. (`null` still runs its fixed chat-loop program locally, but nothing agent-authored.)
+        if self.harness.config.id != "null" and isinstance(
             self.harness.config.runtime, SubprocessConfig
         ):
             logger.warning(

{verifiers-0.1.15.dev408 → verifiers-0.1.15.dev409}/verifiers/v1/harnesses/__init__.py

@@ -2,7 +2,6 @@
 
 Re-exports each harness's class + config off the package."""
 
-from verifiers.v1.harnesses.bash import BashHarness, BashHarnessConfig
 from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig
 from verifiers.v1.harnesses.default import DefaultHarness, DefaultHarnessConfig
 from verifiers.v1.harnesses.kimi_code import KimiCodeHarness, KimiCodeHarnessConfig
@@ -10,18 +9,19 @@ from verifiers.v1.harnesses.mini_swe_agent import (
     MiniSWEAgentHarness,
     MiniSWEAgentHarnessConfig,
 )
+from verifiers.v1.harnesses.null import NullHarness, NullHarnessConfig
 from verifiers.v1.harnesses.rlm import RLMHarness, RLMHarnessConfig
 from verifiers.v1.harnesses.terminus_2 import Terminus2Harness, Terminus2HarnessConfig
 
 __all__ = [
-    "BashHarness",
-    "BashHarnessConfig",
     "CodexHarness",
     "CodexHarnessConfig",
     "DefaultHarness",
     "DefaultHarnessConfig",
     "KimiCodeHarness",
     "KimiCodeHarnessConfig",
+    "NullHarness",
+    "NullHarnessConfig",
     "MiniSWEAgentHarness",
     "MiniSWEAgentHarnessConfig",
     "RLMHarness",

verifiers-0.1.15.dev409/verifiers/v1/harnesses/default/__init__.py

@@ -0,0 +1,11 @@
+"""default — v1's built-in agentic harness (the fallback when no `--harness.id` is given): a chat
+loop with a local `bash` tool and an optional `edit` tool (single-occurrence string replacement,
+on by default). Its `harness.py` (class + config) and the `program.py` script staged into the
+runtime. Resolved by id via `load_harness`."""
+
+from verifiers.v1.harnesses.default.harness import (
+    DefaultHarness,
+    DefaultHarnessConfig,
+)
+
+__all__ = ["DefaultHarness", "DefaultHarnessConfig"]

{verifiers-0.1.15.dev408/verifiers/v1/harnesses/bash_edit → verifiers-0.1.15.dev409/verifiers/v1/harnesses/default}/harness.py

@@ -1,11 +1,11 @@
-"""The built-in bash_edit harness: the `bash` chat loop plus a local `edit` tool.
+"""The built-in default harness: a chat loop with a local `bash` tool and an optional `edit` tool.
 
-Same growing-message-list chat loop as the `bash` harness — a local `bash` tool that runs shell
-commands in the runtime, plus the taskset's MCP tools — with one extra local tool: `edit`, a
-single-occurrence string replacement in a file (ported from the rlm `edit` skill). For agentic
-tasks where targeted file edits are common and a model handles `edit` more reliably than
-hand-built `sed`/heredoc shell. Its uv script (deps: openai, mcp) is prepared during setup, then
-launched as the harness program.
+A growing-message-list chat loop with a local `bash` tool that runs shell commands in the runtime,
+the taskset's MCP tools, and — on by default, gated by the `edit` config flag — a local `edit`
+tool for single-occurrence string replacement in a file (ported from the rlm `edit` skill; a model
+handles it more reliably than hand-built `sed`/heredoc shell). This is the fallback harness when no
+`--harness.id` is given. Its uv script (deps: openai, mcp) is prepared during setup, then launched
+as the harness program. For a pure chat loop with no local tools, use the `null` harness.
 """
 
 import json
@@ -20,19 +20,25 @@ from verifiers.v1.trace import Trace
 PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
 
 # Frames the model as a coding agent and names its local tools (a pure-text chat loop gets no
-# harness-injected prompt).
-BASH_EDIT_SYSTEM_PROMPT = (
-    "You are a coding agent. You have access to a bash tool for running shell commands and "
-    "an edit tool for single-occurrence string replacement in a file."
+# harness-injected prompt). The edit clause is appended only when the `edit` tool is enabled.
+BASH_SYSTEM_PROMPT = (
+    "You are a coding agent. You have access to a bash tool for running shell commands."
+)
+EDIT_SYSTEM_PROMPT = (
+    "You also have an edit tool for single-occurrence string replacement in a file."
 )
 
 
-class BashEditHarnessConfig(HarnessConfig):
-    """The built-in bash_edit harness. A uv script (deps: openai, mcp), so it runs in any runtime
+class DefaultHarnessConfig(HarnessConfig):
+    """The built-in default harness. A uv script (deps: openai, mcp), so it runs in any runtime
     that has `uv` (the harness bootstraps it) with no other setup."""
 
+    edit: bool = True
+    """Offer the local `edit` tool (single-occurrence string replacement in a file) alongside
+    `bash`. On by default; set `--harness.edit false` for a bash-only agent."""
+
 
-class BashEditHarness(Harness[BashEditHarnessConfig]):
+class DefaultHarness(Harness[DefaultHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = True
     SUPPORTS_MCP = True
     SUPPORTS_USER_SIM = True
@@ -51,9 +57,10 @@ class BashEditHarness(Harness[BashEditHarnessConfig]):
         mcp_urls: dict[str, str],
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(trace.task)
-        system_prompt = "\n\n".join(
-            p for p in (BASH_EDIT_SYSTEM_PROMPT, system_prompt) if p
-        )
+        harness_prompt = BASH_SYSTEM_PROMPT
+        if self.config.edit:
+            harness_prompt = f"{BASH_SYSTEM_PROMPT} {EDIT_SYSTEM_PROMPT}"
+        system_prompt = "\n\n".join(p for p in (harness_prompt, system_prompt) if p)
         env = {**self.config.resolved_env}
         args = [
             f"--base-url={endpoint}",
@@ -61,6 +68,8 @@ class BashEditHarness(Harness[BashEditHarnessConfig]):
             f"--model={ctx.model}",
             f"--system-prompt={system_prompt}",
         ]
+        if self.config.edit:
+            args.append("--edit")
         if mcp_urls:
             # The program connects to the tool servers over HTTP; hand it a standard
             # `mcpServers` URL config (the `mcp` client itself comes from the uv deps).

{verifiers-0.1.15.dev408/verifiers/v1/harnesses/bash_edit → verifiers-0.1.15.dev409/verifiers/v1/harnesses/default}/program.py

@@ -2,13 +2,13 @@
 # requires-python = ">=3.10"
 # dependencies = ["openai", "mcp"]
 # ///
-"""The bash_edit harness's program: a chat loop with local `bash` + `edit` tools (+ optional MCP).
+"""The default harness's program: a chat loop with a local `bash` tool (+ optional `edit`, MCP).
 
 A growing-message-list chat loop. It always offers a local `bash` tool that runs shell commands in
-the runtime and a local `edit` tool that replaces a unique string in a file; when the harness sets
-MCP_CONFIG (a standard `mcpServers` URL map) it also connects to those servers over streamable
-HTTP, exposes their tools to the model as `<server>_<tool>`, and routes those calls to the server.
-The loop runs until the model answers without a tool call.
+the runtime; with `--edit` it also offers a local `edit` tool that replaces a unique string in a
+file. When the harness sets MCP_CONFIG (a standard `mcpServers` URL map) it also connects to those
+servers over streamable HTTP, exposes their tools to the model as `<server>_<tool>`, and routes
+those calls to the server. The loop runs until the model answers without a tool call.
 
 It runs as a uv script (deps: openai, mcp), so the chat + tool plumbing is just the SDKs — the
 harness bootstraps `uv` in the runtime. The interception endpoint, per-rollout secret, and model
@@ -182,6 +182,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--system-prompt", default="")
     parser.add_argument("--prompt", default="")
     parser.add_argument("--mcp-config", default="")
+    parser.add_argument("--edit", action="store_true")
     return parser.parse_args()
 
 
@@ -193,7 +194,10 @@ async def main() -> None:
         mcp_tools, dispatch = (
             await connect_mcp(stack, config) if config.get("mcpServers") else ([], {})
         )
-        tools = [BASH_TOOL, EDIT_TOOL] + mcp_tools
+        tools = [BASH_TOOL]
+        if args.edit:
+            tools.append(EDIT_TOOL)
+        tools += mcp_tools
         messages = (
             [{"role": "system", "content": args.system_prompt}]
             if args.system_prompt
@@ -226,13 +230,24 @@ async def main() -> None:
                         }
                     )
                     continue
+                # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls
+                # below assume a dict, so reject anything else as a tool error rather than crashing.
+                if not isinstance(tool_args, dict):
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "tool_call_id": call.id,
+                            "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
+                        }
+                    )
+                    continue
                 if name in dispatch:
                     content = await call_mcp(dispatch, name, tool_args)
                 elif name == "bash":
                     content = await asyncio.to_thread(
                         run_bash, tool_args.get("command", "")
                     )
-                elif name == "edit":
+                elif name == "edit" and args.edit:
                     content = await asyncio.to_thread(
                         run_edit,
                         tool_args.get("path"),

verifiers-0.1.15.dev409/verifiers/v1/harnesses/null/__init__.py

@@ -0,0 +1,7 @@
+"""null — v1's built-in tool-less harness: its `harness.py` (class + config) and the
+`program.py` script it stages into the runtime. A plain chat loop with the taskset's MCP tools
+and no tools of its own. Resolved by id via `load_harness`."""
+
+from verifiers.v1.harnesses.null.harness import NullHarness, NullHarnessConfig
+
+__all__ = ["NullHarness", "NullHarnessConfig"]

{verifiers-0.1.15.dev408/verifiers/v1/harnesses/default → verifiers-0.1.15.dev409/verifiers/v1/harnesses/null}/harness.py

@@ -1,9 +1,10 @@
-"""The built-in default harness: runs a small chat-loop program as a uv script.
+"""The built-in null harness: runs a small chat-loop program as a uv script, no tools of its own.
 
 A growing-message-list chat loop with the taskset's MCP tools (host-side, resolved to URLs by
-the Environment) — and no tools of its own. Its uv script (deps: openai, mcp) is prepared
-during setup, then launched as the harness program. For a shell-driving agent, use a
-dedicated agentic harness (e.g. `mini-swe-agent`).
+the Environment) — and no tools of its own (it's "null" precisely because it adds no
+harness-side tooling). Its uv script (deps: openai, mcp) is prepared during setup, then launched
+as the harness program. For a shell-driving agent, use a dedicated agentic harness (e.g.
+`mini-swe-agent`).
 """
 
 import json
@@ -18,12 +19,12 @@ from verifiers.v1.trace import Trace
 PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
 
 
-class DefaultHarnessConfig(HarnessConfig):
-    """The built-in harness. A uv script (deps: openai, mcp), so it runs in any runtime that
+class NullHarnessConfig(HarnessConfig):
+    """The built-in null harness. A uv script (deps: openai, mcp), so it runs in any runtime that
     has `uv` (the harness bootstraps it) with no other setup."""
 
 
-class DefaultHarness(Harness[DefaultHarnessConfig]):
+class NullHarness(Harness[NullHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = True
     SUPPORTS_MCP = True
     SUPPORTS_USER_SIM = True

{verifiers-0.1.15.dev408/verifiers/v1/harnesses/default → verifiers-0.1.15.dev409/verifiers/v1/harnesses/null}/program.py

@@ -2,7 +2,7 @@
 # requires-python = ">=3.10"
 # dependencies = ["openai", "mcp"]
 # ///
-"""The default harness's program: a chat loop with the taskset's MCP tools (and none of its own).
+"""The null harness's program: a chat loop with the taskset's MCP tools (and none of its own).
 
 A growing-message-list chat loop. When the harness sets MCP_CONFIG (a standard `mcpServers` URL
 map) it connects to those servers over streamable HTTP, exposes their tools to the model as
@@ -142,6 +142,17 @@ async def main() -> None:
                         }
                     )
                     continue
+                # Valid JSON can still be a non-object (`[]`, `42`, `null`); the MCP dispatch
+                # assumes a dict, so reject anything else as a tool error rather than crashing.
+                if not isinstance(tool_args, dict):
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "tool_call_id": call.id,
+                            "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
+                        }
+                    )
+                    continue
                 if name in dispatch:
                     content = await call_mcp(dispatch, name, tool_args)
                 else:

{verifiers-0.1.15.dev408 → verifiers-0.1.15.dev409}/verifiers/v1/loaders.py

@@ -130,8 +130,9 @@ def harness_class(harness_id: str) -> type[Harness]:
 def default_harness_id(taskset_id: str) -> str:
     """The harness id to use when none is given. A taskset that bundles its own harness — its
     module also exports a `Harness` subclass via `__all__`, so the taskset id doubles as the
-    harness id — runs with that harness by default; otherwise the shared `default` harness. An
-    explicit `--harness.id` / toml id always takes precedence (this only supplies the fallback)."""
+    harness id — runs with that harness by default; otherwise the shared `default` harness (a
+    bash + edit agent). An explicit `--harness.id` / toml id always takes precedence (this only
+    supplies the fallback); for a tool-less chat loop, pass `--harness.id null`."""
     if not taskset_id:
         return "default"
     try:

verifiers-0.1.15.dev408/verifiers/v1/harnesses/bash/__init__.py

@@ -1,7 +0,0 @@
-"""bash — v1's built-in agentic harness: the `default` chat loop plus a single local `bash`
-tool. Its `harness.py` (class + config) and the `program.py` script staged into the runtime.
-Resolved by id via `load_harness`."""
-
-from verifiers.v1.harnesses.bash.harness import BashHarness, BashHarnessConfig
-
-__all__ = ["BashHarness", "BashHarnessConfig"]

verifiers-0.1.15.dev408/verifiers/v1/harnesses/bash/harness.py

@@ -1,79 +0,0 @@
-"""The built-in bash harness: the default chat loop plus a single local `bash` tool.
-
-The same growing-message-list chat loop as the `default` harness, but its program also offers a
-`bash` tool that runs shell commands in the runtime — for agentic tasks that drive a terminal
-(e.g. harbor / terminal-bench). MCP tools from the taskset are wired in too. Its uv script
-(deps: openai, mcp) is prepared during setup, then launched as the harness program.
-"""
-
-import json
-from pathlib import Path
-
-from verifiers.v1.harness import Harness, HarnessConfig
-from verifiers.v1.clients import RolloutContext
-from verifiers.v1.dialects.chat import message_to_wire
-from verifiers.v1.runtimes import ProgramResult, Runtime
-from verifiers.v1.trace import Trace
-
-PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
-
-# Tells the model it can run shell commands (a pure-text chat loop gets no harness-injected prompt).
-BASH_SYSTEM_PROMPT = "You have access to a bash tool; use it to run shell commands."
-
-
-class BashHarnessConfig(HarnessConfig):
-    """The built-in bash harness. A uv script (deps: openai, mcp), so it runs in any runtime that
-    has `uv` (the harness bootstraps it) with no other setup."""
-
-
-class BashHarness(Harness[BashHarnessConfig]):
-    APPENDS_SYSTEM_PROMPT = True
-    SUPPORTS_MCP = True
-    SUPPORTS_USER_SIM = True
-    SUPPORTS_MESSAGE_PROMPT = True
-
-    async def setup(self, runtime: Runtime) -> None:
-        await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env)
-
-    async def launch(
-        self,
-        ctx: RolloutContext,
-        trace: Trace,
-        runtime: Runtime,
-        endpoint: str,
-        secret: str,
-        mcp_urls: dict[str, str],
-    ) -> ProgramResult:
-        system_prompt, prompt = self.resolve_prompt(trace.task)
-        system_prompt = "\n\n".join(p for p in (BASH_SYSTEM_PROMPT, system_prompt) if p)
-        env = {**self.config.resolved_env}
-        args = [
-            f"--base-url={endpoint}",
-            f"--api-key={secret}",
-            f"--model={ctx.model}",
-            f"--system-prompt={system_prompt}",
-        ]
-        if mcp_urls:
-            # The program connects to the tool servers over HTTP; hand it a standard
-            # `mcpServers` URL config (the `mcp` client itself comes from the uv deps).
-            args.append(
-                "--mcp-config="
-                + json.dumps(
-                    {
-                        "mcpServers": {
-                            name: {"url": url} for name, url in mcp_urls.items()
-                        }
-                    }
-                )
-            )
-        # A Messages prompt (e.g. an image-bearing prompt) seeds the chat loop directly via env
-        # (it can be large multimodal content that overflows argv); a plain string is the single
-        # first user message; None means the task has no prompt and the user simulator opens it.
-        if isinstance(prompt, str):
-            args.append(f"--prompt={prompt}")
-        elif prompt is not None:
-            env["INITIAL_MESSAGES"] = json.dumps([message_to_wire(m) for m in prompt])
-        program = await runtime.prepare_uv_script(
-            PROGRAM_SOURCE, self.config.resolved_env
-        )
-        return await runtime.run_program([*program, *args], env)