verifiers 0.1.15.dev407__tar.gz → 0.1.15.dev409__tar.gz

{verifiers-0.1.15.dev407 → verifiers-0.1.15.dev409}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev407
+Version: 0.1.15.dev409
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev407 → verifiers-0.1.15.dev409}/pyproject.toml

@@ -249,8 +249,8 @@ markers = [
     "modal: v1 e2e cases touching the modal runtime (local-only setup; excluded in CI)",
     "colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
     "shared: v1 e2e cases with a shared (one-per-eval) tool server",
+    "null: v1 e2e cases on the null harness",
     "default: v1 e2e cases on the default harness",
-    "bash: v1 e2e cases on the bash harness",
     "rlm: v1 e2e cases on the rlm harness",
     "kimi_code: v1 e2e cases on the kimi-code harness",
     "codex: v1 e2e cases on the codex harness",

{verifiers-0.1.15.dev407 → verifiers-0.1.15.dev409}/tests/v1/conftest.py

@@ -15,12 +15,12 @@ Every matrix value carries a pytest mark, so subsets select with `-m`:
     uv run pytest tests/v1 -n auto                                # the whole matrix (needs modal setup)
     uv run pytest tests/v1 -n auto -m "not prime and not modal"  # the CI matrix (host + docker only)
     uv run pytest tests/v1 -n auto -m docker                      # any case touching the docker runtime
-    uv run pytest tests/v1 -n auto -m bash                        # only the bash harness
+    uv run pytest tests/v1 -n auto -m default                     # only the default harness
     uv run pytest tests/v1 -n auto -m prime                       # only prime (real sandboxes; local)
     uv run pytest tests/v1 -n auto -m modal                       # only modal (needs local setup)
 
 Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placements `colocated` / `shared`,
-harnesses `default` / `bash` / `rlm` / `kimi_code` / `codex`. A mark is applied per axis, so it
+harnesses `null` / `default` / `rlm` / `kimi_code` / `codex`. A mark is applied per axis, so it
 selects every case touching that value on ANY axis; for one exact combination use `-k` on the test
 id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). prime/modal provision real remote
 sandboxes (slow, infra-flaky, need setup), so they're local-only — CI runs `-m "not prime and not modal"`.
@@ -108,15 +108,15 @@ def tool_runtime(request) -> dict:
     return {"runtime": {"type": request.param}}
 
 
-# Harnesses, composed with the runtime fixtures, each carrying its harness mark (`-m bash`, ...).
+# Harnesses, composed with the runtime fixtures, each carrying its harness mark (`-m default`, ...).
 # Built-ins are bundled in the `harnesses` package; the agent CLIs (`rlm` / `kimi-code` / `codex`)
 # install their dependencies at rollout. `compact` (an example harness) and `terminus-2` (drives
-# the host tmux) are excluded. `test_agentic` skips `default` (a chat loop with no shell);
+# the host tmux) are excluded. `test_agentic` skips `null` (a chat loop with no shell);
 # `test_single_turn` skips `codex` (a coding agent, unreliable on a no-op echo).
 @pytest.fixture(
     params=[
+        pytest.param("null", marks=pytest.mark.null, id="null"),
         pytest.param("default", marks=pytest.mark.default, id="default"),
-        pytest.param("bash", marks=pytest.mark.bash, id="bash"),
         pytest.param("rlm", marks=pytest.mark.rlm, id="rlm"),
         pytest.param("kimi-code", marks=pytest.mark.kimi_code, id="kimi-code"),
         pytest.param("codex", marks=pytest.mark.codex, id="codex"),
@@ -167,7 +167,7 @@ def _eval_config(
     taskset: str,
     *,
     output_dir: Path,
-    harness: str = "default",
+    harness: str = "null",
     n: int = 1,
     num_tasks: int = 1,
     max_tokens: int = 2048,

{verifiers-0.1.15.dev407 → verifiers-0.1.15.dev409}/tests/v1/test_e2e.py

@@ -51,7 +51,7 @@ async def test_user(run_v1, harness_runtime, user_runtime, tmp_path):
         )
     (trace,) = await run_v1(
         "echo-user-sim-v1",
-        harness="default",
+        harness="null",
         harness_overrides={"runtime": {"type": harness_runtime}},
         output_dir=tmp_path,
         max_turns=6,
@@ -83,7 +83,7 @@ async def test_tool(run_v1, run_v1_server, harness_runtime, tool_runtime, tmp_pa
     run = run_v1_server if tool_runtime.get("shared") else run_v1
     (trace,) = await run(
         "echo-tool-v1",
-        harness="default",
+        harness="null",
         harness_overrides={"runtime": {"type": harness_runtime}},
         output_dir=tmp_path,
         max_turns=6,
@@ -114,7 +114,7 @@ async def test_tool_state(run_v1, harness_runtime, tool_runtime, tmp_path):
         )
     (trace,) = await run_v1(
         "counter-tool-v1",
-        harness="default",
+        harness="null",
         harness_overrides={"runtime": {"type": harness_runtime}},
         output_dir=tmp_path,
         max_turns=8,
@@ -167,7 +167,7 @@ async def test_shared_tool_isolation(
         )
     traces = await run_v1_server(
         "scratchpad-v1",
-        harness="default",
+        harness="null",
         harness_overrides={"runtime": {"type": harness_runtime}},
         output_dir=tmp_path,
         num_tasks=2,  # two distinct words, run concurrently against the one shared server
@@ -195,7 +195,7 @@ async def test_tool_response_image(run_v1, tmp_path):
     """MCP image content from a tool result survives into the v1 trace (needs a vision model)."""
     (trace,) = await run_v1(
         "tool-response-image-v1",
-        harness="default",
+        harness="null",
         harness_overrides={"runtime": {"type": "subprocess"}},
         model="qwen/qwen3-vl-8b-instruct",
         output_dir=tmp_path,
@@ -209,9 +209,9 @@ async def test_tool_response_image(run_v1, tmp_path):
 @pytest.mark.e2e
 async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
     """Agentic: write a phrase to a file with the agent's shell, checked in the runtime."""
-    if harness == "default":
+    if harness == "null":
         pytest.skip(
-            "default is a chat loop with no shell — it can't do the file-write task"
+            "null is a chat loop with no shell — it can't do the file-write task"
         )
     (trace,) = await run_v1(
         "echo-agentic-v1",

{verifiers-0.1.15.dev407 → verifiers-0.1.15.dev409}/verifiers/v1/GUIDE.md

@@ -552,15 +552,16 @@ Otherwise pick a built-in, selected with `--harness.id`:
 
 | id | what it is |
 | --- | --- |
-| `default` | a tiny OpenAI chat loop (MCP tools only, no tools of its own) |
-| `bash` | the `default` chat loop plus a local `bash` tool, for shell-driving agents |
+| `default` | a `bash` + `edit` coding agent (the `edit` tool is on by default — `--harness.edit false` for bash-only) — the fallback when no harness is given |
+| `null` | a tiny OpenAI chat loop (MCP tools only, no tools of its own) |
 | `rlm` | the RLM CLI agent |
 | `codex` | the Codex CLI (Responses dialect + SSE relay) |
 | `mini-swe-agent` | the mini-swe-agent CLI (a minimal SWE agent) |
 | `kimi-code` | the Kimi Code CLI agent |
 
 ```bash
-uv run eval gsm8k-v1 -n 1                    # default harness
+uv run eval gsm8k-v1 -n 1                    # default harness (bash + edit; the fallback)
+uv run eval gsm8k-v1 -n 1 --harness.id null  # bare chat loop, no local tools
 uv run eval gsm8k-v1 -n 1 --harness.id rlm   # same taskset, different driver
 ```
 
@@ -792,7 +793,7 @@ form. In a prime-rl config:
 [[orchestrator.train.env]]
 name    = "gsm8k"
 taskset = { id = "gsm8k-v1", dataset_name = "..." }                 # any v1 taskset id
-harness = { id = "default", runtime = { type = "subprocess" } }
+harness = { id = "null", runtime = { type = "subprocess" } }
 timeout = { scoring = 10 }                                          # per-stage cap (default: no limit)
 # pool  = { type = "elastic", max_workers = 8, multiplex = 128 }    # env-server pool (default elastic, self-sizing)
 ```

{verifiers-0.1.15.dev407 → verifiers-0.1.15.dev409}/verifiers/v1/README.md

@@ -96,7 +96,8 @@ Harness examples (under `environments/`):
 The program that drives the rollout — same taskset, different driver:
 
 ```bash
-uv run eval gsm8k-v1 -n 1                     # default: bare agent (MCP tools only)
+uv run eval gsm8k-v1 -n 1                     # default harness (fallback): a bash + edit agent
+uv run eval gsm8k-v1 -n 1 --harness.id null   # bare chat loop, no local tools (MCP tools only)
 uv run eval gsm8k-v1 -n 1 --harness.id rlm    # the rlm harness
 uv run eval gsm8k-v1 -n 1 --harness.id codex  # the codex harness
 ```
@@ -106,7 +107,7 @@ filesystem, so run it under a containerized runtime: `docker` locally, or a remo
 `modal` sandbox (not the default `subprocess`). 
 
 ```bash
-uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id bash            # bash-only agent
+uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id default         # the bash + edit agent
 uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id mini-swe-agent  # the mini-swe-agent CLI
 uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id rlm             # the rlm CLI agent
 uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id codex           # the codex CLI agent

{verifiers-0.1.15.dev407 → verifiers-0.1.15.dev409}/verifiers/v1/cli/dashboard/eval.py

@@ -13,6 +13,7 @@ above a rule.
 import contextlib
 import time
 
+from pydantic import BaseModel
 from rich.console import Console, Group
 from rich.markup import escape
 from rich.progress_bar import ProgressBar
@@ -26,7 +27,12 @@ from verifiers.v1.configs.eval import EvalConfig
 from verifiers.v1.rollout import Phase, Rollout
 from verifiers.v1.trace import Trace
 from verifiers.v1.types import Usage
-from verifiers.v1.utils.format import format_count, format_mean, format_time
+from verifiers.v1.utils.format import (
+    format_count,
+    format_mean,
+    format_override,
+    format_time,
+)
 from verifiers.utils.pricing_utils import format_cost_usd
 
 # For sizing pages to the terminal: detects the real terminal height/width each access (the live
@@ -109,9 +115,9 @@ def _aligned(rows: list[list[str]]) -> list[str]:
 
 
 def _warning(config: EvalConfig) -> Text | None:
-    """A local-runtime caution for a code-running harness (none for the tool-less `default`),
+    """A local-runtime caution for a code-running harness (none for the tool-less `null`),
     shown above the overview rather than as a row in it."""
-    if config.harness.id != "default" and config.harness.runtime.type == "subprocess":
+    if config.harness.id != "null" and config.harness.runtime.type == "subprocess":
         return Text(
             "warning  Runs on the local system; local files and settings may affect this "
             "evaluation. Use subprocess only for debugging, or use docker or prime for an "
@@ -121,6 +127,54 @@ def _warning(config: EvalConfig) -> Text | None:
     return None
 
 
+def overrides(
+    config: BaseModel,
+    default: BaseModel | None = None,
+    skip: frozenset[str] = frozenset(),
+) -> list[str]:
+    """`field=value` segments for the fields the *user* customized, sorted, diffed against each
+    field's declared default. Not `model_fields_set`: a `--resume` run reloads its config via
+    `model_validate(config.toml)`, and that toml is dumped with `exclude_none` (every field), so
+    `model_fields_set` would flag them all. `default` is the reference instance, threaded through
+    recursion so a pinned nested default (a taskset's `user=UserConfig(colocated=True)`) reads as
+    unchanged. `skip` holds dotted paths (`harness.runtime.type`)."""
+    segments: list[str] = []
+    fields = type(config).model_fields
+    for field in sorted(fields):
+        if field in skip:
+            continue
+        value = getattr(config, field)
+        # `get_default` returns `PydanticUndefined` for a required field, so it always reads as set.
+        field_def = (
+            fields[field].get_default(call_default_factory=True)
+            if default is None
+            else getattr(default, field, None)
+        )
+        if isinstance(value, BaseModel):  # nested config: flatten as `field.<sub>`
+            child_skip = frozenset(
+                s[len(field) + 1 :] for s in skip if s.startswith(f"{field}.")
+            )
+            # A switched discriminator (e.g. subprocess→docker) is an override the per-field diff
+            # misses — within the new class `type` equals its own default — so surface it.
+            class_changed = type(value) is not type(field_def)
+            if (
+                class_changed
+                and "type" not in child_skip
+                and (discriminator := getattr(value, "type", None)) is not None
+            ):
+                segments.append(f"{field}.type={format_override(discriminator)}")
+            # A switched class is a new shape, so diff against its own defaults, not the instance.
+            segments.extend(
+                f"{field}.{seg}"
+                for seg in overrides(
+                    value, default=None if class_changed else field_def, skip=child_skip
+                )
+            )
+        elif value != field_def:
+            segments.append(f"{field}={format_override(value)}")
+    return segments
+
+
 def Overview(config: EvalConfig) -> Table:
     sampling = ", ".join(
         f"{k}={v}" for k, v in config.sampling.model_dump(exclude_none=True).items()
@@ -134,6 +188,16 @@ def Overview(config: EvalConfig) -> Table:
     )
     model = f"{config.model}  ({sampling})" if sampling else config.model
     grid.add_row("model", f"{model}  via {config.client.base_url}")
+    # Non-default knobs the user set, one row each when non-empty. `escape` the cell: an override
+    # value (or our `[...]`/`{...}` delimiters) can carry Rich markup that would otherwise be
+    # parsed as styling and dropped. `id` is in the `env` row; harness `runtime.type` too (hidden
+    # here), but only for the harness — a taskset's `user.runtime.type` has no other display.
+    if taskset_over := overrides(config.taskset, skip=frozenset({"id"})):
+        grid.add_row("taskset", escape("  ·  ".join(taskset_over)))
+    if harness_over := overrides(
+        config.harness, skip=frozenset({"id", "runtime.type"})
+    ):
+        grid.add_row("harness", escape("  ·  ".join(harness_over)))
     limits, timeouts = _aligned([_limits(config), _timeouts(config)])
     grid.add_row("limits", limits)
     grid.add_row("timeouts", timeouts)
@@ -190,10 +254,12 @@ def _breakdown(done: list[Trace]) -> Table | None:
             names.extend(n for n in getattr(trace, source) if n not in names)
         if not names:
             continue
-        segments = [
-            f"{name} {format_mean(done, lambda t, n=name, s=source: getattr(t, s).get(n, 0.0))}"
-            for name in names
-        ]
+        segments = []
+        for name in names:
+            mean = format_mean(
+                done, lambda t, n=name, s=source: getattr(t, s).get(n, 0.0)
+            )
+            segments.append(f"{name} {mean}")
         grid.add_row(label, "  ·  ".join(segments))
 
     # Resource use over every completed rollout (errored ones still spent tokens/time): tokens and

{verifiers-0.1.15.dev407 → verifiers-0.1.15.dev409}/verifiers/v1/env.py

@@ -262,7 +262,11 @@ class Environment:
                 "(NEEDS_CONTAINER), but the harness runs on the subprocess runtime; "
                 "use --harness.runtime.type docker or prime."
             )
-        if self.harness.config.id != "default" and isinstance(
+        # The warning is about the *agent* running arbitrary code on the host: every harness hands
+        # it local execution (bash/edit, or a CLI agent) except the tool-less `null` chat loop,
+        # whose program only relays the model and remote MCP tools — so exempt `null`, warn for the
+        # rest. (`null` still runs its fixed chat-loop program locally, but nothing agent-authored.)
+        if self.harness.config.id != "null" and isinstance(
             self.harness.config.runtime, SubprocessConfig
         ):
             logger.warning(

{verifiers-0.1.15.dev407 → verifiers-0.1.15.dev409}/verifiers/v1/harnesses/__init__.py

@@ -2,7 +2,6 @@
 
 Re-exports each harness's class + config off the package."""
 
-from verifiers.v1.harnesses.bash import BashHarness, BashHarnessConfig
 from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig
 from verifiers.v1.harnesses.default import DefaultHarness, DefaultHarnessConfig
 from verifiers.v1.harnesses.kimi_code import KimiCodeHarness, KimiCodeHarnessConfig
@@ -10,18 +9,19 @@ from verifiers.v1.harnesses.mini_swe_agent import (
     MiniSWEAgentHarness,
     MiniSWEAgentHarnessConfig,
 )
+from verifiers.v1.harnesses.null import NullHarness, NullHarnessConfig
 from verifiers.v1.harnesses.rlm import RLMHarness, RLMHarnessConfig
 from verifiers.v1.harnesses.terminus_2 import Terminus2Harness, Terminus2HarnessConfig
 
 __all__ = [
-    "BashHarness",
-    "BashHarnessConfig",
     "CodexHarness",
     "CodexHarnessConfig",
     "DefaultHarness",
     "DefaultHarnessConfig",
     "KimiCodeHarness",
     "KimiCodeHarnessConfig",
+    "NullHarness",
+    "NullHarnessConfig",
     "MiniSWEAgentHarness",
     "MiniSWEAgentHarnessConfig",
     "RLMHarness",

verifiers-0.1.15.dev409/verifiers/v1/harnesses/default/__init__.py

@@ -0,0 +1,11 @@
+"""default — v1's built-in agentic harness (the fallback when no `--harness.id` is given): a chat
+loop with a local `bash` tool and an optional `edit` tool (single-occurrence string replacement,
+on by default). Its `harness.py` (class + config) and the `program.py` script staged into the
+runtime. Resolved by id via `load_harness`."""
+
+from verifiers.v1.harnesses.default.harness import (
+    DefaultHarness,
+    DefaultHarnessConfig,
+)
+
+__all__ = ["DefaultHarness", "DefaultHarnessConfig"]

{verifiers-0.1.15.dev407/verifiers/v1/harnesses/bash_edit → verifiers-0.1.15.dev409/verifiers/v1/harnesses/default}/harness.py

@@ -1,11 +1,11 @@
-"""The built-in bash_edit harness: the `bash` chat loop plus a local `edit` tool.
+"""The built-in default harness: a chat loop with a local `bash` tool and an optional `edit` tool.
 
-Same growing-message-list chat loop as the `bash` harness — a local `bash` tool that runs shell
-commands in the runtime, plus the taskset's MCP tools — with one extra local tool: `edit`, a
-single-occurrence string replacement in a file (ported from the rlm `edit` skill). For agentic
-tasks where targeted file edits are common and a model handles `edit` more reliably than
-hand-built `sed`/heredoc shell. Its uv script (deps: openai, mcp) is prepared during setup, then
-launched as the harness program.
+A growing-message-list chat loop with a local `bash` tool that runs shell commands in the runtime,
+the taskset's MCP tools, and — on by default, gated by the `edit` config flag — a local `edit`
+tool for single-occurrence string replacement in a file (ported from the rlm `edit` skill; a model
+handles it more reliably than hand-built `sed`/heredoc shell). This is the fallback harness when no
+`--harness.id` is given. Its uv script (deps: openai, mcp) is prepared during setup, then launched
+as the harness program. For a pure chat loop with no local tools, use the `null` harness.
 """
 
 import json
@@ -20,19 +20,25 @@ from verifiers.v1.trace import Trace
 PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
 
 # Frames the model as a coding agent and names its local tools (a pure-text chat loop gets no
-# harness-injected prompt).
-BASH_EDIT_SYSTEM_PROMPT = (
-    "You are a coding agent. You have access to a bash tool for running shell commands and "
-    "an edit tool for single-occurrence string replacement in a file."
+# harness-injected prompt). The edit clause is appended only when the `edit` tool is enabled.
+BASH_SYSTEM_PROMPT = (
+    "You are a coding agent. You have access to a bash tool for running shell commands."
+)
+EDIT_SYSTEM_PROMPT = (
+    "You also have an edit tool for single-occurrence string replacement in a file."
 )
 
 
-class BashEditHarnessConfig(HarnessConfig):
-    """The built-in bash_edit harness. A uv script (deps: openai, mcp), so it runs in any runtime
+class DefaultHarnessConfig(HarnessConfig):
+    """The built-in default harness. A uv script (deps: openai, mcp), so it runs in any runtime
     that has `uv` (the harness bootstraps it) with no other setup."""
 
+    edit: bool = True
+    """Offer the local `edit` tool (single-occurrence string replacement in a file) alongside
+    `bash`. On by default; set `--harness.edit false` for a bash-only agent."""
+
 
-class BashEditHarness(Harness[BashEditHarnessConfig]):
+class DefaultHarness(Harness[DefaultHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = True
     SUPPORTS_MCP = True
     SUPPORTS_USER_SIM = True
@@ -51,9 +57,10 @@ class BashEditHarness(Harness[BashEditHarnessConfig]):
         mcp_urls: dict[str, str],
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(trace.task)
-        system_prompt = "\n\n".join(
-            p for p in (BASH_EDIT_SYSTEM_PROMPT, system_prompt) if p
-        )
+        harness_prompt = BASH_SYSTEM_PROMPT
+        if self.config.edit:
+            harness_prompt = f"{BASH_SYSTEM_PROMPT} {EDIT_SYSTEM_PROMPT}"
+        system_prompt = "\n\n".join(p for p in (harness_prompt, system_prompt) if p)
         env = {**self.config.resolved_env}
         args = [
             f"--base-url={endpoint}",
@@ -61,6 +68,8 @@ class BashEditHarness(Harness[BashEditHarnessConfig]):
             f"--model={ctx.model}",
             f"--system-prompt={system_prompt}",
         ]
+        if self.config.edit:
+            args.append("--edit")
         if mcp_urls:
             # The program connects to the tool servers over HTTP; hand it a standard
             # `mcpServers` URL config (the `mcp` client itself comes from the uv deps).

{verifiers-0.1.15.dev407/verifiers/v1/harnesses/bash_edit → verifiers-0.1.15.dev409/verifiers/v1/harnesses/default}/program.py

@@ -2,13 +2,13 @@
 # requires-python = ">=3.10"
 # dependencies = ["openai", "mcp"]
 # ///
-"""The bash_edit harness's program: a chat loop with local `bash` + `edit` tools (+ optional MCP).
+"""The default harness's program: a chat loop with a local `bash` tool (+ optional `edit`, MCP).
 
 A growing-message-list chat loop. It always offers a local `bash` tool that runs shell commands in
-the runtime and a local `edit` tool that replaces a unique string in a file; when the harness sets
-MCP_CONFIG (a standard `mcpServers` URL map) it also connects to those servers over streamable
-HTTP, exposes their tools to the model as `<server>_<tool>`, and routes those calls to the server.
-The loop runs until the model answers without a tool call.
+the runtime; with `--edit` it also offers a local `edit` tool that replaces a unique string in a
+file. When the harness sets MCP_CONFIG (a standard `mcpServers` URL map) it also connects to those
+servers over streamable HTTP, exposes their tools to the model as `<server>_<tool>`, and routes
+those calls to the server. The loop runs until the model answers without a tool call.
 
 It runs as a uv script (deps: openai, mcp), so the chat + tool plumbing is just the SDKs — the
 harness bootstraps `uv` in the runtime. The interception endpoint, per-rollout secret, and model
@@ -182,6 +182,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--system-prompt", default="")
     parser.add_argument("--prompt", default="")
     parser.add_argument("--mcp-config", default="")
+    parser.add_argument("--edit", action="store_true")
     return parser.parse_args()
 
 
@@ -193,7 +194,10 @@ async def main() -> None:
         mcp_tools, dispatch = (
             await connect_mcp(stack, config) if config.get("mcpServers") else ([], {})
         )
-        tools = [BASH_TOOL, EDIT_TOOL] + mcp_tools
+        tools = [BASH_TOOL]
+        if args.edit:
+            tools.append(EDIT_TOOL)
+        tools += mcp_tools
         messages = (
             [{"role": "system", "content": args.system_prompt}]
             if args.system_prompt
@@ -226,13 +230,24 @@ async def main() -> None:
                         }
                     )
                     continue
+                # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls
+                # below assume a dict, so reject anything else as a tool error rather than crashing.
+                if not isinstance(tool_args, dict):
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "tool_call_id": call.id,
+                            "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
+                        }
+                    )
+                    continue
                 if name in dispatch:
                     content = await call_mcp(dispatch, name, tool_args)
                 elif name == "bash":
                     content = await asyncio.to_thread(
                         run_bash, tool_args.get("command", "")
                     )
-                elif name == "edit":
+                elif name == "edit" and args.edit:
                     content = await asyncio.to_thread(
                         run_edit,
                         tool_args.get("path"),

verifiers-0.1.15.dev409/verifiers/v1/harnesses/null/__init__.py

@@ -0,0 +1,7 @@
+"""null — v1's built-in tool-less harness: its `harness.py` (class + config) and the
+`program.py` script it stages into the runtime. A plain chat loop with the taskset's MCP tools
+and no tools of its own. Resolved by id via `load_harness`."""
+
+from verifiers.v1.harnesses.null.harness import NullHarness, NullHarnessConfig
+
+__all__ = ["NullHarness", "NullHarnessConfig"]

{verifiers-0.1.15.dev407/verifiers/v1/harnesses/default → verifiers-0.1.15.dev409/verifiers/v1/harnesses/null}/harness.py

@@ -1,9 +1,10 @@
-"""The built-in default harness: runs a small chat-loop program as a uv script.
+"""The built-in null harness: runs a small chat-loop program as a uv script, no tools of its own.
 
 A growing-message-list chat loop with the taskset's MCP tools (host-side, resolved to URLs by
-the Environment) — and no tools of its own. Its uv script (deps: openai, mcp) is prepared
-during setup, then launched as the harness program. For a shell-driving agent, use a
-dedicated agentic harness (e.g. `mini-swe-agent`).
+the Environment) — and no tools of its own (it's "null" precisely because it adds no
+harness-side tooling). Its uv script (deps: openai, mcp) is prepared during setup, then launched
+as the harness program. For a shell-driving agent, use a dedicated agentic harness (e.g.
+`mini-swe-agent`).
 """
 
 import json
@@ -18,12 +19,12 @@ from verifiers.v1.trace import Trace
 PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
 
 
-class DefaultHarnessConfig(HarnessConfig):
-    """The built-in harness. A uv script (deps: openai, mcp), so it runs in any runtime that
+class NullHarnessConfig(HarnessConfig):
+    """The built-in null harness. A uv script (deps: openai, mcp), so it runs in any runtime that
     has `uv` (the harness bootstraps it) with no other setup."""
 
 
-class DefaultHarness(Harness[DefaultHarnessConfig]):
+class NullHarness(Harness[NullHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = True
     SUPPORTS_MCP = True
     SUPPORTS_USER_SIM = True

{verifiers-0.1.15.dev407/verifiers/v1/harnesses/default → verifiers-0.1.15.dev409/verifiers/v1/harnesses/null}/program.py

@@ -2,7 +2,7 @@
 # requires-python = ">=3.10"
 # dependencies = ["openai", "mcp"]
 # ///
-"""The default harness's program: a chat loop with the taskset's MCP tools (and none of its own).
+"""The null harness's program: a chat loop with the taskset's MCP tools (and none of its own).
 
 A growing-message-list chat loop. When the harness sets MCP_CONFIG (a standard `mcpServers` URL
 map) it connects to those servers over streamable HTTP, exposes their tools to the model as
@@ -142,6 +142,17 @@ async def main() -> None:
                         }
                     )
                     continue
+                # Valid JSON can still be a non-object (`[]`, `42`, `null`); the MCP dispatch
+                # assumes a dict, so reject anything else as a tool error rather than crashing.
+                if not isinstance(tool_args, dict):
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "tool_call_id": call.id,
+                            "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
+                        }
+                    )
+                    continue
                 if name in dispatch:
                     content = await call_mcp(dispatch, name, tool_args)
                 else:

{verifiers-0.1.15.dev407 → verifiers-0.1.15.dev409}/verifiers/v1/loaders.py

@@ -130,8 +130,9 @@ def harness_class(harness_id: str) -> type[Harness]:
 def default_harness_id(taskset_id: str) -> str:
     """The harness id to use when none is given. A taskset that bundles its own harness — its
     module also exports a `Harness` subclass via `__all__`, so the taskset id doubles as the
-    harness id — runs with that harness by default; otherwise the shared `default` harness. An
-    explicit `--harness.id` / toml id always takes precedence (this only supplies the fallback)."""
+    harness id — runs with that harness by default; otherwise the shared `default` harness (a
+    bash + edit agent). An explicit `--harness.id` / toml id always takes precedence (this only
+    supplies the fallback); for a tool-less chat loop, pass `--harness.id null`."""
     if not taskset_id:
         return "default"
     try:

{verifiers-0.1.15.dev407 → verifiers-0.1.15.dev409}/verifiers/v1/utils/format.py

@@ -34,6 +34,15 @@ def format_count(n: int) -> str:
     return f"{n / 1e6:.1f}M"
 
 
+def format_override(value: object) -> str:
+    """A value as one compact segment: dict `{k=v,k=v}`, list/tuple `[a,b]`, else `str`."""
+    if isinstance(value, dict):
+        return "{" + ",".join(f"{k}={v}" for k, v in value.items()) + "}"
+    if isinstance(value, (list, tuple)):
+        return "[" + ",".join(str(v) for v in value) + "]"
+    return str(value)
+
+
 def format_mean(
     traces: list[Trace], value: Callable[[Trace], float], digits: int = 2
 ) -> str:

verifiers-0.1.15.dev407/verifiers/v1/harnesses/bash/__init__.py

@@ -1,7 +0,0 @@
-"""bash — v1's built-in agentic harness: the `default` chat loop plus a single local `bash`
-tool. Its `harness.py` (class + config) and the `program.py` script staged into the runtime.
-Resolved by id via `load_harness`."""
-
-from verifiers.v1.harnesses.bash.harness import BashHarness, BashHarnessConfig
-
-__all__ = ["BashHarness", "BashHarnessConfig"]