toolbench 0.2.0__tar.gz → 0.3.0__tar.gz

toolbench-0.3.0/CHANGELOG.md

@@ -0,0 +1,42 @@
+# Changelog
+
+All notable changes to `toolbench` will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/), and this
+project adheres to [Semantic Versioning](https://semver.org/).
+
+---
+
+## [0.3.0] — 2026-07-24
+
+### Added
+
+- **MCP preflight verification.** Before any trial runs (and in `--dry-run`), a
+  run now verifies that every MCP-serving harness × tools-loadout can actually
+  reach its tools: it starts the loadout's `toolbase serve --profile …` exactly
+  as the CLI runtime would and completes an MCP `initialize` + `tools/list`
+  handshake. If the profile serves zero tools, the handshake times out, or the
+  resolved tools are missing, the run **aborts with exit 2 before executing a
+  single trial** and prints which cell failed. Previously a `tools` loadout that
+  could not reach its tools ran the entire arm silently tool-less and still
+  graded as a valid tools result. Verification is gated to the runtimes that
+  serve toolbase over MCP (`claude_code`, `codex`) via
+  `runtime_serves_toolbase_mcp`; in-process runtimes (orchestral) resolve tools
+  directly and are unaffected. New helpers: `verify_toolbase_mcp`,
+  `runtime_serves_toolbase_mcp`.
+
+### Fixed
+
+- **`_toolbase_command()` now fails loudly.** When the `toolbase` executable
+  cannot be located (neither on `PATH` nor beside the running Python), it raises
+  with an actionable message instead of returning a bare `"toolbase"`. The bare
+  fallback silently produced a broken `.mcp.json` whose stdio server failed to
+  launch (`command not found` on the child's `PATH`), leaving the agent with
+  "No such tool available" while the trial still graded as if tools were
+  present — a failure mode that surfaced when an env's `toolbase` install was
+  in flux. Combined with the preflight above, an unresolvable toolbase is now
+  caught before the benchmark begins.
+
+## [0.2.1] — 2026-07-24
+
+- Campaign runtime & metrics hardening.

{toolbench-0.2.0 → toolbench-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toolbench
-Version: 0.2.0
+Version: 0.3.0
 Summary: A platform and CLI for building benchmarks for agentic tools and harnesses.
 Project-URL: Homepage, https://github.com/tonymenzo/toolbench
 Project-URL: Repository, https://github.com/tonymenzo/toolbench

{toolbench-0.2.0 → toolbench-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "toolbench"
-version = "0.2.0"
+version = "0.3.0"
 description = "A platform and CLI for building benchmarks for agentic tools and harnesses."
 readme = "README.md"
 requires-python = ">=3.12"

{toolbench-0.2.0 → toolbench-0.3.0}/tests/test_codex_runtime.py

@@ -4,7 +4,7 @@ from pathlib import Path
 
 from toolbench.core.harness import Harness
 from toolbench.core.llm_factory import SubscriptionLLM
-from toolbench.core.runtime import CodexAgent, _codex_factory
+from toolbench.core.runtime import CodexAgent, _codex_factory, _toolbase_command
 
 
 def _agent(tmp_path: Path, **kw) -> CodexAgent:
@@ -29,6 +29,12 @@ def test_toolbase_mcp_is_auto_approved_for_noninteractive_exec(tmp_path):
     assert any("default_tools_approval_mode=\"approve\"" in x for x in args)
 
 
+def test_toolbase_mcp_command_resolves_to_an_executable():
+    command = Path(_toolbase_command())
+    assert command.is_absolute()
+    assert command.is_file()
+
+
 def test_resume_uses_only_resume_supported_flags(tmp_path):
     a = _agent(tmp_path)
     a.thread_id = "thread-123"

{toolbench-0.2.0 → toolbench-0.3.0}/tests/test_metrics.py

@@ -242,6 +242,18 @@ class TestCostUsd(unittest.TestCase):
         self.assertAlmostEqual(estimate["usd"], 56.0)
         self.assertTrue(estimate["long_context_pricing_applied"])
 
+    def test_gpt56_sol_subscription_api_equivalent(self):
+        estimate = subscription_api_equivalent_cost(
+            "gpt-5.6-sol",
+            input_tokens=1_000_000,
+            cache_read_tokens=1_000_000,
+            output_tokens=1_000_000,
+            initial_input_tokens=272_001,
+        )
+        self.assertAlmostEqual(estimate["usd"], 56.0)
+        self.assertTrue(estimate["long_context_pricing_applied"])
+        self.assertIn("gpt-5.6-sol", estimate["source"])
+
     def test_unknown_subscription_model_is_not_guessed(self):
         self.assertIsNone(subscription_api_equivalent_cost("future-model"))
 

toolbench-0.3.0/tests/test_runner_toolbase_protection.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+from toolbench.core.runner import _toolbase_protected_paths
+
+
+def test_toolbase_and_editable_sources_are_protected(tmp_path, monkeypatch):
+    toolbase_home = tmp_path / ".toolbase"
+    slot = toolbase_home / "cache" / "physics-kit" / "editable"
+    slot.mkdir(parents=True)
+    source = tmp_path / "physics-kit-source"
+    source.mkdir()
+    runtime_data = tmp_path / "physics-kit-runtime"
+    runtime_data.mkdir()
+    (slot / ".install_meta.yaml").write_text(
+        "editable: true\n"
+        f"source_path: {source}\n"
+    )
+    monkeypatch.setenv("TOOLBASE_HOME", str(toolbase_home))
+    config_dir = toolbase_home / "config"
+    config_dir.mkdir()
+    (config_dir / "physics-kit.yaml").write_text(
+        f"base_directory: {runtime_data}\n"
+    )
+
+    paths = _toolbase_protected_paths()
+
+    assert str(toolbase_home.resolve()) in paths
+    assert str(source.resolve()) in paths
+    assert str(runtime_data.resolve()) in paths

{toolbench-0.2.0 → toolbench-0.3.0}/toolbench/__init__.py

@@ -2,7 +2,7 @@
 
 from pathlib import Path
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 
 # Repository root (the directory that contains this `toolbench/` package in an
 # editable install). The CLI reads `REPO_ROOT/.env` for provider keys + tool

{toolbench-0.2.0 → toolbench-0.3.0}/toolbench/cli.py

@@ -598,6 +598,52 @@ def cmd_run(args: argparse.Namespace) -> int:
                         release_sources(tmp)
         manifest["resolution"] = resolution_reports
         write_json(run_dir / "manifest.json", manifest)
+
+        # MCP preflight: for every MCP-serving harness x loadout that serves a
+        # toolbase profile, actually start `toolbase serve` and complete a
+        # tools/list handshake BEFORE any trial. A profile that resolves but
+        # serves no tools (mis-wired toolbase command, env churn) otherwise runs
+        # the entire "tools" arm silently tool-less and still grades it as valid.
+        # Hard-fail the run here instead; runs verbatim in dry-run too.
+        from toolbench.core.runtime import (
+            runtime_serves_toolbase_mcp, verify_toolbase_mcp,
+            _loadout_toolbase_profile)
+        mcp_failures: list[str] = []
+        for h in harnesses:
+            if not runtime_serves_toolbase_mcp(h.runtime_name):
+                continue
+            for lo in loadouts:
+                profile, proj = _loadout_toolbase_profile(lo)
+                if not profile:
+                    continue
+                expected = [t for r in resolution_reports
+                            if r.get("harness") == h.id and r.get("loadout") == lo.name
+                            for s in r.get("sources", [])
+                            if s.get("backend") == "toolbase"
+                            for t in s.get("tools", [])]
+                try:
+                    served = verify_toolbase_mcp(profile, cwd=(proj or bench_dir))
+                    missing = [t for t in expected if t not in served]
+                    if missing:
+                        mcp_failures.append(
+                            f"{h.id}/{lo.name} (profile {profile}): server served "
+                            f"{len(served)} tools, missing {missing}")
+                    else:
+                        print(f"  MCP preflight OK: {h.id}/{lo.name} — "
+                              f"{len(served)} tools served ({profile})")
+                except Exception as e:
+                    mcp_failures.append(
+                        f"{h.id}/{lo.name} (profile {profile}): "
+                        f"{type(e).__name__}: {e}")
+        if mcp_failures:
+            print("\n  MCP PREFLIGHT FAILED — aborting before any trial ran:")
+            for f in mcp_failures:
+                print(f"    ✗ {f}")
+            print("  A `tools` loadout could not reach its tools. Check that "
+                  "`toolbase` is installed in this env and the profile serves "
+                  "tools, then re-run.")
+            return 2
+
         if args.dry_run:
             print("  DRY-RUN: agent.run() will be skipped.")
 

{toolbench-0.2.0 → toolbench-0.3.0}/toolbench/core/metrics.py

@@ -73,6 +73,15 @@ SUBSCRIPTION_API_EQUIVALENT_PRICING: dict[tuple[str, str], dict[str, float | int
         "long_output_multiplier": 1.5,
         "source": "https://developers.openai.com/api/docs/models/gpt-5.5",
     },
+    ("openai", "gpt-5.6-sol"): {
+        "input": 5.00,
+        "cached_input": 0.50,
+        "output": 30.00,
+        "long_context_threshold": 272_000,
+        "long_input_multiplier": 2.0,
+        "long_output_multiplier": 1.5,
+        "source": "https://developers.openai.com/api/docs/models/gpt-5.6-sol",
+    },
 }
 
 

{toolbench-0.2.0 → toolbench-0.3.0}/toolbench/core/runner.py

@@ -19,6 +19,7 @@ import traceback
 from dataclasses import dataclass
 from pathlib import Path
 
+import yaml
 from orchestral.tools.hooks import TruncateOutputHook
 
 from .artifact_policy import DEFAULT_POLICY, ArtifactPolicy
@@ -67,6 +68,42 @@ _PROVIDER_CONTROL_KEYS = ("name", "cache_bust")
 CODE_ARG_KEYS = ("code", "script", "source", "program")
 
 
+def _toolbase_protected_paths() -> list[str]:
+    """Paths a benchmark agent must not inspect outside the MCP interface.
+
+    Toolbase may serve editable toolkit checkouts. A CLI agent with ordinary
+    filesystem read access could otherwise locate and import those exact tool
+    implementations in a ``core_only`` arm, contaminating the control. MCP
+    server children are not confined by Codex/Claude's shell filesystem
+    profile, so denying these paths to the agent does not prevent the declared
+    Toolbase loadout from executing them.
+    """
+    root = Path(os.environ.get("TOOLBASE_HOME", Path.home() / ".toolbase")).resolve()
+    protected = [str(root)]
+    for meta_path in root.glob("cache/*/*/.install_meta.yaml"):
+        try:
+            meta = yaml.safe_load(meta_path.read_text()) or {}
+            source = meta.get("source_path") if meta.get("editable") else None
+            if source:
+                protected.append(str(Path(source).expanduser().resolve()))
+        except Exception:
+            continue
+    # Toolkit runtime configs can point at persistent working/data directories.
+    # Those may contain outputs from earlier tool-assisted runs and are equally
+    # inappropriate inputs to a fresh or core-only benchmark trial.
+    for config_path in root.glob("config/*.yaml"):
+        try:
+            config = yaml.safe_load(config_path.read_text()) or {}
+            base_directory = config.get("base_directory")
+            if base_directory:
+                protected.append(
+                    str(Path(base_directory).expanduser().resolve())
+                )
+        except Exception:
+            continue
+    return list(dict.fromkeys(protected))
+
+
 def _warn(prefix: str, exc: BaseException) -> None:
     """Emit a one-line stderr breadcrumb for a non-fatal failure.
 
@@ -452,8 +489,11 @@ class TrialRunner:
                     # The benchmark tree holds the ground-truth answer key
                     # (soln/); the Bash sandbox (if the harness enables it)
                     # deny-reads these so a trial cannot reach it.
-                    protected_paths=(benchmark_dir if isinstance(benchmark_dir, list)
-                                     else [benchmark_dir]) if benchmark_dir else [],
+                    protected_paths=(
+                        ((benchmark_dir if isinstance(benchmark_dir, list)
+                          else [benchmark_dir]) if benchmark_dir else [])
+                        + _toolbase_protected_paths()
+                    ),
                 )
                 # One resume loop over the SAME agent / sandbox / context.
                 # After each agent.run we either:

{toolbench-0.2.0 → toolbench-0.3.0}/toolbench/core/runtime.py

@@ -185,6 +185,35 @@ register_runtime("orchestral", _orchestral_factory, dist="orchestral-ai")
 
 # The MCP server name `tb connect claude-code` writes into .mcp.json.
 _TOOLBASE_MCP_SERVER = "toolbase"
+
+
+def _toolbase_command() -> str:
+    """Resolve the Toolbase executable for child MCP processes.
+
+    Invoking Toolbench with an absolute virtual-environment Python does not
+    activate that environment or prepend its ``bin`` directory to ``PATH``.
+    Prefer PATH when available, then the executable beside the running Python.
+
+    Raises ``RuntimeError`` when Toolbase cannot be located, instead of
+    returning a bare ``"toolbase"``. A bare command is what silently broke a
+    campaign: a child MCP process launched as ``toolbase serve ...`` failed to
+    start (``command not found`` on the child's PATH), the agent merely saw
+    "No such tool available", and the trial still graded as if it had tools.
+    Fail loudly here — the run preflight (`verify_toolbase_mcp`) turns this into
+    an abort before any trial runs.
+    """
+    found = shutil.which("toolbase")
+    if found:
+        return found
+    sibling = Path(sys.executable).resolve().with_name("toolbase")
+    if sibling.is_file():
+        return str(sibling)
+    raise RuntimeError(
+        "toolbase executable not found: neither on PATH nor beside the running "
+        f"Python ({Path(sys.executable).resolve().with_name('toolbase')}). "
+        "Install toolbase into this environment (`pip install toolbase`); a CLI "
+        "runtime cannot serve the loadout's MCP tools without it."
+    )
 # No hardcoded toolkit: the MCP profile a CLI runtime serves is derived from the
 # benchmark's loadout (its `toolbase: {profile: ...}` source) via
 # `_loadout_toolbase_profile`. None => serve no MCP server (the `core` baseline
@@ -235,6 +264,109 @@ def _loadout_toolbase_profile(loadout) -> tuple[str | None, str | None]:
     return None, None
 
 
+# Runtimes that serve the loadout's toolbase profile to their agent over a
+# stdio MCP server (`toolbase serve --profile ...`) rather than resolving tools
+# in-process. For these the MCP connection is verified in the run preflight: a
+# profile that resolves but serves zero tools (a mis-wired toolbase command,
+# env churn) otherwise runs the whole "tools" arm silently tool-less. A new CLI
+# runtime that serves toolbase over MCP registers its name here.
+_MCP_SERVING_RUNTIMES = {"claude_code", "codex"}
+
+
+def runtime_serves_toolbase_mcp(runtime_name: str) -> bool:
+    """True when a harness's runtime drives its agent's tools through a
+    `toolbase serve` MCP child (claude-code, codex), so the run preflight must
+    verify the server actually serves its tools. In-process runtimes
+    (orchestral) resolve tools directly and need no MCP check."""
+    return (runtime_name or "").lower() in _MCP_SERVING_RUNTIMES
+
+
+def verify_toolbase_mcp(profile: str, *, cwd: str, call_timeout_s: int = 60,
+                        timeout: float = 45.0) -> list[str]:
+    """Start `toolbase serve --profile <profile>` exactly as a CLI runtime does
+    and complete an MCP initialize + tools/list handshake, returning the served
+    tool names.
+
+    Raises ``RuntimeError`` if the server cannot launch, the handshake times
+    out, or it serves zero tools. This is the preflight guard that a `tools`
+    loadout actually reaches its tools *before* any trial runs (a served-but-
+    empty toolset previously graded as a valid tools arm)."""
+    import queue
+    import threading
+    import time as _time
+
+    cmd = [_toolbase_command(), "serve", "--profile", profile,
+           "--call-timeout", str(call_timeout_s)]
+    try:
+        proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE, text=True, cwd=cwd, bufsize=1)
+    except Exception as e:
+        raise RuntimeError(f"could not launch `{' '.join(cmd)}`: {e}") from e
+
+    err_chunks: list[str] = []
+    threading.Thread(target=_drain_stream, args=(proc.stderr, err_chunks),
+                     daemon=True).start()
+    out_q: "queue.Queue" = queue.Queue()
+
+    def _read_out():
+        try:
+            for line in proc.stdout:  # type: ignore[union-attr]
+                out_q.put(line)
+        except Exception:
+            pass
+        out_q.put(None)
+
+    threading.Thread(target=_read_out, daemon=True).start()
+
+    def _send(obj):
+        proc.stdin.write(json.dumps(obj) + "\n")  # type: ignore[union-attr]
+        proc.stdin.flush()  # type: ignore[union-attr]
+
+    def _fail(msg):
+        tail = "".join(err_chunks)[-400:].strip()
+        raise RuntimeError(f"{msg} (server exit={proc.poll()}"
+                           + (f"; stderr: …{tail}" if tail else "") + ")")
+    try:
+        _send({"jsonrpc": "2.0", "id": 1, "method": "initialize",
+               "params": {"protocolVersion": "2024-11-05", "capabilities": {},
+                          "clientInfo": {"name": "toolbench-preflight",
+                                         "version": "0"}}})
+        _send({"jsonrpc": "2.0", "method": "notifications/initialized"})
+        _send({"jsonrpc": "2.0", "id": 2, "method": "tools/list"})
+        deadline = _time.monotonic() + timeout
+        while _time.monotonic() < deadline:
+            try:
+                line = out_q.get(timeout=1.0)
+            except queue.Empty:
+                if proc.poll() is not None:
+                    _fail(f"MCP server for profile {profile!r} exited before "
+                          "returning tools/list")
+                continue
+            if line is None:
+                _fail(f"MCP server for profile {profile!r} closed its output "
+                      "before returning tools/list")
+            try:
+                msg = json.loads(line)
+            except Exception:
+                continue
+            if msg.get("id") == 2:
+                tools = [t.get("name")
+                         for t in (msg.get("result") or {}).get("tools", [])]
+                if not tools:
+                    _fail(f"MCP server served 0 tools for profile {profile!r}")
+                return tools
+        _fail(f"MCP handshake for profile {profile!r} timed out after "
+              f"{timeout:.0f}s")
+    finally:
+        try:
+            proc.terminate(); proc.wait(timeout=5)
+        except Exception:
+            try:
+                proc.kill()
+            except Exception:
+                pass
+
+
 class _ClaudeCodeResponse:
     """Minimal response object matching the runner's defensive reads:
     `.text`, `.tool_calls`, and a `.context.messages` iterable."""
@@ -323,7 +455,7 @@ class ClaudeCodeAgent:
             "mcpServers": {
                 _TOOLBASE_MCP_SERVER: {
                     "type": "stdio",
-                    "command": "toolbase",
+                    "command": _toolbase_command(),
                     "args": args,
                 }
             }
@@ -601,18 +733,25 @@ def _apply_harness_env(env: dict, overrides) -> None:
 
 
 def _claude_code_factory(*, system_prompt, sandbox_dir=None, harness=None,
-                         loadout=None, tool_hooks=None, protected_paths=None,
-                         **_):
-    # `**_` absorbs the orchestral-shaped kwargs (llm, tools, display_hook)
-    # this runtime doesn't use. We DO use tool_hooks: the runner's
-    # TrajectoryHook is in there, and firing it per streamed tool call gives
-    # us the same console.log + transcript as orchestral.
+                         loadout=None, tool_hooks=None, llm=None,
+                         protected_paths=None, **_):
+    # `**_` absorbs the orchestral-shaped kwargs (tools, display_hook) this
+    # runtime doesn't use. We DO use tool_hooks (the runner's TrajectoryHook,
+    # for console.log + transcript parity with orchestral) and llm (it carries
+    # the run matrix's --models value; see below).
     if not sandbox_dir:
         raise ValueError(
             "claude_code runtime requires sandbox_dir (the runner passes it)."
         )
     model, call_timeout_s, profile, project_root, traj_hook, env_overrides = \
         _cli_runtime_common(harness, loadout, tool_hooks)
+    # The run matrix's --models value is carried by SubscriptionLLM and MUST
+    # override the harness's provider.model default; otherwise every cell runs
+    # the harness default model regardless of its --models label (mirrors the
+    # codex factory).
+    requested_model = getattr(llm, "model", None)
+    if requested_model:
+        model = requested_model
     return ClaudeCodeAgent(
         system_prompt=system_prompt, sandbox_dir=sandbox_dir, model=model,
         profile=profile, project_root=project_root,
@@ -721,7 +860,7 @@ class CodexAgent:
                       "--call-timeout", str(self.call_timeout_s)]
         return [
             "-c", f"mcp_servers.{_TOOLBASE_MCP_SERVER}.command="
-                  + json.dumps("toolbase"),
+                  + json.dumps(_toolbase_command()),
             "-c", f"mcp_servers.{_TOOLBASE_MCP_SERVER}.args="
                   + json.dumps(serve_args),
             # `codex exec` is noninteractive. Without a server-level default,