toolbench 0.1.0__tar.gz → 0.2.0__tar.gz

{toolbench-0.1.0 → toolbench-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toolbench
-Version: 0.1.0
+Version: 0.2.0
 Summary: A platform and CLI for building benchmarks for agentic tools and harnesses.
 Project-URL: Homepage, https://github.com/tonymenzo/toolbench
 Project-URL: Repository, https://github.com/tonymenzo/toolbench
@@ -88,10 +88,10 @@ axes — vary any of them on the command line to run an ablation:
 | Concept       | What it is                                                            | Where it lives          |
 |---------------|----------------------------------------------------------------------|-------------------------|
 | **Benchmark** | The task + grading rubric + ground truth.                            | `benchmark.yaml`        |
-| **Harness**   | The agent runtime, provider, core tools, and loop policy.           | `harnesses/*.yaml`      |
+| **Harness**   | The agent runtime (`orchestral`, `claude_code`, `codex`), provider, core tools, and loop policy. | `harnesses/*.yaml` |
 | **Loadout**   | The domain tools the agent gets (beyond the harness core).          | `loadouts/*.yaml`       |
 | **Variant**   | The prompt + sandbox seed (scaffolding axis), orthogonal to tools.  | `variants/<name>/`      |
-| **Rubric**    | Ordered, weighted stages of checks; trial score = prefix product.   | inside `benchmark.yaml` |
+| **Rubric**    | Ordered, weighted stages of checks; trial score = weighted reach.   | inside `benchmark.yaml` |
 
 A **loadout source** is one of:
 
@@ -113,11 +113,14 @@ A **loadout source** is one of:
 
 For each (model × condition) cell over `k` trials:
 
-- **reach̄ₖ** — mean rubric-weighted reach (how far through the staged pipeline the agent gets).
-- **pass@k** — probability at least one of k trials passes every stage (best-of-k).
-- **pass^k** — probability all k trials pass (worst-of-k).
+- $\overline{\text{reach}}_k$ — mean rubric-weighted reach: how far through the task the agent gets, on average.
+- $\text{pass@}k$ — probability that at least one of $k$ trials passes (best-of-$k$).
+- $\text{pass}^{k}$ — probability that all $k$ trials pass (worst-of-$k$).
 
-with bootstrap 95% confidence intervals and a metric-correlation matrix.
+with bootstrap 95% confidence intervals and a metric-correlation matrix. A trial *passes*
+when it clears the rubric's pass criterion — every stage by default, or reach ≥ a
+`pass_threshold` once the rubric uses partial-credit stages. See
+[Metrics](https://toolbench-ai.com/docs/reference/metrics/) for the exact estimators.
 
 ## Commands
 
@@ -129,6 +132,19 @@ with bootstrap 95% confidence intervals and a metric-correlation matrix.
 
 Run `toolbench --help` (or `tbe --help`) for the full reference.
 
+## Also
+
+- **Runtimes** — besides the API-driven `orchestral` runtime, toolbench drives the
+  `claude_code` and `codex` CLIs directly (subscription auth, no API key), with per-turn
+  token accounting and a filesystem sandbox. Copy-paste starting points live in
+  [`harness_templates/`](harness_templates/).
+- **Judges** — grade with the deterministic rule judge (default) or add an LLM second
+  opinion (`--judge rule+llm`); the rule grade always stays authoritative. Any judge can be
+  applied after the fact with `toolbench regrade --judge …`.
+- **Safeguards** — trials that read the ground-truth answer key are quarantined
+  (`INTEGRITY_LEAK`, scored 0) so a leak can't inflate the headline; every trial also gets a
+  readable `audit.txt` of its full trajectory.
+
 ## License
 
 MIT

{toolbench-0.1.0 → toolbench-0.2.0}/README.md

@@ -46,10 +46,10 @@ axes — vary any of them on the command line to run an ablation:
 | Concept       | What it is                                                            | Where it lives          |
 |---------------|----------------------------------------------------------------------|-------------------------|
 | **Benchmark** | The task + grading rubric + ground truth.                            | `benchmark.yaml`        |
-| **Harness**   | The agent runtime, provider, core tools, and loop policy.           | `harnesses/*.yaml`      |
+| **Harness**   | The agent runtime (`orchestral`, `claude_code`, `codex`), provider, core tools, and loop policy. | `harnesses/*.yaml` |
 | **Loadout**   | The domain tools the agent gets (beyond the harness core).          | `loadouts/*.yaml`       |
 | **Variant**   | The prompt + sandbox seed (scaffolding axis), orthogonal to tools.  | `variants/<name>/`      |
-| **Rubric**    | Ordered, weighted stages of checks; trial score = prefix product.   | inside `benchmark.yaml` |
+| **Rubric**    | Ordered, weighted stages of checks; trial score = weighted reach.   | inside `benchmark.yaml` |
 
 A **loadout source** is one of:
 
@@ -71,11 +71,14 @@ A **loadout source** is one of:
 
 For each (model × condition) cell over `k` trials:
 
-- **reach̄ₖ** — mean rubric-weighted reach (how far through the staged pipeline the agent gets).
-- **pass@k** — probability at least one of k trials passes every stage (best-of-k).
-- **pass^k** — probability all k trials pass (worst-of-k).
+- $\overline{\text{reach}}_k$ — mean rubric-weighted reach: how far through the task the agent gets, on average.
+- $\text{pass@}k$ — probability that at least one of $k$ trials passes (best-of-$k$).
+- $\text{pass}^{k}$ — probability that all $k$ trials pass (worst-of-$k$).
 
-with bootstrap 95% confidence intervals and a metric-correlation matrix.
+with bootstrap 95% confidence intervals and a metric-correlation matrix. A trial *passes*
+when it clears the rubric's pass criterion — every stage by default, or reach ≥ a
+`pass_threshold` once the rubric uses partial-credit stages. See
+[Metrics](https://toolbench-ai.com/docs/reference/metrics/) for the exact estimators.
 
 ## Commands
 
@@ -87,6 +90,19 @@ with bootstrap 95% confidence intervals and a metric-correlation matrix.
 
 Run `toolbench --help` (or `tbe --help`) for the full reference.
 
+## Also
+
+- **Runtimes** — besides the API-driven `orchestral` runtime, toolbench drives the
+  `claude_code` and `codex` CLIs directly (subscription auth, no API key), with per-turn
+  token accounting and a filesystem sandbox. Copy-paste starting points live in
+  [`harness_templates/`](harness_templates/).
+- **Judges** — grade with the deterministic rule judge (default) or add an LLM second
+  opinion (`--judge rule+llm`); the rule grade always stays authoritative. Any judge can be
+  applied after the fact with `toolbench regrade --judge …`.
+- **Safeguards** — trials that read the ground-truth answer key are quarantined
+  (`INTEGRITY_LEAK`, scored 0) so a leak can't inflate the headline; every trial also gets a
+  readable `audit.txt` of its full trajectory.
+
 ## License
 
 MIT

toolbench-0.2.0/harness_templates/README.md

@@ -0,0 +1,91 @@
+# Harness templates
+
+A harness answers one question: **what drives the model, and how does the
+loop behave?** It is the `harness` axis of a run
+(`benchmark × harness × loadout × variant × model`).
+
+These are copy-paste starting points, not a registry — nothing in toolbench
+reads this directory. Copy a file into a benchmark and edit it:
+
+```
+<benchmark>/harnesses/<runtime>/<name>.yaml
+```
+
+The path is the harness id: `harnesses/orchestral/groq.yaml` is referred to
+as `--harness orchestral/groq`.
+
+## Layout
+
+One directory per **runtime**. Each has a `README.md` documenting every key
+that runtime reads, plus runnable templates.
+
+```
+claude-code/     default.yaml                       provider-locked (subscription)
+codex/           default.yaml                       provider-locked (subscription)
+orchestral/      anthropic.yaml  openai.yaml        provider-agnostic:
+                 google.yaml     groq.yaml            one template per provider
+                 litellm.yaml
+```
+
+## Runtime vs. provider
+
+The distinction drives everything else:
+
+- **CLI runtimes** (`claude_code`, `codex`) shell out to a coding agent that
+  owns the model request. `provider:` is a credential-free placeholder and
+  only `{name, model}` apply — request params like `max_tokens` are inert.
+  `core: {builtin: true}` uses the agent's own tools.
+- **In-process runtimes** (`orchestral`) call a provider SDK or proxy
+  directly. `provider:` is fully live: `name` selects the backend and every
+  other key is forwarded as a model-request parameter on each call. `core:`
+  is an explicit tool list you curate per arm.
+
+Because orchestral is the provider-agnostic one, it gets a template per
+provider. The filename is the **provider key**, not the model family — Gemini
+lives in `google.yaml`. Registered providers: `anthropic`, `openai`,
+`google`, `groq`, `litellm`, `subscription`. Others (ollama, vllm, custom
+routes) register themselves via `register_provider`; see
+`toolbench/core/llm_factory.py`.
+
+## Two things that bite
+
+**`cache_bust` on proxy routes.** `provider:` keys are forwarded to the model
+*except* the control keys `name` and `cache_bust`. A proxy with response-level
+caching hands every trial in a cell the same completion — k trials stop being
+independent samples, and pass@k / pass^k silently collapse into reach. Set
+`cache_bust: true` on any proxy route (it's on in `litellm.yaml`). Direct
+providers don't need it: their prompt/KV caches don't affect sampling.
+
+**`--max-cost-usd` binds only when *something* prices the model.** Cost
+resolves through a three-step cascade (`runner.py:_extract_usage`):
+
+1. `usage.cost` on the Response — orchestral's own per-provider pricing model.
+   This is where `anthropic`, `openai`, `google` and `groq` all get priced.
+2. The LiteLLM proxy's pricing snapshot, fetched at run start — proxy routes
+   only, and only if the proxy is reachable at that moment.
+3. `toolbench.core.metrics.PRICING_TABLE` — a static last resort.
+
+Step 1 is an **exact model-id string match with no `default` rate**, so an
+unrecognized id costs `0.0` rather than raising. Combined with steps 2 and 3
+missing, the run reports `$0.00 spent` and the cap never trips — silently.
+Always pass the fully-qualified id the provider's pricing table uses
+(`openai/gpt-oss-120b`, not `gpt-oss-120b`), and sanity-check the reported
+spend on the first run of any new route.
+
+Note that step 3 is reached through a provider *guess* from the model name
+(`"claude" -> anthropic`, `"gpt" -> openai`), so `PRICING_TABLE` can never be
+consulted for a `groq` or `google` key — adding entries there would not help.
+Price a new provider in orchestral's pricing model instead.
+
+## Curating the tool set
+
+`core.tools` is where a harness enforces arm policy, and it is the natural
+place to drop tools that would function as an oracle:
+
+- `WebSearchTool` — remove when the answer is published anywhere.
+- `RunCommandTool` — a shell is the natural way to `cat`/`find` the ground
+  truth; remove it when the task never needs one.
+
+A "closed-book" arm is just a harness whose `core.tools` has no execution
+tools at all — file tools only, so the model must work in-context and can
+still deposit its answer.

toolbench-0.2.0/harness_templates/claude-code/README.md

@@ -0,0 +1,54 @@
+# `claude-code` harness templates
+
+The `claude_code` runtime shells out to `claude -p ... --output-format
+stream-json` under a Claude subscription. It is **provider-locked**: the
+`provider:` block is a credential-free placeholder and only `{name, model}`
+apply — the CLI owns the model request, so `max_tokens` and friends are
+inert here (cap output with `env: CLAUDE_CODE_MAX_OUTPUT_TOKENS`).
+
+Copy `default.yaml` to `<benchmark>/harnesses/claude-code/<name>.yaml`.
+
+## Every key this runtime reads
+
+```yaml
+# TEMPLATE — claude_code runtime (drives `claude -p` under a subscription).
+# Copy into a benchmark's harnesses/claude-code/<name>.yaml and keep only what
+# you override. The keys below are the ONLY ones this runtime reads; anything
+# else is ignored. Correctness-critical flags are hardcoded (see bottom).
+name: claude-code/<name>
+runtime:
+  name: claude_code                 # selects this runtime (required)
+  call_timeout_s: 3600              # per-MCP-call timeout (seconds)
+  disallowed_tools: [WebSearch, WebFetch]  # --disallowedTools
+  # effort: high                    # --effort: low|medium|high|xhigh|max
+  # fallback_model: claude-sonnet-5 # --fallback-model when the model is overloaded
+  # max_budget_usd: 5               # --max-budget-usd: per-session spend cap
+  # add_dir: [../shared]            # --add-dir: extra dirs the tools may access
+  env:                              # any var, set on the claude subprocess
+    ENABLE_TOOL_SEARCH: "false"     # eager-load MCP tools; no ToolSearch deferral
+    # MAX_THINKING_TOKENS: "31999"          # extended-thinking token budget
+    # CLAUDE_CODE_MAX_OUTPUT_TOKENS: "8192" # per-response output-token cap
+provider:                           # CLI runtime: only {name, model} apply
+  name: subscription                # credential-free placeholder llm (never called)
+  model: claude-haiku-4-5           # --model default; --models CLI arg overrides
+  # provider.max_tokens and other request params are INERT here — the claude CLI
+  # owns the model request. Cap output via env CLAUDE_CODE_MAX_OUTPUT_TOKENS.
+core:
+  builtin: true                     # use Claude Code's own core tools
+loop:                               # consumed by the runner (not the CLI)
+  max_iterations: 150               # agent.run() round-trip cap
+  max_format_retries: 2             # resume on MODEL_FORMAT_CRASH (bad tool-call JSON)
+  continue_nudges: 0                # presence-gated resumes on early stop (0 = strict)
+  max_rate_limit_retries: 3         # resume on provider 429/529 with backoff
+  max_transient_retries: 4          # resume on transient API/transport errors
+  ux_feedback: false                # post-run unscored tool-UX critique; false |
+                                    # true | graded (graded shows the agent its
+                                    # score + stage pass/fail first, no evidence)
+
+# HARDCODED (not overridable from this file):
+#   -p, --output-format stream-json, --verbose    trajectory capture
+#   --permission-mode acceptEdits                 non-interactive
+#   --allowedTools (builtins + mcp__<server>__*)  oracle hygiene + loadout
+#   --mcp-config, --strict-mcp-config, --resume   toolbase wiring + session resume
+#   env MCP_TOOL_TIMEOUT, MCP_TIMEOUT; ANTHROPIC_API_KEY stripped
+```

toolbench-0.2.0/harness_templates/claude-code/default.yaml

@@ -0,0 +1,20 @@
+# claude_code runtime: drives `claude -p` under a subscription.
+# See README.md in this directory for every key the runtime reads.
+name: claude-code/default
+runtime:
+  name: claude_code
+  call_timeout_s: 3600              # per-MCP-call timeout; raise for long tool calls
+  disallowed_tools: [WebSearch, WebFetch]
+  env:
+    ENABLE_TOOL_SEARCH: "false"     # eager-load MCP tools; no ToolSearch deferral
+provider:
+  name: subscription                # credential-free placeholder llm (never called)
+  model: claude-haiku-4-5           # --model default; --models overrides
+core:
+  builtin: true                     # use Claude Code's own core tools
+loop:
+  max_iterations: 150
+  max_format_retries: 2
+  continue_nudges: 0
+  max_rate_limit_retries: 3
+  max_transient_retries: 4

toolbench-0.2.0/harness_templates/codex/README.md

@@ -0,0 +1,43 @@
+# `codex` harness templates
+
+The `codex` runtime shells out to `codex exec --json` under a ChatGPT
+subscription. Like `claude-code` it is **provider-locked**: only
+`name` identifies subscription auth; the model comes from `toolbench run
+--models`. Request params are inert.
+
+Not plumbed for codex: `disallowed_tools`, `effort`, `fallback_model`,
+`max_budget_usd`, `add_dir` — those are `claude_code`-only curated flags.
+
+Copy `default.yaml` to `<benchmark>/harnesses/codex/<name>.yaml`.
+
+## Every key this runtime reads
+
+```yaml
+# TEMPLATE — codex runtime (drives `codex exec --json` under a ChatGPT
+# subscription). Copy into a benchmark's harnesses/codex/<name>.yaml and keep
+# only what you override. The keys below are the ONLY ones this runtime reads.
+name: codex/<name>
+runtime:
+  name: codex                       # selects this runtime (required)
+  call_timeout_s: 3600              # per-MCP-call timeout (seconds)
+  sandbox: workspace-write          # -s/--sandbox: read-only|workspace-write|danger-full-access
+  reasoning_effort: medium          # low|medium|high|xhigh|max; explicit for reproducibility
+  env:                              # any var, set on the codex subprocess
+    # SOME_VAR: "value"
+provider:
+  name: subscription                # credential-free placeholder llm (never called)
+  # Model comes from `toolbench run --models`; request params are not accepted.
+core:
+  builtin: true                     # use Codex's own core tools
+loop:                               # consumed by the runner (same knobs as claude_code)
+  max_format_retries: 2
+  continue_nudges: 0
+  max_rate_limit_retries: 3
+  max_transient_retries: 4
+  ux_feedback: false                # post-run unscored tool-UX critique (dev aid)
+
+# NOT plumbed for codex: disallowed_tools, fallback_model, max_budget_usd,
+#   add_dir (these are claude_code-only curated flags).
+# HARDCODED: --ignore-user-config; codex exec --json; MCP via
+#   -c mcp_servers.*; codex exec resume <id>.
+```

toolbench-0.2.0/harness_templates/codex/default.yaml

@@ -0,0 +1,18 @@
+# codex runtime: drives `codex exec --json` under a ChatGPT subscription.
+# See README.md in this directory for every key the runtime reads.
+name: codex/default
+runtime:
+  name: codex
+  call_timeout_s: 3600              # per-MCP-call timeout; raise for long tool calls
+  sandbox: workspace-write          # read-only | workspace-write | danger-full-access
+  reasoning_effort: medium          # explicit; never inherit the user's Codex config
+provider:
+  name: subscription                # credential-free placeholder llm (never called)
+  # The model always comes from toolbench run --models.
+core:
+  builtin: true                     # use Codex's own core tools
+loop:
+  max_format_retries: 2
+  continue_nudges: 0
+  max_rate_limit_retries: 3
+  max_transient_retries: 4

toolbench-0.2.0/harness_templates/orchestral/README.md

@@ -0,0 +1,49 @@
+# `orchestral` harness templates
+
+The orchestral runtime runs the model **in-process** through a provider
+SDK or proxy, so unlike the CLI runtimes the `provider:` block is fully
+live: `name` selects the backend and every other key is forwarded as a
+model-request parameter on each call.
+
+One file per **provider** — the filename is the provider key, not the
+model family (Gemini lives in `google.yaml`). Copy the one you need to
+`<benchmark>/harnesses/orchestral/<provider>.yaml` and trim `core.tools`
+to the arm's tool policy.
+
+Registered providers are `anthropic`, `openai`, `google`, `groq`,
+`litellm`, `subscription`. Others (ollama, vllm, custom routes) register
+themselves via `register_provider` in an adapter module — see
+`toolbench/core/llm_factory.py`.
+
+## Every key this runtime reads
+
+```yaml
+# TEMPLATE — orchestral runtime (in-process model via a provider SDK/proxy).
+# Copy into a benchmark's harnesses/orchestral/<name>.yaml. Unlike the CLI
+# runtimes, provider is FULLY live here: name + model + any request param
+# (max_tokens, temperature, ...) are forwarded on every model call.
+name: orchestral/<name>
+runtime:
+  name: orchestral                  # selects this runtime (required)
+  version: ">=1.3"                  # enforced against the installed orchestral
+provider:
+  name: anthropic                   # provider backend: anthropic|openai|litellm|... (control key)
+  model: claude-opus-4-8            # request model; --models CLI arg overrides
+  max_tokens: 8192                  # forwarded on every model call
+  # temperature: 1.0                # any provider request param is forwarded
+  # cache_bust: false               # toolbench control key: per-request cache nonce
+core:
+  # Explicit in-process core toolset (no toolbase builtin). Drop tools an arm bans.
+  tools: [RunCommandTool, WriteFileTool, ReadFileTool, EditFileTool,
+          FindFilesTool, FileSearchTool, RunPythonTool, TodoRead, TodoWrite]
+loop:                               # consumed by the runner
+  max_iterations: 150
+  max_format_retries: 2
+  continue_nudges: 0
+  max_rate_limit_retries: 3
+  max_transient_retries: 4
+  ux_feedback: false                # post-run unscored tool-UX critique (dev aid)
+
+# provider control keys (configure toolbench, NOT sent to the model): name, cache_bust.
+# All other provider keys are forwarded verbatim as model-request params.
+```

toolbench-0.2.0/harness_templates/orchestral/anthropic.yaml

@@ -0,0 +1,22 @@
+# Anthropic direct (ANTHROPIC_API_KEY). Priced in toolbench's PRICING_TABLE,
+# so --max-cost-usd binds on this route.
+name: orchestral/anthropic
+runtime:
+  name: orchestral
+  version: ">=1.3"
+provider:
+  name: anthropic
+  model: claude-haiku-4-5    # default; --models overrides
+  max_tokens: 8192
+core:
+  # Explicit in-process core toolset. Drop tools an arm bans (e.g. remove
+  # WebSearchTool when the answer is published, RunCommandTool when a shell
+  # would reach the ground truth).
+  tools: [RunCommandTool, WriteFileTool, ReadFileTool, EditFileTool,
+          FindFilesTool, FileSearchTool, RunPythonTool, TodoRead, TodoWrite]
+loop:
+  max_iterations: 150        # agent.run() round-trip cap
+  max_format_retries: 2      # resume on MODEL_FORMAT_CRASH (bad tool-call JSON)
+  continue_nudges: 0         # presence-gated resumes on early stop (0 = strict)
+  max_rate_limit_retries: 3  # resume on provider 429/529 with backoff
+  max_transient_retries: 4   # resume on transient API/transport errors

toolbench-0.2.0/harness_templates/orchestral/google.yaml

@@ -0,0 +1,22 @@
+# Google / Gemini (GOOGLE_API_KEY). The provider key is `google`; the file is
+# named for the provider, not the model family.
+name: orchestral/google
+runtime:
+  name: orchestral
+  version: ">=1.3"
+provider:
+  name: google
+  model: gemini-2.0-flash    # default; --models overrides
+  max_tokens: 8192
+core:
+  # Explicit in-process core toolset. Drop tools an arm bans (e.g. remove
+  # WebSearchTool when the answer is published, RunCommandTool when a shell
+  # would reach the ground truth).
+  tools: [RunCommandTool, WriteFileTool, ReadFileTool, EditFileTool,
+          FindFilesTool, FileSearchTool, RunPythonTool, TodoRead, TodoWrite]
+loop:
+  max_iterations: 150        # agent.run() round-trip cap
+  max_format_retries: 2      # resume on MODEL_FORMAT_CRASH (bad tool-call JSON)
+  continue_nudges: 0         # presence-gated resumes on early stop (0 = strict)
+  max_rate_limit_retries: 3  # resume on provider 429/529 with backoff
+  max_transient_retries: 4   # resume on transient API/transport errors

toolbench-0.2.0/harness_templates/orchestral/groq.yaml

@@ -0,0 +1,28 @@
+# Groq direct (GROQ_API_KEY) -- fast inference for open-weight models.
+# COST: orchestral prices groq itself (orchestral/llm/groq/pricing_model.py),
+# so usage.cost is populated and --max-cost-usd DOES bind. The lookup is an
+# EXACT string match with no 'default' fallback -- an unrecognized id silently
+# costs 0.00 and the budget cap never trips. Use the fully-qualified id
+# (`openai/gpt-oss-120b`, not `gpt-oss-120b`).
+# The orchestral Groq client hardcodes a 30s HTTP timeout: fine for short
+# tasks, too tight for harnesses whose tools make long calls.
+name: orchestral/groq
+runtime:
+  name: orchestral
+  version: ">=1.3"
+provider:
+  name: groq
+  model: openai/gpt-oss-120b # default; --models overrides
+  max_tokens: 8192
+core:
+  # Explicit in-process core toolset. Drop tools an arm bans (e.g. remove
+  # WebSearchTool when the answer is published, RunCommandTool when a shell
+  # would reach the ground truth).
+  tools: [RunCommandTool, WriteFileTool, ReadFileTool, EditFileTool,
+          FindFilesTool, FileSearchTool, RunPythonTool, TodoRead, TodoWrite]
+loop:
+  max_iterations: 150        # agent.run() round-trip cap
+  max_format_retries: 2      # resume on MODEL_FORMAT_CRASH (bad tool-call JSON)
+  continue_nudges: 0         # presence-gated resumes on early stop (0 = strict)
+  max_rate_limit_retries: 3  # resume on provider 429/529 with backoff
+  max_transient_retries: 4   # resume on transient API/transport errors

toolbench-0.2.0/harness_templates/orchestral/litellm.yaml

@@ -0,0 +1,32 @@
+# LiteLLM proxy (LITELLM_HOST incl. /v1, LITELLM_API_KEY) -- one route to many
+# backends. COST: orchestral's openai pricing model doesn't know proxied
+# open-weight ids, so cost resolves from the PRICING SNAPSHOT toolbench fetches
+# from the proxy at run start. If the proxy is unreachable then, or has no
+# pricing configured for the model, the run reports $0.00 and --max-cost-usd
+# does not bind -- bound it with --n and cell count instead.
+name: orchestral/litellm
+runtime:
+  name: orchestral
+  version: ">=1.3"
+provider:
+  name: litellm
+  model: gpt-oss-120b        # default; --models overrides
+  max_tokens: 8192
+  # A proxy may have RESPONSE-level caching on, which would hand every trial
+  # of a cell the same completion -- k trials stop being independent samples
+  # and pass@k / pass^k collapse into reach. cache_bust appends a per-trial
+  # nonce to defeat it. Leave it on for any proxy route; direct providers
+  # don't need it (their prompt/KV caches don't affect sampling).
+  cache_bust: true
+core:
+  # Explicit in-process core toolset. Drop tools an arm bans (e.g. remove
+  # WebSearchTool when the answer is published, RunCommandTool when a shell
+  # would reach the ground truth).
+  tools: [RunCommandTool, WriteFileTool, ReadFileTool, EditFileTool,
+          FindFilesTool, FileSearchTool, RunPythonTool, TodoRead, TodoWrite]
+loop:
+  max_iterations: 150        # agent.run() round-trip cap
+  max_format_retries: 2      # resume on MODEL_FORMAT_CRASH (bad tool-call JSON)
+  continue_nudges: 0         # presence-gated resumes on early stop (0 = strict)
+  max_rate_limit_retries: 3  # resume on provider 429/529 with backoff
+  max_transient_retries: 4   # resume on transient API/transport errors

toolbench-0.2.0/harness_templates/orchestral/openai.yaml

@@ -0,0 +1,22 @@
+# OpenAI direct (OPENAI_API_KEY). Priced for gpt-4o / gpt-4o-mini; other
+# model ids fall outside PRICING_TABLE and --max-cost-usd will not bind.
+name: orchestral/openai
+runtime:
+  name: orchestral
+  version: ">=1.3"
+provider:
+  name: openai
+  model: gpt-4o              # default; --models overrides
+  max_tokens: 8192
+core:
+  # Explicit in-process core toolset. Drop tools an arm bans (e.g. remove
+  # WebSearchTool when the answer is published, RunCommandTool when a shell
+  # would reach the ground truth).
+  tools: [RunCommandTool, WriteFileTool, ReadFileTool, EditFileTool,
+          FindFilesTool, FileSearchTool, RunPythonTool, TodoRead, TodoWrite]
+loop:
+  max_iterations: 150        # agent.run() round-trip cap
+  max_format_retries: 2      # resume on MODEL_FORMAT_CRASH (bad tool-call JSON)
+  continue_nudges: 0         # presence-gated resumes on early stop (0 = strict)
+  max_rate_limit_retries: 3  # resume on provider 429/529 with backoff
+  max_transient_retries: 4   # resume on transient API/transport errors

{toolbench-0.1.0 → toolbench-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "toolbench"
-version = "0.1.0"
+version = "0.2.0"
 description = "A platform and CLI for building benchmarks for agentic tools and harnesses."
 readme = "README.md"
 requires-python = ">=3.12"