toolbench 0.0.1__tar.gz → 0.2.0__tar.gz

{toolbench-0.0.1 → toolbench-0.2.0}/.gitignore

@@ -41,9 +41,22 @@ ENV/
 htmlcov/
 .hypothesis/
 
+# Claude Code local state
+.claude/
+
 # IDE
 .vscode/
 .idea/
 *.swp
 *.swo
 .DS_Store
+
+# local handoff / scratch — not for commit
+PORT_STATUS.md
+*.orig
+
+# built docs
+site/
+
+# run output
+runs/

toolbench-0.2.0/PKG-INFO

@@ -0,0 +1,150 @@
+Metadata-Version: 2.4
+Name: toolbench
+Version: 0.2.0
+Summary: A platform and CLI for building benchmarks for agentic tools and harnesses.
+Project-URL: Homepage, https://github.com/tonymenzo/toolbench
+Project-URL: Repository, https://github.com/tonymenzo/toolbench
+Project-URL: Issues, https://github.com/tonymenzo/toolbench/issues
+Author: Tony Menzo
+License: MIT
+License-File: LICENSE
+Keywords: agents,ai,benchmark,evaluation,llm,orchestral,tools
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.12
+Requires-Dist: click>=8.0
+Requires-Dist: matplotlib>=3.10.8
+Requires-Dist: numpy>=2.3.5
+Requires-Dist: orchestral-ai>=1.6.3
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: requests>=2.32.0
+Requires-Dist: rich>=14.2.0
+Requires-Dist: scipy>=1.16.3
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.5; extra == 'docs'
+Provides-Extra: dotenv
+Requires-Dist: python-dotenv>=1.2.1; extra == 'dotenv'
+Provides-Extra: mcp
+Requires-Dist: mcp>=1.0; extra == 'mcp'
+Provides-Extra: toolbase
+Requires-Dist: toolbase>=0.3.0; extra == 'toolbase'
+Description-Content-Type: text/markdown
+
+# toolbench
+
+**toolbench is a platform and CLI for building benchmarks for agentic tools and harnesses.**
+Compose a benchmark (a task + a grading rubric) with a *harness* (the agent runtime), a
+*loadout* (the tools the agent is given), and a *variant* (the prompt + sandbox), run N
+trials per cell against a model, and get reach / pass@k / pass^k metrics with plots.
+
+It is the benchmarking sibling of [toolbase](https://github.com/alexr314/toolbase) — the
+package manager and runtime for agent tools. toolbase serves the tools; toolbench measures
+how well an agent uses them. Together they close the loop for agentic tool and harness
+development.
+
+## Install
+
+```bash
+pip install toolbench                 # core framework
+pip install 'toolbench[toolbase]'     # + resolve tools from toolbase profiles
+```
+
+Requires Python ≥ 3.12. The CLI is available as both `toolbench` and the short alias `tbe`.
+
+## Quickstart
+
+The example `geometry` benchmark (Euclidean distance + midpoint between two 2-D points) is a
+self-contained, dependency-free example that exercises the whole framework.
+
+```bash
+# Validate the wiring with no LLM calls or cost:
+toolbench run --benchmark examples/geometry --model stub \
+    --loadouts full_local --n 1 --max-cost-usd 0 --dry-run
+
+# A real run (a few cheap trials):
+toolbench run --benchmark examples/geometry --model claude-haiku-4-5 \
+    --loadouts core_only,full_local --n 3 --max-cost-usd 0.50
+```
+
+Each run writes a directory under `runs/<run_id>/` with a `manifest.json`, per-trial
+transcripts and artifacts, an aggregated `summary.json` / `summary.txt`, and headline plots
+(k-sweep, parallel-coordinates, per-stage breakdown).
+
+## Concepts
+
+A benchmark lives in `examples/<name>/` and is composed from four declarative
+axes — vary any of them on the command line to run an ablation:
+
+| Concept       | What it is                                                            | Where it lives          |
+|---------------|----------------------------------------------------------------------|-------------------------|
+| **Benchmark** | The task + grading rubric + ground truth.                            | `benchmark.yaml`        |
+| **Harness**   | The agent runtime (`orchestral`, `claude_code`, `codex`), provider, core tools, and loop policy. | `harnesses/*.yaml` |
+| **Loadout**   | The domain tools the agent gets (beyond the harness core).          | `loadouts/*.yaml`       |
+| **Variant**   | The prompt + sandbox seed (scaffolding axis), orthogonal to tools.  | `variants/<name>/`      |
+| **Rubric**    | Ordered, weighted stages of checks; trial score = weighted reach.   | inside `benchmark.yaml` |
+
+A **loadout source** is one of:
+
+- `python:` — import a module exposing `TOOLS` / `make_tools()` (the no-dependency escape hatch),
+- `toolbase:` — resolve tools from a [toolbase](https://github.com/alexr314/toolbase) profile,
+  in-process (requires `toolbench[toolbase]`); served toolkit versions are recorded in the
+  run manifest as reproducibility provenance, or
+- `mcp:` — serve any MCP server's tools, stdio or HTTP (requires `toolbench[mcp]`):
+
+  ```yaml
+  tools:
+    sources:
+      - toolbase: { profile: my-profile }
+        select: [calculator__add]          # optional: ablate within the profile
+      - mcp: { url: "https://host/mcp", headers: { Authorization: "Bearer ${TOK}" } }
+  ```
+
+## Metrics
+
+For each (model × condition) cell over `k` trials:
+
+- $\overline{\text{reach}}_k$ — mean rubric-weighted reach: how far through the task the agent gets, on average.
+- $\text{pass@}k$ — probability that at least one of $k$ trials passes (best-of-$k$).
+- $\text{pass}^{k}$ — probability that all $k$ trials pass (worst-of-$k$).
+
+with bootstrap 95% confidence intervals and a metric-correlation matrix. A trial *passes*
+when it clears the rubric's pass criterion — every stage by default, or reach ≥ a
+`pass_threshold` once the rubric uses partial-credit stages. See
+[Metrics](https://toolbench-ai.com/docs/reference/metrics/) for the exact estimators.
+
+## Commands
+
+| Command            | What it does                                                          |
+|--------------------|----------------------------------------------------------------------|
+| `toolbench run`    | Run a benchmark across the harness × loadout × variant × model grid. |
+| `toolbench resume` | Resume an interrupted run; run only the seeds not yet completed.      |
+| `toolbench regrade`| Re-judge a finished run's preserved artifacts after a rubric change.  |
+
+Run `toolbench --help` (or `tbe --help`) for the full reference.
+
+## Also
+
+- **Runtimes** — besides the API-driven `orchestral` runtime, toolbench drives the
+  `claude_code` and `codex` CLIs directly (subscription auth, no API key), with per-turn
+  token accounting and a filesystem sandbox. Copy-paste starting points live in
+  [`harness_templates/`](harness_templates/).
+- **Judges** — grade with the deterministic rule judge (default) or add an LLM second
+  opinion (`--judge rule+llm`); the rule grade always stays authoritative. Any judge can be
+  applied after the fact with `toolbench regrade --judge …`.
+- **Safeguards** — trials that read the ground-truth answer key are quarantined
+  (`INTEGRITY_LEAK`, scored 0) so a leak can't inflate the headline; every trial also gets a
+  readable `audit.txt` of its full trajectory.
+
+## License
+
+MIT

toolbench-0.2.0/README.md

@@ -0,0 +1,108 @@
+# toolbench
+
+**toolbench is a platform and CLI for building benchmarks for agentic tools and harnesses.**
+Compose a benchmark (a task + a grading rubric) with a *harness* (the agent runtime), a
+*loadout* (the tools the agent is given), and a *variant* (the prompt + sandbox), run N
+trials per cell against a model, and get reach / pass@k / pass^k metrics with plots.
+
+It is the benchmarking sibling of [toolbase](https://github.com/alexr314/toolbase) — the
+package manager and runtime for agent tools. toolbase serves the tools; toolbench measures
+how well an agent uses them. Together they close the loop for agentic tool and harness
+development.
+
+## Install
+
+```bash
+pip install toolbench                 # core framework
+pip install 'toolbench[toolbase]'     # + resolve tools from toolbase profiles
+```
+
+Requires Python ≥ 3.12. The CLI is available as both `toolbench` and the short alias `tbe`.
+
+## Quickstart
+
+The example `geometry` benchmark (Euclidean distance + midpoint between two 2-D points) is a
+self-contained, dependency-free example that exercises the whole framework.
+
+```bash
+# Validate the wiring with no LLM calls or cost:
+toolbench run --benchmark examples/geometry --model stub \
+    --loadouts full_local --n 1 --max-cost-usd 0 --dry-run
+
+# A real run (a few cheap trials):
+toolbench run --benchmark examples/geometry --model claude-haiku-4-5 \
+    --loadouts core_only,full_local --n 3 --max-cost-usd 0.50
+```
+
+Each run writes a directory under `runs/<run_id>/` with a `manifest.json`, per-trial
+transcripts and artifacts, an aggregated `summary.json` / `summary.txt`, and headline plots
+(k-sweep, parallel-coordinates, per-stage breakdown).
+
+## Concepts
+
+A benchmark lives in `examples/<name>/` and is composed from four declarative
+axes — vary any of them on the command line to run an ablation:
+
+| Concept       | What it is                                                            | Where it lives          |
+|---------------|----------------------------------------------------------------------|-------------------------|
+| **Benchmark** | The task + grading rubric + ground truth.                            | `benchmark.yaml`        |
+| **Harness**   | The agent runtime (`orchestral`, `claude_code`, `codex`), provider, core tools, and loop policy. | `harnesses/*.yaml` |
+| **Loadout**   | The domain tools the agent gets (beyond the harness core).          | `loadouts/*.yaml`       |
+| **Variant**   | The prompt + sandbox seed (scaffolding axis), orthogonal to tools.  | `variants/<name>/`      |
+| **Rubric**    | Ordered, weighted stages of checks; trial score = weighted reach.   | inside `benchmark.yaml` |
+
+A **loadout source** is one of:
+
+- `python:` — import a module exposing `TOOLS` / `make_tools()` (the no-dependency escape hatch),
+- `toolbase:` — resolve tools from a [toolbase](https://github.com/alexr314/toolbase) profile,
+  in-process (requires `toolbench[toolbase]`); served toolkit versions are recorded in the
+  run manifest as reproducibility provenance, or
+- `mcp:` — serve any MCP server's tools, stdio or HTTP (requires `toolbench[mcp]`):
+
+  ```yaml
+  tools:
+    sources:
+      - toolbase: { profile: my-profile }
+        select: [calculator__add]          # optional: ablate within the profile
+      - mcp: { url: "https://host/mcp", headers: { Authorization: "Bearer ${TOK}" } }
+  ```
+
+## Metrics
+
+For each (model × condition) cell over `k` trials:
+
+- $\overline{\text{reach}}_k$ — mean rubric-weighted reach: how far through the task the agent gets, on average.
+- $\text{pass@}k$ — probability that at least one of $k$ trials passes (best-of-$k$).
+- $\text{pass}^{k}$ — probability that all $k$ trials pass (worst-of-$k$).
+
+with bootstrap 95% confidence intervals and a metric-correlation matrix. A trial *passes*
+when it clears the rubric's pass criterion — every stage by default, or reach ≥ a
+`pass_threshold` once the rubric uses partial-credit stages. See
+[Metrics](https://toolbench-ai.com/docs/reference/metrics/) for the exact estimators.
+
+## Commands
+
+| Command            | What it does                                                          |
+|--------------------|----------------------------------------------------------------------|
+| `toolbench run`    | Run a benchmark across the harness × loadout × variant × model grid. |
+| `toolbench resume` | Resume an interrupted run; run only the seeds not yet completed.      |
+| `toolbench regrade`| Re-judge a finished run's preserved artifacts after a rubric change.  |
+
+Run `toolbench --help` (or `tbe --help`) for the full reference.
+
+## Also
+
+- **Runtimes** — besides the API-driven `orchestral` runtime, toolbench drives the
+  `claude_code` and `codex` CLIs directly (subscription auth, no API key), with per-turn
+  token accounting and a filesystem sandbox. Copy-paste starting points live in
+  [`harness_templates/`](harness_templates/).
+- **Judges** — grade with the deterministic rule judge (default) or add an LLM second
+  opinion (`--judge rule+llm`); the rule grade always stays authoritative. Any judge can be
+  applied after the fact with `toolbench regrade --judge …`.
+- **Safeguards** — trials that read the ground-truth answer key are quarantined
+  (`INTEGRITY_LEAK`, scored 0) so a leak can't inflate the headline; every trial also gets a
+  readable `audit.txt` of its full trajectory.
+
+## License
+
+MIT

toolbench-0.2.0/examples/calculator/requirements.txt

@@ -0,0 +1 @@
+orchestral-ai>=1.3.0

toolbench-0.2.0/examples/calculator/toolkit.yaml

@@ -0,0 +1,37 @@
+name: calculator
+version: 1.0.0
+description: A small calculator toolkit — basic arithmetic plus a scientific bundle.
+author: toolbench examples
+license: MIT
+category: utils
+python_version: "3.12"
+
+bundles:
+  basic: {}
+  scientific: {}
+
+tools:
+  - name: add
+    function: tools.basic.add
+    description: Add two numbers.
+    bundle: basic
+  - name: subtract
+    function: tools.basic.subtract
+    description: Subtract b from a.
+    bundle: basic
+  - name: multiply
+    function: tools.basic.multiply
+    description: Multiply two numbers.
+    bundle: basic
+  - name: divide
+    function: tools.basic.divide
+    description: Divide a by b.
+    bundle: basic
+  - name: power
+    function: tools.scientific.power
+    description: Raise base to the given exponent (use exp=0.5 for square root).
+    bundle: scientific
+  - name: sqrt
+    function: tools.scientific.sqrt
+    description: Square root of a number.
+    bundle: scientific

toolbench-0.2.0/examples/calculator/tools/__init__.py

@@ -0,0 +1,4 @@
+from .basic import add, subtract, multiply, divide
+from .scientific import power, sqrt
+
+__all__ = ["add", "subtract", "multiply", "divide", "power", "sqrt"]

toolbench-0.2.0/examples/calculator/tools/basic.py

@@ -0,0 +1,29 @@
+"""Basic arithmetic tools for the calculator example toolkit."""
+
+import json
+
+from orchestral import define_tool
+
+
+@define_tool
+def add(a: float, b: float) -> str:
+    """Add two numbers."""
+    return json.dumps({"result": float(a) + float(b)})
+
+
+@define_tool
+def subtract(a: float, b: float) -> str:
+    """Subtract b from a."""
+    return json.dumps({"result": float(a) - float(b)})
+
+
+@define_tool
+def multiply(a: float, b: float) -> str:
+    """Multiply two numbers."""
+    return json.dumps({"result": float(a) * float(b)})
+
+
+@define_tool
+def divide(a: float, b: float) -> str:
+    """Divide a by b."""
+    return json.dumps({"result": float(a) / float(b)})

toolbench-0.2.0/examples/calculator/tools/scientific.py

@@ -0,0 +1,18 @@
+"""Scientific tools for the calculator example toolkit."""
+
+import json
+import math
+
+from orchestral import define_tool
+
+
+@define_tool
+def power(base: float, exp: float) -> str:
+    """Raise base to the given exponent (use exp=0.5 for a square root)."""
+    return json.dumps({"result": float(base) ** float(exp)})
+
+
+@define_tool
+def sqrt(x: float) -> str:
+    """Square root of a number."""
+    return json.dumps({"result": math.sqrt(float(x))})

toolbench-0.2.0/examples/geometry/benchmark.yaml

@@ -0,0 +1,47 @@
+name: geometry
+version: 0.1.0
+description: Euclidean distance and midpoint between two 2-D points.
+
+default_harness: orchestral/anthropic
+default_loadout: full_local
+# Three variants (direct/derived/polar) — see variants/. All three
+# share the same ground truth (distance 5.0, midpoint [1.5, 2.0]) so
+# the family-level rubric grades them identically and cross-variant
+# reach deltas are meaningful (and not contaminated by reference-value
+# differences). Names describe what each variant supplies, not a
+# presumed difficulty ordering.
+default_variant: direct
+
+ground_truth:
+  dir: ./ground_truth
+
+rubric:
+  type: stagewise           # ordered stages; trial score = prefix product
+  stages:
+    - id: answer_written
+      description: output/answer.json exists with the required keys
+      weight: 0.2
+      checks:
+        - json_with_keys:
+            file: output/answer.json
+            required_keys: [distance, midpoint]
+    - id: midpoint_correct
+      description: midpoint is within 1% of ground truth
+      weight: 0.3
+      checks:
+        - close_to:
+            file: output/answer.json
+            field: midpoint
+            reference: ./ground_truth/answer.json
+            tolerance_frac: 0.01
+      expected_tool_calls: [add, divide]        # optional diagnostic, not graded
+    - id: distance_correct
+      description: Euclidean distance is within 1% of ground truth
+      weight: 0.5
+      checks:
+        - close_to:
+            file: output/answer.json
+            field: distance
+            reference: ./ground_truth/answer.json
+            tolerance_frac: 0.01
+      expected_tool_calls: [euclidean_distance]

toolbench-0.2.0/examples/geometry/ground_truth/answer.json

@@ -0,0 +1 @@
+{"distance": 5.0, "midpoint": [1.5, 2.0]}

toolbench-0.2.0/examples/geometry/harnesses/orchestral/anthropic.yaml

@@ -0,0 +1,13 @@
+name: orchestral/anthropic
+runtime:
+  name: orchestral
+  version: ">=1.3"
+provider:
+  name: anthropic
+  max_tokens: 8192
+core:
+  tools: [RunCommandTool, WriteFileTool, ReadFileTool, RunPythonTool, TodoWrite]
+loop:
+  max_iterations: 150        # agent.run() round-trip cap (consumed by the runner)
+  max_format_retries: 2      # resume the session on a MODEL_FORMAT_CRASH (bad tool-call JSON)
+  continue_nudges: 0         # presence-gated resumes when the model stops early (0 = strict autonomy)

toolbench-0.2.0/examples/geometry/harnesses/orchestral/groq.yaml

@@ -0,0 +1,13 @@
+name: orchestral/groq
+runtime:
+  name: orchestral
+  version: ">=1.3"
+provider:
+  name: groq
+  max_tokens: 8192
+core:
+  tools: [RunCommandTool, WriteFileTool, ReadFileTool, RunPythonTool, TodoWrite]
+loop:
+  max_iterations: 150        # agent.run() round-trip cap (consumed by the runner)
+  max_format_retries: 2      # resume the session on a MODEL_FORMAT_CRASH (bad tool-call JSON)
+  continue_nudges: 0         # presence-gated resumes when the model stops early (0 = strict autonomy)

toolbench-0.2.0/examples/geometry/harnesses/orchestral/litellm.yaml

@@ -0,0 +1,19 @@
+name: orchestral/litellm
+runtime:
+  name: orchestral
+  version: ">=1.3"
+provider:
+  name: litellm           # routes via the LiteLLM proxy (LITELLM_HOST / LITELLM_API_KEY)
+  max_tokens: 8192
+  # A LiteLLM proxy may have *response-level* caching enabled, which would
+  # hand every trial of a cell the same completion (k trials stop being
+  # independent samples). cache_bust appends a per-trial nonce to the
+  # prompt to defeat it. Off by default for direct providers: their
+  # prompt/KV caches don't affect sampling independence.
+  cache_bust: true
+core:
+  tools: [RunCommandTool, WriteFileTool, ReadFileTool, RunPythonTool, TodoWrite]
+loop:
+  max_iterations: 150        # agent.run() round-trip cap (consumed by the runner)
+  max_format_retries: 2      # resume the session on a MODEL_FORMAT_CRASH (bad tool-call JSON)
+  continue_nudges: 0         # presence-gated resumes when the model stops early (0 = strict autonomy)

toolbench-0.2.0/examples/geometry/harnesses/orchestral/openai.yaml

@@ -0,0 +1,13 @@
+name: orchestral/openai
+runtime:
+  name: orchestral
+  version: ">=1.3"
+provider:
+  name: openai
+  max_tokens: 8192
+core:
+  tools: [RunCommandTool, WriteFileTool, ReadFileTool, RunPythonTool, TodoWrite]
+loop:
+  max_iterations: 150        # agent.run() round-trip cap (consumed by the runner)
+  max_format_retries: 2      # resume the session on a MODEL_FORMAT_CRASH (bad tool-call JSON)
+  continue_nudges: 0         # presence-gated resumes when the model stops early (0 = strict autonomy)

toolbench-0.2.0/examples/geometry/loadouts/additive_only.yaml

@@ -0,0 +1,6 @@
+name: additive_only
+tools:
+  sources:
+    - python: tools/dunderkit.py
+      select: [additive]               # only add + subtract (no divide, no power)
+skills: []

toolbench-0.2.0/examples/geometry/loadouts/all_metrics.yaml

@@ -0,0 +1,8 @@
+name: all_metrics
+tools:
+  sources:
+    - python: tools/dunderkit.py     # for the midpoint
+    - python: tools/euclid.py        # correct metric for this task -> 5.0
+    - python: tools/taxicab.py       # L1 -> 7.0 (wrong)
+    - python: tools/chebyshev.py     # L-inf -> 4.0 (wrong)
+skills: []

toolbench-0.2.0/examples/geometry/loadouts/calc_toolbase.yaml

@@ -0,0 +1,8 @@
+name: calc_toolbase
+# Resolve tools from a toolbase profile (the `calculator` example toolkit).
+# Requires: toolbench[toolbase] installed, and the calculator toolkit installed
+# + activated into the named profile (see docs/guides/toolbase.md).
+tools:
+  sources:
+    - toolbase: { profile: default }
+skills: []

toolbench-0.2.0/examples/geometry/loadouts/core_only.yaml

@@ -0,0 +1,4 @@
+name: core_only
+tools:
+  sources: []          # empty toolkit; the agent has only the harness's core tools
+skills: []

toolbench-0.2.0/examples/geometry/loadouts/full_local.yaml

@@ -0,0 +1,6 @@
+name: full_local
+tools:
+  sources:
+    - python: tools/dunderkit.py     # all bundles (no `select:` = all)
+    - python: tools/euclid.py
+skills: []

toolbench-0.2.0/examples/geometry/loadouts/full_mixed.yaml

@@ -0,0 +1,9 @@
+name: full_mixed
+tools:
+  sources:
+    - python: tools/dunderkit.py      # local (escape hatch)
+    - toolbase:                          # frozen, via toolbase
+        toolsets:
+          euclid:
+            version: "==1.0.0"
+skills: []

toolbench-0.2.0/examples/geometry/loadouts/full_toolbase.yaml

@@ -0,0 +1,11 @@
+name: full_toolbase
+tools:
+  sources:
+    - toolbase:                          # mirrors a serve.yaml toolkit block
+        toolsets:
+          dunderkit:
+            version: ">=1.0,<2.0"
+            bundles: [additive, multiplicative, powers]
+          euclid:
+            version: "==1.0.0"
+skills: []

toolbench-0.2.0/examples/geometry/loadouts/guided.yaml

@@ -0,0 +1,8 @@
+name: guided
+tools:
+  sources:
+    - python: tools/dunderkit.py     # NOTE: no euclid toolset; the skill teaches assembly
+skills:
+  - name: distance_recipe
+    file: ./skills/distance_recipe.md
+    mode: on_demand

toolbench-0.2.0/examples/geometry/loadouts/primitives_only.yaml

@@ -0,0 +1,5 @@
+name: primitives_only
+tools:
+  sources:
+    - python: tools/dunderkit.py     # all dunder primitives, no metric tool
+skills: []

toolbench-0.2.0/examples/geometry/skills/distance_recipe.md

@@ -0,0 +1,9 @@
+# Distance + midpoint
+
+Given two points `p1 = [x1, y1]` and `p2 = [x2, y2]`:
+
+1. `dx = subtract(x2, x1)`, `dy = subtract(y2, y1)`
+2. `distance = power( add(power(dx, 2), power(dy, 2)), 0.5 )`
+3. `midpoint = [ divide(add(x1, x2), 2), divide(add(y1, y2), 2) ]`
+
+Write the result to `output/answer.json` as `{"distance": ..., "midpoint": [...]}`.

toolbench-0.2.0/examples/geometry/tools/chebyshev.py

@@ -0,0 +1,14 @@
+"""The Chebyshev (L-infinity) distance metric as its own one-tool toolset."""
+
+from orchestral.tools import define_tool
+
+
+@define_tool
+def chebyshev_distance(p1: list[float], p2: list[float]) -> dict:
+    """L-infinity distance: max |a_i - b_i|."""
+    return {"result": max(abs(float.__sub__(float(a), float(b)))
+                          for a, b in zip(p1, p2))}
+
+
+TOOLS = [chebyshev_distance]
+BUNDLES = {"metric": [chebyshev_distance]}

toolbench-0.2.0/examples/geometry/tools/dunderkit.py

@@ -0,0 +1,43 @@
+"""Arithmetic primitives, each tool's body a single Python dunder call."""
+
+from orchestral.tools import define_tool
+
+
+@define_tool
+def add(a: float, b: float) -> dict:
+    """a + b, via float.__add__."""
+    return {"result": float.__add__(float(a), float(b))}
+
+
+@define_tool
+def subtract(a: float, b: float) -> dict:
+    """a - b, via float.__sub__."""
+    return {"result": float.__sub__(float(a), float(b))}
+
+
+@define_tool
+def multiply(a: float, b: float) -> dict:
+    """a * b, via float.__mul__."""
+    return {"result": float.__mul__(float(a), float(b))}
+
+
+@define_tool
+def divide(a: float, b: float) -> dict:
+    """a / b, via float.__truediv__."""
+    return {"result": float.__truediv__(float(a), float(b))}
+
+
+@define_tool
+def power(base: float, exp: float) -> dict:
+    """base ** exp, via float.__pow__. (sqrt = power(x, 0.5))"""
+    return {"result": float.__pow__(float(base), float(exp))}
+
+
+TOOLS = [add, subtract, multiply, divide, power]   # discovery entrypoint (all tools)
+
+# BUNDLES — author-declared groupings within this toolset (a toolbase "bundle").
+BUNDLES = {
+    "additive":       [add, subtract],
+    "multiplicative": [multiply, divide],
+    "powers":         [power],
+}

toolbench-0.2.0/examples/geometry/tools/euclid.py

@@ -0,0 +1,15 @@
+"""The Euclidean (L2) distance metric as its own one-tool toolset."""
+
+from orchestral.tools import define_tool
+
+
+@define_tool
+def euclidean_distance(p1: list[float], p2: list[float]) -> dict:
+    """L2 distance: sqrt(sum (a_i - b_i)^2)."""
+    s = sum(float.__pow__(float.__sub__(float(a), float(b)), 2.0)
+            for a, b in zip(p1, p2))
+    return {"result": float.__pow__(s, 0.5)}
+
+
+TOOLS = [euclidean_distance]
+BUNDLES = {"metric": [euclidean_distance]}