toolcallbench 1.0.0__py3-none-any.whl

toolcallbench/__init__.py

@@ -0,0 +1,3 @@
+"""ToolCallBench: measure how reliably local LLMs call tools on your own repo and hardware."""
+
+__version__ = "1.0.0"

toolcallbench/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

toolcallbench/backends/__init__.py

@@ -0,0 +1,43 @@
+"""Model backends and a factory."""
+from __future__ import annotations
+
+from typing import Any
+
+from ..models import Task
+from .base import Backend, BackendError
+from .ollama import OllamaBackend
+from .openai_compat import OpenAICompatBackend
+from .scripted import PROFILES, ScriptedBackend
+
+BACKEND_KINDS = ("ollama", "openai-compat", "scripted")
+
+__all__ = [
+    "BACKEND_KINDS",
+    "PROFILES",
+    "Backend",
+    "BackendError",
+    "OllamaBackend",
+    "OpenAICompatBackend",
+    "ScriptedBackend",
+    "make_backend",
+]
+
+
+def make_backend(
+    kind: str,
+    *,
+    model: str,
+    host: str,
+    options: dict[str, Any],
+    think: bool | None,
+    timeout: float,
+    tasks: list[Task],
+    profile: str = "oracle",
+) -> Backend:
+    if kind == "ollama":
+        return OllamaBackend(model, host=host, options=options, think=think, timeout=timeout)
+    if kind == "openai-compat":
+        return OpenAICompatBackend(model, base_url=host, options=options, timeout=timeout)
+    if kind == "scripted":
+        return ScriptedBackend(tasks, profile=profile)
+    raise ValueError(f"unknown backend '{kind}'; choose from {', '.join(BACKEND_KINDS)}")

toolcallbench/backends/base.py

@@ -0,0 +1,42 @@
+"""Backend interface and a small JSON-over-HTTP helper (standard library only)."""
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from typing import Any
+
+from ..models import BackendResponse
+
+
+class BackendError(RuntimeError):
+    """The model server could not be reached or returned an error. Not a model failure."""
+
+
+class Backend:
+    name = "backend"
+
+    def chat(self, messages: list[dict[str, Any]], tools: list[dict[str, Any]]) -> BackendResponse:
+        raise NotImplementedError
+
+    def check(self) -> None:
+        """Raise BackendError if the server or the model is not usable. Called once before a sweep."""
+
+    def describe(self) -> dict[str, Any]:
+        """Facts about the server and model, recorded in the session settings."""
+        return {"backend": self.name}
+
+
+def request_json(method: str, url: str, payload: dict[str, Any] | None, timeout: float) -> Any:
+    data = json.dumps(payload).encode("utf-8") if payload is not None else None
+    request = urllib.request.Request(url, data=data, method=method, headers={"Content-Type": "application/json"})
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            return json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as exc:
+        body = exc.read().decode("utf-8", "replace")[:500]
+        raise BackendError(f"HTTP {exc.code} from {url}: {body}") from exc
+    except json.JSONDecodeError as exc:
+        raise BackendError(f"{url} returned invalid JSON") from exc
+    except (urllib.error.URLError, TimeoutError, OSError) as exc:
+        raise BackendError(f"cannot reach {url}: {exc}") from exc

toolcallbench/backends/ollama.py

@@ -0,0 +1,86 @@
+"""Ollama backend: POST /api/chat with tools, non-streaming."""
+from __future__ import annotations
+
+from typing import Any
+
+from ..models import BackendResponse, RawToolCall
+from .base import Backend, BackendError, request_json
+
+
+class OllamaBackend(Backend):
+    name = "ollama"
+
+    def __init__(
+        self,
+        model: str,
+        host: str = "http://localhost:11434",
+        options: dict[str, Any] | None = None,
+        think: bool | None = None,
+        timeout: float = 300.0,
+        keep_alive: str = "30m",
+    ):
+        self.model = model
+        self.host = host.rstrip("/")
+        self.options = dict(options or {})
+        self.think = think
+        self.timeout = timeout
+        self.keep_alive = keep_alive
+
+    # ------------------------------------------------------------ server facts
+
+    def version(self) -> str:
+        return str(request_json("GET", f"{self.host}/api/version", None, self.timeout).get("version", "unknown"))
+
+    def list_models(self) -> list[str]:
+        data = request_json("GET", f"{self.host}/api/tags", None, self.timeout)
+        return [m.get("name", "") for m in data.get("models", [])]
+
+    def show(self) -> dict[str, Any]:
+        return request_json("POST", f"{self.host}/api/show", {"model": self.model}, self.timeout)
+
+    def check(self) -> None:
+        installed = self.list_models()
+        wanted = {self.model, self.model if ":" in self.model else self.model + ":latest"}
+        if not wanted & set(installed):
+            raise BackendError(f"model '{self.model}' is not installed in Ollama; run: ollama pull {self.model}")
+
+    def describe(self) -> dict[str, Any]:
+        info: dict[str, Any] = {"backend": self.name, "host": self.host}
+        try:
+            info["backend_version"] = self.version()
+            shown = self.show()
+            details = shown.get("details", {})
+            info["quant"] = details.get("quantization_level", "")
+            info["parameter_size"] = details.get("parameter_size", "")
+            info["family"] = details.get("family", "")
+            info["capabilities"] = shown.get("capabilities", [])
+        except BackendError:
+            pass
+        return info
+
+    # -------------------------------------------------------------------- chat
+
+    def chat(self, messages: list[dict[str, Any]], tools: list[dict[str, Any]]) -> BackendResponse:
+        payload: dict[str, Any] = {
+            "model": self.model,
+            "messages": messages,
+            "tools": tools,
+            "stream": False,
+            "options": self.options,
+            "keep_alive": self.keep_alive,
+        }
+        if self.think is not None:
+            payload["think"] = self.think
+        data = request_json("POST", f"{self.host}/api/chat", payload, self.timeout)
+        message = data.get("message") or {}
+        calls = []
+        for item in message.get("tool_calls") or []:
+            function = item.get("function") or {}
+            calls.append(RawToolCall(name=str(function.get("name", "")), arguments=function.get("arguments", {})))
+        return BackendResponse(
+            content=message.get("content") or "",
+            tool_calls=calls,
+            eval_tokens=int(data.get("eval_count") or 0),
+            eval_seconds=float(data.get("eval_duration") or 0) / 1e9,
+            prompt_tokens=int(data.get("prompt_eval_count") or 0),
+        )

toolcallbench/backends/openai_compat.py

@@ -0,0 +1,96 @@
+"""OpenAI-compatible backend for llama.cpp `llama-server`, LM Studio, vLLM and similar servers.
+
+Tool arguments arrive as JSON strings here, so this backend exercises the invalid-JSON path.
+"""
+from __future__ import annotations
+
+import json
+import time
+from typing import Any
+
+from ..models import BackendResponse, RawToolCall
+from .base import Backend, BackendError, request_json
+
+
+def to_openai_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Convert the runner's Ollama-style messages to the OpenAI chat format (adds tool call ids)."""
+    out: list[dict[str, Any]] = []
+    pending: list[str] = []
+    counter = 0
+    for message in messages:
+        if message["role"] == "assistant" and message.get("tool_calls"):
+            pending = []
+            calls = []
+            for item in message["tool_calls"]:
+                counter += 1
+                call_id = f"call_{counter}"
+                pending.append(call_id)
+                function = item["function"]
+                calls.append(
+                    {
+                        "id": call_id,
+                        "type": "function",
+                        "function": {"name": function["name"], "arguments": json.dumps(function.get("arguments", {}))},
+                    }
+                )
+            out.append({"role": "assistant", "content": message.get("content") or "", "tool_calls": calls})
+        elif message["role"] == "tool":
+            call_id = pending.pop(0) if pending else "call_0"
+            out.append({"role": "tool", "tool_call_id": call_id, "content": message["content"]})
+        else:
+            out.append({"role": message["role"], "content": message.get("content") or ""})
+    return out
+
+
+class OpenAICompatBackend(Backend):
+    name = "openai-compat"
+
+    def __init__(
+        self,
+        model: str,
+        base_url: str = "http://localhost:8080",
+        options: dict[str, Any] | None = None,
+        timeout: float = 300.0,
+    ):
+        self.model = model
+        self.base_url = base_url.rstrip("/")
+        self.options = dict(options or {})
+        self.timeout = timeout
+
+    def check(self) -> None:
+        request_json("GET", f"{self.base_url}/v1/models", None, self.timeout)
+
+    def describe(self) -> dict[str, Any]:
+        return {"backend": self.name, "host": self.base_url}
+
+    def chat(self, messages: list[dict[str, Any]], tools: list[dict[str, Any]]) -> BackendResponse:
+        payload: dict[str, Any] = {
+            "model": self.model,
+            "messages": to_openai_messages(messages),
+            "tools": tools,
+            "stream": False,
+        }
+        for key in ("temperature", "seed"):
+            if key in self.options:
+                payload[key] = self.options[key]
+        started = time.monotonic()
+        data = request_json("POST", f"{self.base_url}/v1/chat/completions", payload, self.timeout)
+        elapsed = time.monotonic() - started
+        try:
+            message = data["choices"][0]["message"]
+        except (KeyError, IndexError, TypeError) as exc:
+            raise BackendError("response has no choices[0].message") from exc
+        calls = []
+        for item in message.get("tool_calls") or []:
+            function = item.get("function") or {}
+            calls.append(RawToolCall(name=str(function.get("name", "")), arguments=function.get("arguments", "")))
+        usage = data.get("usage") or {}
+        timings = data.get("timings") or {}
+        eval_seconds = float(timings["predicted_ms"]) / 1000 if "predicted_ms" in timings else elapsed
+        return BackendResponse(
+            content=message.get("content") or "",
+            tool_calls=calls,
+            eval_tokens=int(usage.get("completion_tokens") or 0),
+            eval_seconds=eval_seconds,
+            prompt_tokens=int(usage.get("prompt_tokens") or 0),
+        )

toolcallbench/backends/scripted.py

@@ -0,0 +1,90 @@
+"""A fake model that replays each task's reference solution, or fails in a chosen way.
+
+It is used to validate the tasks, to test the harness without a real model, and for dry runs.
+"""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from ..models import BackendResponse, RawToolCall, Task
+from .base import Backend, BackendError
+
+PROFILES = (
+    "oracle",
+    "invalid_json",
+    "unknown_tool",
+    "wrong_tool",
+    "text_call",
+    "no_call",
+    "call_limit",
+    "bad_edit",
+)
+
+
+class ScriptedBackend(Backend):
+    name = "scripted"
+
+    def __init__(self, tasks: list[Task], profile: str = "oracle"):
+        if profile not in PROFILES:
+            raise ValueError(f"unknown profile '{profile}'; choose from {', '.join(PROFILES)}")
+        self.by_prompt = {task.prompt: task for task in tasks}
+        self.profile = profile
+
+    def describe(self) -> dict[str, Any]:
+        return {"backend": self.name, "profile": self.profile}
+
+    def chat(self, messages: list[dict[str, Any]], tools: list[dict[str, Any]]) -> BackendResponse:
+        prompt = next((m["content"] for m in messages if m["role"] == "user"), "")
+        task = self.by_prompt.get(prompt)
+        if task is None:
+            raise BackendError("scripted backend received a prompt that matches no task")
+        turn = sum(1 for m in messages if m["role"] == "assistant")
+        content, calls = self._step(task, turn)
+        return BackendResponse(content=content, tool_calls=calls, eval_tokens=8, eval_seconds=0.01, prompt_tokens=100)
+
+    # ------------------------------------------------------------------ steps
+
+    @staticmethod
+    def _oracle(task: Task, turn: int) -> tuple[str, list[RawToolCall]]:
+        if turn >= len(task.reference):
+            return "Done.", []
+        step = task.reference[turn]
+        if "calls" in step:
+            return "", [RawToolCall(c["tool"], dict(c.get("args", {}))) for c in step["calls"]]
+        return str(step["final"]), []
+
+    def _step(self, task: Task, turn: int) -> tuple[str, list[RawToolCall]]:
+        profile = self.profile
+        if profile == "oracle":
+            return self._oracle(task, turn)
+        if profile == "call_limit":
+            return "", [RawToolCall("list_dir", {"path": "."})]
+        if profile == "no_call":
+            return "I believe the answer is 42.", []
+        if profile == "bad_edit":
+            content, calls = self._oracle(task, turn)
+            for call in calls:
+                if call.name == "edit_file":
+                    call.arguments = {**call.arguments, "old_text": "THIS TEXT IS NOT IN THE FILE"}
+                elif call.name == "apply_patch":
+                    call.arguments = {"patch": "--- a/missing.py\n+++ b/missing.py\n@@ -1,1 +1,1 @@\n-x\n+y\n"}
+            return content, calls
+        if turn >= 1:
+            return "Done.", []
+        first = next((c for step in task.reference for c in step.get("calls", [])), None)
+        if first is None:
+            # The task needs no tools. A wrong_tool model calls one anyway; the others answer correctly.
+            if profile == "wrong_tool":
+                return "", [RawToolCall("list_dir", {})]
+            return self._oracle(task, turn)
+        tool, args = first["tool"], dict(first.get("args", {}))
+        if profile == "invalid_json":
+            return "", [RawToolCall(tool, '{"path": ')]
+        if profile == "unknown_tool":
+            return "", [RawToolCall("search_everything", {"query": "x"})]
+        if profile == "wrong_tool":
+            return "", [RawToolCall("list_dir", {})]
+        if profile == "text_call":
+            return json.dumps({"name": tool, "arguments": args}), []
+        raise AssertionError(profile)  # pragma: no cover

toolcallbench/checks.py

@@ -0,0 +1,225 @@
+"""Deterministic scoring of one run. There is no AI judge: every verdict comes from plain code.
+
+A run passes when Parse, Tool and Outcome all succeed. Arguments and Efficiency are reported separately.
+"""
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+from typing import Any
+
+from .models import CallRecord, CheckResult, Task, Transcript
+from .tools import run_trusted
+
+EDIT_TOOLS = {"edit_file", "apply_patch"}
+
+FAILURE_TYPES = (
+    "invalid_json",
+    "invalid_args",
+    "unknown_tool",
+    "text_tool_call",
+    "no_call",
+    "unneeded_call",
+    "call_limit",
+    "not_parallel",
+    "wrong_tool",
+    "too_few_calls",
+    "patch_failed",
+    "wrong_args",
+    "outcome_mismatch",
+)
+
+
+def _norm_path(value: Any) -> str:
+    text = str(value).strip().replace("\\", "/")
+    while text.startswith("./"):
+        text = text[2:]
+    if len(text) > 1:
+        text = text.rstrip("/")
+    return text or "."
+
+
+def _norm(key: str, value: Any) -> str:
+    return _norm_path(value) if key == "path" else str(value)
+
+
+def _value_matches(spec: Any, actual: Any, key: str) -> bool:
+    got = _norm(key, actual)
+    if isinstance(spec, dict):
+        if "contains" in spec:
+            return str(spec["contains"]) in got
+        if "regex" in spec:
+            return re.search(spec["regex"], got) is not None
+        if "one_of" in spec:
+            return got in [_norm(key, v) for v in spec["one_of"]]
+        raise ValueError(f"unknown argument matcher: {sorted(spec)}")
+    return got == _norm(key, spec)
+
+
+def _tools(req: dict[str, Any]) -> list[str]:
+    tool = req["tool"]
+    return list(tool) if isinstance(tool, list) else [tool]
+
+
+def _tool_matches(req: dict[str, Any], call: CallRecord) -> bool:
+    return call.valid and call.tool in _tools(req)
+
+
+def _args_match(req: dict[str, Any], call: CallRecord) -> bool:
+    args = call.arguments if isinstance(call.arguments, dict) else {}
+    return all(key in args and _value_matches(spec, args[key], key) for key, spec in (req.get("args") or {}).items())
+
+
+def _assign(reqs: list[dict[str, Any]], calls: list[CallRecord], ordered: bool, start: int = 0, used: frozenset = frozenset()):
+    """Match each requirement to a distinct call by tool name. Returns the call positions or None."""
+    if not reqs:
+        return []
+    first, rest = reqs[0], reqs[1:]
+    for position in range(start if ordered else 0, len(calls)):
+        if position in used or not _tool_matches(first, calls[position]):
+            continue
+        tail = _assign(rest, calls, ordered, position + 1, used | {position})
+        if tail is not None:
+            return [position, *tail]
+    return None
+
+
+def _parallel_ok(reqs: list[dict[str, Any]], calls: list[CallRecord]) -> bool:
+    turns: dict[int, list[CallRecord]] = {}
+    for call in calls:
+        turns.setdefault(call.turn, []).append(call)
+    return any(_assign(reqs, group, False) is not None for group in turns.values())
+
+
+def _check_outcome(task: Task, tr: Transcript, root: Path, fixture_dir: Path, details: list[str]) -> bool:
+    exp = task.expect
+    ok = True
+
+    def fail(message: str) -> None:
+        nonlocal ok
+        ok = False
+        details.append(message)
+
+    answer = tr.final_text or ""
+    for needle in exp.get("answer_contains", []):
+        if needle.lower() not in answer.lower():
+            fail(f"answer does not contain {needle!r}")
+    for pattern in exp.get("answer_regex", []):
+        if not re.search(pattern, answer, re.IGNORECASE):
+            fail(f"answer does not match /{pattern}/")
+    for rel in exp.get("files_exist", []):
+        if not (root / rel).is_file():
+            fail(f"expected file {rel} does not exist")
+    for rel, spec in (exp.get("final_files") or {}).items():
+        path = root / rel
+        if not path.is_file():
+            fail(f"expected file {rel} does not exist")
+            continue
+        content = path.read_text(encoding="utf-8", errors="replace")
+        for text in spec.get("contains", []):
+            if text not in content:
+                fail(f"{rel} does not contain {text!r}")
+        for text in spec.get("not_contains", []):
+            if text in content:
+                fail(f"{rel} still contains {text!r}")
+        if spec.get("unchanged") and content != (fixture_dir / rel).read_text(encoding="utf-8"):
+            fail(f"{rel} was changed but must stay as it was")
+    for item in exp.get("command", []):
+        try:
+            proc = run_trusted(item["argv"], root)
+        except subprocess.TimeoutExpired:
+            fail(f"check command timed out: {item['argv']}")
+            continue
+        want = item.get("exit_code", 0)
+        if proc.returncode != want:
+            tail = (proc.stdout + proc.stderr).strip()[-300:]
+            fail(f"check command exited {proc.returncode}, wanted {want}: {tail}")
+    return ok
+
+
+def evaluate(task: Task, tr: Transcript, root: Path, fixture_dir: Path, default_max_calls: int = 12) -> CheckResult:
+    exp = task.expect
+    details: list[str] = []
+    calls = tr.calls
+    valid = [c for c in calls if c.valid]
+
+    # Parse: every call must be well formed and name a real tool.
+    parse_type = ""
+    if tr.text_tool_call:
+        parse_type = "text_tool_call"
+    else:
+        bad = next((c for c in calls if not c.valid), None)
+        if bad:
+            parse_type = bad.error_kind
+    parse_ok = parse_type == ""
+    if parse_type:
+        details.append(f"parse: {parse_type}")
+
+    # Tool: the right tools, in the right amount.
+    reqs = exp.get("calls") or []
+    ordered = bool(exp.get("ordered"))
+    max_calls = exp.get("max_calls", default_max_calls)
+    min_calls = exp.get("min_calls", len(reqs))
+    tool_type = ""
+    if tr.hit_limit:
+        tool_type = "call_limit"
+    elif exp.get("no_tool_calls"):
+        if calls:
+            tool_type = "unneeded_call"
+    elif not calls:
+        tool_type = "no_call" if min_calls > 0 else ""
+    elif len(calls) > max_calls:
+        tool_type = "call_limit"
+    elif len(calls) < min_calls:
+        tool_type = "too_few_calls"
+    elif any(c.tool in exp.get("forbidden_tools", []) for c in calls):
+        tool_type = "wrong_tool"
+    elif reqs and _assign(reqs, valid, ordered) is None:
+        tool_type = "wrong_tool"
+    elif exp.get("parallel") and not _parallel_ok(reqs, valid):
+        tool_type = "not_parallel"
+    tool_ok = tool_type == ""
+    if tool_type:
+        details.append(f"tool: {tool_type}")
+
+    # Arguments: reported separately from pass/fail.
+    args_ok = True
+    for req in reqs:
+        if req.get("args") and not any(_tool_matches(req, c) and _args_match(req, c) for c in valid):
+            args_ok = False
+            details.append(f"arguments: no {_tools(req)} call matched {req['args']}")
+
+    # Outcome: the answer text and the final state of the repository.
+    outcome_ok = _check_outcome(task, tr, root, fixture_dir, details)
+
+    if not parse_ok:
+        failure = parse_type
+    elif not tool_ok:
+        failure = tool_type
+    elif not outcome_ok:
+        if any(c.tool in EDIT_TOOLS and c.valid and not c.ok for c in calls):
+            failure = "patch_failed"
+        elif not args_ok:
+            failure = "wrong_args"
+        else:
+            failure = "outcome_mismatch"
+    else:
+        failure = "ok"
+
+    baseline = task.baseline_calls
+    if baseline > 0:
+        efficiency: float | None = round(len(calls) / baseline, 2)
+    else:
+        efficiency = 1.0 if not calls else None
+
+    return CheckResult(
+        parse_ok=parse_ok,
+        tool_ok=tool_ok,
+        args_ok=args_ok,
+        outcome_ok=outcome_ok,
+        passed=parse_ok and tool_ok and outcome_ok,
+        failure_type=failure,
+        efficiency=efficiency,
+        details=details,
+    )