callprobe 0.5.0__py3-none-any.whl

callprobe/__init__.py

@@ -0,0 +1,10 @@
+"""callprobe: test whether a model can actually call your tools."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("callprobe")
+except PackageNotFoundError:  # running from a source checkout, not installed
+    __version__ = "0.0.0+unknown"

callprobe/bootstrap.py

@@ -0,0 +1,49 @@
+"""Bootstrap confidence intervals for a success rate.
+
+At temperature 0 the repeats of a task mostly vary tool order, not the
+model's reasoning, so results from the same task id are correlated with
+each other rather than independent trials. Resampling individual results
+would understate the true uncertainty. Resampling task ids instead, and
+pulling every result for each sampled id along with it, keeps that
+correlation intact.
+"""
+
+from __future__ import annotations
+
+import random
+from collections import defaultdict
+from typing import Any
+
+SEED = 1234
+ITERATIONS = 2000
+
+
+def bootstrap_ci(
+    results: list[Any],
+    field: str = "success",
+    *,
+    confidence: float = 0.95,
+    iterations: int = ITERATIONS,
+    seed: int = SEED,
+) -> tuple[float, float]:
+    by_task: dict[str, list[Any]] = defaultdict(list)
+    for r in results:
+        by_task[r.task_id].append(r)
+    task_ids = list(by_task)
+    if not task_ids:
+        return (0.0, 0.0)
+
+    rng = random.Random(seed)
+    n = len(task_ids)
+    rates = []
+    for _ in range(iterations):
+        pooled = []
+        for _ in range(n):
+            pooled.extend(by_task[task_ids[rng.randrange(n)]])
+        rates.append(sum(1 for r in pooled if getattr(r, field)) / len(pooled))
+    rates.sort()
+
+    tail = (1 - confidence) / 2
+    lo = rates[int(tail * iterations)]
+    hi = rates[min(int((1 - tail) * iterations), iterations - 1)]
+    return (lo, hi)

callprobe/cli.py

@@ -0,0 +1,346 @@
+"""Command line interface.
+
+    callprobe run --model qwen3:8b --endpoint http://localhost:11434/v1
+    callprobe run --model llama3.1:8b --pad 0,8,16 --repeats 3 --quant q4_K_M
+    callprobe leaderboard results/qwen3-8b.json results/llama31-8b.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.resources
+import json
+import os
+import sys
+import tempfile
+from pathlib import Path
+
+import yaml
+
+from . import __version__
+from .client import ChatClient, probe_server_version
+from .compare import category_deltas, flipped_tasks, render_compare
+from .gates import evaluate_gate, load_policy, render_gate
+from .init import generate_suite_files
+from .loader import load_suite
+from .models import Run, RunConfig
+from .report import failure_digest, render_markdown, render_text, summarize
+from .runner import prepare_config, run_suite, validate_resume
+from .validate import validate_suite
+
+# Sentinel meaning "use the suite packaged inside callprobe itself", resolved
+# lazily so a git checkout and a pip install both find suites/core.
+DEFAULT_SUITE = None
+
+
+def _read_run(path: str) -> Run:
+    return Run.model_validate_json(Path(path).read_text(encoding="utf-8"))
+
+
+def _write_run(path: str, run: Run) -> None:
+    """An interrupted write must not destroy a resumable checkpoint."""
+    out = Path(path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    temporary = None
+    try:
+        with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=out.parent,
+                                         prefix=f".{out.name}.", delete=False) as handle:
+            temporary = Path(handle.name)
+            handle.write(run.model_dump_json(indent=2))
+        temporary.replace(out)
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)
+
+
+def _resolve_suite(suite_arg: str | None):
+    if suite_arg is None:
+        suite_path = importlib.resources.files("callprobe") / "suites" / "core"
+        suite_label = "callprobe/suites/core (packaged)"
+    else:
+        suite_path = suite_arg
+        suite_label = suite_arg
+    return load_suite(suite_path), suite_label
+
+
+def _json_safe(value):
+    """inf shows up as tokens/seconds per success with no successes.
+
+    json.dumps emits it as a bare Infinity token, which is not valid JSON
+    per the spec and trips up strict parsers like jq. Map it to null.
+    """
+    if isinstance(value, float) and (value != value or value in (float("inf"), float("-inf"))):
+        return None
+    if isinstance(value, dict):
+        return {k: _json_safe(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_json_safe(v) for v in value]
+    return value
+
+
+def _run(args: argparse.Namespace) -> int:
+    suite, suite_label = _resolve_suite(args.suite)
+
+    pads = [int(p) for p in args.pad.split(",") if p.strip()]
+    server_name, server_version = probe_server_version(args.endpoint)
+    config = RunConfig(
+        model=args.model,
+        endpoint=args.endpoint,
+        suite=suite_label,
+        pads=pads,
+        repeats=args.repeats,
+        temperature=args.temperature,
+        max_tokens=args.max_tokens,
+        quantization=args.quant,
+        notes=args.notes,
+        callprobe_version=__version__,
+        suite_name=suite.name,
+        suite_version=suite.version,
+        suite_hash=suite.hash,
+        server_name=server_name,
+        server_version=server_version,
+    )
+    api_key = args.api_key or os.getenv("API_KEY") or os.getenv("OPENAI_API_KEY")
+    config = prepare_config(suite, config)
+
+    resume_run = None
+    if args.resume:
+        resume_run = _read_run(args.resume)
+        validate_resume(config, resume_run)
+
+    client = ChatClient(args.endpoint, api_key=api_key, retries=args.retries)
+
+    total = len(suite.tasks) * len(pads) * args.repeats
+    state = {"done": 0}
+
+    def progress(result) -> None:
+        state["done"] += 1
+        if not args.quiet:
+            mark = "." if result.success else "x"
+            sys.stderr.write(mark)
+            if state["done"] % 50 == 0:
+                sys.stderr.write(f" {state['done']}/{total}\n")
+            sys.stderr.flush()
+
+    def write_partial(partial_run: Run) -> None:
+        if args.out:
+            _write_run(args.out, partial_run)
+
+    try:
+        run = run_suite(
+            suite,
+            client,
+            config,
+            on_result=progress,
+            on_progress=write_partial if args.out else None,
+            concurrency=args.concurrency,
+            resume=resume_run,
+        )
+    finally:
+        client.close()
+    if not args.quiet:
+        sys.stderr.write("\n\n")
+
+    summary = summarize(run)
+    if args.format == "json":
+        print(json.dumps(_json_safe(summary), indent=2))
+    else:
+        print(render_text(run))
+        print()
+        print(failure_digest(run))
+
+    if args.out:
+        _write_run(args.out, run)
+        if args.format != "json":
+            print(f"\nwrote {args.out}")
+
+    if args.fail_under is not None:
+        if summary["errors"] or len(run.results) != total or not summary["n"]:
+            sys.stderr.write("CI gate failed: run is incomplete or contains request errors\n")
+            return 1
+        if summary["overall"]["success"] < args.fail_under:
+            return 1
+    return 0
+
+
+def _validate(args: argparse.Namespace) -> int:
+    suite, suite_label = _resolve_suite(args.suite)
+    problems = validate_suite(suite)
+    print(f"suite: {suite_label}  ({len(suite.tasks)} tasks)")
+    for problem in problems:
+        print(f"  {problem}")
+    if problems:
+        print(f"\n{len(problems)} problem(s)")
+        return 1
+    print("no problems found")
+    return 0
+
+
+def _init(args: argparse.Namespace) -> int:
+    try:
+        data = json.loads(Path(args.from_file).read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        sys.stderr.write(f"error: {args.from_file} not found\n")
+        return 1
+
+    out = Path(args.out)
+    try:
+        files = generate_suite_files(data, out.name)
+    except ValueError as exc:
+        sys.stderr.write(f"error: {exc}\n")
+        return 1
+
+    out.mkdir(parents=True, exist_ok=True)
+    for filename, content in files.items():
+        (out / filename).write_text(content, encoding="utf-8")
+
+    print(f"wrote {len(files)} files to {out}/")
+    print("uncomment and fill in the example tasks in tasks.yaml, then run:")
+    print(f"  callprobe validate --suite {out}")
+    return 0
+
+
+def _compare(args: argparse.Namespace) -> int:
+    a, b = _read_run(args.a), _read_run(args.b)
+    gate = None
+    if args.fail_on_regression or args.policy:
+        policy = load_policy(args.policy)
+        if args.fail_on_regression:
+            policy.fail_on_regression = True
+        gate = evaluate_gate(a, b, policy)
+    if a.config.suite_hash != b.config.suite_hash:
+        sys.stderr.write(
+            f"warning: suite hashes differ (a={a.config.suite_hash}, "
+            f"b={b.config.suite_hash}); some of this delta may be the suite "
+            "changing, not the model\n"
+        )
+    if a.config.scoring_version != b.config.scoring_version:
+        sys.stderr.write("warning: scoring versions differ; scores are not directly comparable\n")
+    if args.format == "json":
+        regressions, improvements = flipped_tasks(a, b)
+        print(json.dumps({"by_category": category_deltas(a, b),
+                          "regressed_tasks": regressions, "improved_tasks": improvements,
+                          "gate": gate}, indent=2))
+    else:
+        print(render_compare(a, b))
+        if gate is not None:
+            print("\n" + render_gate(gate))
+    return 1 if gate is not None and not gate["passed"] else 0
+
+
+def _leaderboard(args: argparse.Namespace) -> int:
+    runs = []
+    for path in args.results:
+        data = json.loads(Path(path).read_text(encoding="utf-8"))
+        runs.append(Run(**data))
+    runs.sort(key=lambda r: r.config.model)
+
+    keys = {(r.config.suite_version, r.config.suite_hash, r.config.scoring_version) for r in runs}
+    if len(keys) > 1 and not args.allow_mixed:
+        sys.stderr.write("these runs come from different suites or scoring versions:\n")
+        for r in runs:
+            sys.stderr.write(
+                f"  {r.config.model}: version={r.config.suite_version} "
+                f"hash={r.config.suite_hash} scoring={r.config.scoring_version}\n"
+            )
+        sys.stderr.write("pass --allow-mixed to build the table anyway\n")
+        return 1
+
+    print(render_markdown(runs))
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="callprobe")
+    parser.add_argument(
+        "--version", action="version", version=f"callprobe {__version__}"
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    run_cmd = sub.add_parser("run", help="score a model against a suite")
+    run_cmd.add_argument("--model", required=True)
+    run_cmd.add_argument("--endpoint", default="http://localhost:11434/v1")
+    run_cmd.add_argument("--api-key", default=None)
+    run_cmd.add_argument(
+        "--retries", type=int, default=3, help="retries on 429, 5xx, and connection errors"
+    )
+    run_cmd.add_argument(
+        "--suite", default=DEFAULT_SUITE, help="suite directory, defaults to the packaged core suite"
+    )
+    run_cmd.add_argument("--pad", default="0,8,16")
+    run_cmd.add_argument("--repeats", type=int, default=1)
+    run_cmd.add_argument("--temperature", type=float, default=0.0)
+    run_cmd.add_argument("--max-tokens", type=int, default=2048)
+    run_cmd.add_argument("--quant", default=None, help="label only, e.g. q4_K_M")
+    run_cmd.add_argument("--notes", default=None)
+    run_cmd.add_argument("--out", default=None, help="write raw results as JSON")
+    run_cmd.add_argument("--quiet", action="store_true")
+    run_cmd.add_argument("--format", choices=["text", "json"], default="text")
+    run_cmd.add_argument(
+        "--fail-under",
+        type=float,
+        default=None,
+        help="exit 1 if overall success is below this fraction, e.g. 0.7",
+    )
+    run_cmd.add_argument(
+        "--concurrency", type=int, default=1, help="parallel requests via a thread pool"
+    )
+    run_cmd.add_argument(
+        "--resume",
+        default=None,
+        help="skip (task, pad, repeat) combinations already in this results file",
+    )
+    run_cmd.set_defaults(func=_run)
+
+    init_cmd = sub.add_parser(
+        "init", help="scaffold a suite from an OpenAI-format tools.json"
+    )
+    init_cmd.add_argument(
+        "--from", dest="from_file", required=True, help="path to a tools.json"
+    )
+    init_cmd.add_argument("--out", default="suite", help="directory to write the suite to")
+    init_cmd.set_defaults(func=_init)
+
+    validate_cmd = sub.add_parser(
+        "validate", help="check task expectations against tool schemas, no model needed"
+    )
+    validate_cmd.add_argument(
+        "--suite", default=DEFAULT_SUITE, help="suite directory, defaults to the packaged core suite"
+    )
+    validate_cmd.set_defaults(func=_validate)
+
+    compare_cmd = sub.add_parser(
+        "compare", help="diff two runs: per-category deltas and which tasks flipped"
+    )
+    compare_cmd.add_argument("a")
+    compare_cmd.add_argument("b")
+    compare_cmd.add_argument("--fail-on-regression", action="store_true",
+                             help="fail if any matched passing case regresses; requires complete runs")
+    compare_cmd.add_argument("--policy", help="YAML CI policy; enables gating")
+    compare_cmd.add_argument("--format", choices=["text", "json"], default="text")
+    compare_cmd.set_defaults(func=_compare)
+
+    board = sub.add_parser("leaderboard", help="build a markdown table from runs")
+    board.add_argument("results", nargs="+")
+    board.add_argument(
+        "--allow-mixed",
+        action="store_true",
+        help="build the table even if runs come from different suite versions or content",
+    )
+    board.set_defaults(func=_leaderboard)
+
+    args = parser.parse_args(argv)
+    try:
+        if args.command == "run":
+            if args.concurrency < 1 or args.max_tokens < 1 or args.retries < 0:
+                raise ValueError("concurrency/max-tokens must be positive and retries nonnegative")
+            if args.fail_under is not None and not 0 <= args.fail_under <= 1:
+                raise ValueError("fail-under must be between 0 and 1")
+        return args.func(args)
+    except (ValueError, OSError, yaml.YAMLError) as exc:
+        sys.stderr.write(f"error: {exc}\n")
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

callprobe/client.py

@@ -0,0 +1,196 @@
+"""A deliberately small OpenAI-compatible client.
+
+We avoid the official SDK so that any endpoint that speaks
+/v1/chat/completions works: Ollama, LM Studio, llama.cpp server, vLLM,
+or a hosted provider.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+import time
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import urlsplit, urlunsplit
+
+import httpx
+
+from .models import Call
+
+DEFAULT_RETRIES = 3
+
+
+def probe_server_version(endpoint: str, timeout: float = 2.0) -> tuple[str | None, str | None]:
+    """Best-effort server identification, for provenance in RunConfig.
+
+    Only Ollama's /api/version is checked. Any other server, or no
+    response, leaves both fields null rather than guessing.
+    """
+    parts = urlsplit(endpoint)
+    root = urlunsplit((parts.scheme, parts.netloc, "", "", ""))
+    try:
+        response = httpx.get(f"{root}/api/version", timeout=timeout)
+        response.raise_for_status()
+        version = response.json().get("version")
+    except Exception:  # noqa: BLE001 - purely informational
+        return None, None
+    return ("ollama", version) if version else (None, None)
+
+
+@dataclass
+class Completion:
+    calls: list[Call] = field(default_factory=list)
+    content: str = ""
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    latency_ms: float = 0.0
+    error: str | None = None
+    finish_reason: str = ""
+    # Some servers put chain of thought in its own field rather than content.
+    reasoning: str = ""
+    raw: dict[str, Any] = field(default_factory=dict)
+
+
+def _is_retryable(status_code: int) -> bool:
+    return status_code == 429 or status_code >= 500
+
+
+class ChatClient:
+    def __init__(
+        self,
+        endpoint: str,
+        api_key: str | None = None,
+        timeout: float = 120.0,
+        retries: int = DEFAULT_RETRIES,
+        transport: httpx.BaseTransport | None = None,
+    ) -> None:
+        self.endpoint = endpoint.rstrip("/")
+        headers = {"Content-Type": "application/json"}
+        if api_key:
+            headers["Authorization"] = f"Bearer {api_key}"
+        self._client = httpx.Client(timeout=timeout, headers=headers, transport=transport)
+        self.retries = retries
+
+    def close(self) -> None:
+        self._client.close()
+
+    def _backoff(self, attempt: int, retry_after: str | None) -> float:
+        """Exponential backoff with full jitter, honoring Retry-After."""
+        if retry_after is not None:
+            try:
+                return max(0.0, float(retry_after))
+            except ValueError:
+                pass
+        ceiling = min(20.0, 0.5 * (2**attempt))
+        return random.uniform(0, ceiling)
+
+    def complete(
+        self,
+        model: str,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]],
+        temperature: float = 0.0,
+        max_tokens: int = 512,
+    ) -> Completion:
+        payload = {
+            "model": model,
+            "messages": messages,
+            "temperature": temperature,
+            "max_tokens": max_tokens,
+        }
+        if tools:
+            payload["tools"] = tools
+            payload["tool_choice"] = "auto"
+
+        started = time.perf_counter()
+        attempts = self.retries + 1
+        for attempt in range(attempts):
+            last_attempt = attempt + 1 == attempts
+            try:
+                response = self._client.post(
+                    f"{self.endpoint}/chat/completions", json=payload
+                )
+            except httpx.TransportError as exc:
+                if last_attempt:
+                    return Completion(
+                        latency_ms=(time.perf_counter() - started) * 1000,
+                        error=f"{type(exc).__name__}: {exc}",
+                    )
+                time.sleep(self._backoff(attempt, None))
+                continue
+
+            if _is_retryable(response.status_code) and not last_attempt:
+                time.sleep(self._backoff(attempt, response.headers.get("Retry-After")))
+                continue
+
+            try:
+                response.raise_for_status()
+                body = response.json()
+            except Exception as exc:  # noqa: BLE001 - reported, not raised
+                return Completion(
+                    latency_ms=(time.perf_counter() - started) * 1000,
+                    error=f"{type(exc).__name__}: {exc}",
+                )
+
+            return parse_completion(body, (time.perf_counter() - started) * 1000)
+
+        raise AssertionError("unreachable: loop always returns or retries")
+
+
+def parse_completion(body: dict[str, Any], latency_ms: float) -> Completion:
+    """Turn a chat completion body into calls, tolerating provider quirks."""
+    usage = body.get("usage") or {}
+    choices = body.get("choices") or [{}]
+    choice = choices[0] or {}
+    message = choice.get("message") or {}
+    finish_reason = choice.get("finish_reason") or ""
+    reasoning = message.get("reasoning") or message.get("reasoning_content") or ""
+
+    calls: list[Call] = []
+    for entry in message.get("tool_calls") or []:
+        function = entry.get("function") or {}
+        raw_args = function.get("arguments")
+        if isinstance(raw_args, dict):
+            calls.append(
+                Call(
+                    id=entry.get("id"),
+                    name=function.get("name", ""),
+                    arguments=raw_args,
+                    raw_arguments=json.dumps(raw_args),
+                )
+            )
+            continue
+        raw_args = raw_args or ""
+        try:
+            parsed = json.loads(raw_args) if raw_args.strip() else {}
+            if not isinstance(parsed, dict):
+                raise ValueError("arguments were not a JSON object")
+            calls.append(
+                Call(
+                    id=entry.get("id"),
+                    name=function.get("name", ""),
+                    arguments=parsed,
+                    raw_arguments=raw_args,
+                )
+            )
+        except Exception as exc:  # noqa: BLE001
+            calls.append(
+                Call(
+                    id=entry.get("id"),
+                    name=function.get("name", ""),
+                    raw_arguments=raw_args,
+                    parse_error=str(exc),
+                )
+            )
+
+    return Completion(
+        calls=calls,
+        content=message.get("content") or "",
+        prompt_tokens=int(usage.get("prompt_tokens") or 0),
+        completion_tokens=int(usage.get("completion_tokens") or 0),
+        latency_ms=latency_ms,
+        finish_reason=finish_reason,
+        reasoning=reasoning,
+        raw=body,
+    )

callprobe/coerce.py

@@ -0,0 +1,62 @@
+"""Separate wrong-type from wrong-value."""
+from __future__ import annotations
+import json
+from typing import Any
+
+TRUE = {"true", "yes", "1"}
+FALSE = {"false", "no", "0"}
+
+def _coerce_scalar(value, kind):
+    if not isinstance(value, str):
+        return value
+    text = value.strip()
+    try:
+        if kind == "integer":
+            return int(text, 10) if text.lstrip("-").isdigit() else int(float(text))
+        if kind == "number":
+            return float(text)
+        if kind == "boolean":
+            low = text.lower()
+            if low in TRUE:
+                return True
+            if low in FALSE:
+                return False
+    except (TypeError, ValueError):
+        return value
+    return value
+
+def _types(schema):
+    declared = schema.get("type")
+    if isinstance(declared, str):
+        return [declared]
+    if isinstance(declared, list):
+        return [t for t in declared if isinstance(t, str)]
+    return []
+
+def coerce(value, schema):
+    if not isinstance(schema, dict):
+        return value
+    kinds = _types(schema)
+    if isinstance(value, str) and ({"array", "object"} & set(kinds)):
+        text = value.strip()
+        if text[:1] in "[{":
+            try:
+                value = json.loads(text)
+            except json.JSONDecodeError:
+                return value
+    if isinstance(value, dict):
+        properties = schema.get("properties") or {}
+        return {k: coerce(v, properties.get(k)) for k, v in value.items()}
+    if isinstance(value, list):
+        items = schema.get("items")
+        return [coerce(v, items) for v in value]
+    for kind in kinds:
+        if kind in ("integer", "number", "boolean"):
+            return _coerce_scalar(value, kind)
+    return value
+
+def coerce_arguments(arguments, parameters):
+    if not isinstance(arguments, dict):
+        return arguments, False
+    coerced = coerce(arguments, parameters or {})
+    return coerced, coerced != arguments