whetkit 0.1.1__tar.gz → 0.3.0__tar.gz

{whetkit-0.1.1 → whetkit-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whetkit
-Version: 0.1.1
+Version: 0.3.0
 Summary: Measure and improve how well LLM agents select and use the tools exposed by an MCP server
 Keywords: mcp,model-context-protocol,llm,agents,evals,tool-use,tool-selection,benchmark
 Author: Mohammed Benlamlih

{whetkit-0.1.1 → whetkit-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "whetkit"
-version = "0.1.1"
+version = "0.3.0"
 description = "Measure and improve how well LLM agents select and use the tools exposed by an MCP server"
 readme = "README.md"
 license = "Apache-2.0"

{whetkit-0.1.1 → whetkit-0.3.0}/src/whetkit/cli.py

@@ -16,6 +16,14 @@ app = typer.Typer(
 )
 
 
+def _resolve_server(value: str, http_mode: HttpMode) -> ServerSpec:
+    """resolve_server_spec with a CLI-friendly error instead of a traceback."""
+    try:
+        return resolve_server_spec(value, http_mode=http_mode)
+    except ValueError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+
+
 def _judge_enabled(judge: str, judge_model: str) -> bool:
     """--judge auto: grade with the LLM judge only when its API key is set."""
     if judge in ("on", "off"):
@@ -31,10 +39,10 @@ def _resolve_task_servers(
     tasks: list, server_override: str | None, http_mode: HttpMode
 ) -> dict[str, ServerSpec]:
     if server_override is not None:
-        spec = resolve_server_spec(server_override, http_mode=http_mode)
+        spec = _resolve_server(server_override, http_mode)
         return {task.server: spec for task in tasks}
     return {
-        task.server: resolve_server_spec(task.server, http_mode=http_mode)
+        task.server: _resolve_server(task.server, http_mode)
         for task in {t.server: t for t in tasks}.values()
     }
 
@@ -51,6 +59,35 @@ def _write_report(report, out_dir: str) -> tuple[str, str]:
     return str(html_path), str(json_path)
 
 
+def _summary_payload(group_name: str, summary, task_runs: list) -> dict:
+    """One run's metrics as plain data — what --summary-json emits per run."""
+    return {
+        "group": group_name,
+        "hit_rate": summary.hit_rate,
+        "tool_hit_rate": summary.tool_hit_rate,
+        "judge_pass_rate": summary.judge_pass_rate,
+        "avg_precision": summary.avg_precision,
+        "avg_recall": summary.avg_recall,
+        "avg_extra_calls": summary.avg_extra_calls,
+        "tokens_in": sum(r.total_usage.input_tokens for r in task_runs),
+        "tokens_out": sum(r.total_usage.output_tokens for r in task_runs),
+        "latency_ms": sum(r.total_latency_ms for r in task_runs),
+        "tasks": [
+            {
+                "id": score.task_id,
+                "hit": score.hit,
+                "tool_hit": score.tool_hit,
+                "judge_passed": score.judge.passed if score.judge else None,
+                "spec_gap": score.spec_gap,
+                "called": score.tool_match.called,
+                "missing": score.tool_match.missing_slots,
+                "extra_calls": score.tool_match.extra_calls,
+            }
+            for score in summary.scores
+        ],
+    }
+
+
 def _version_callback(value: bool) -> None:
     if value:
         from importlib.metadata import version
@@ -89,7 +126,7 @@ def inspect(
     ] = HttpMode.STATEFUL,
 ) -> None:
     """Connect to an MCP server and print its tool inventory."""
-    spec = resolve_server_spec(server, http_mode=http_mode)
+    spec = _resolve_server(server, http_mode)
     inventory = asyncio.run(inspect_server(spec))
 
     for line in inventory.summary_lines():
@@ -136,7 +173,7 @@ def doctor(
     if fail_on not in ("error", "warn", "never"):
         raise typer.BadParameter("--fail-on must be 'error', 'warn', or 'never'")
 
-    spec = resolve_server_spec(server, http_mode=http_mode)
+    spec = _resolve_server(server, http_mode)
     inventory = asyncio.run(inspect_server(spec))
     findings = diagnose(inventory)
 
@@ -181,16 +218,30 @@ def generate(
     model: Annotated[
         str, typer.Option("--model", help="Generator model as provider:model_id")
     ] = "anthropic:claude-sonnet-5",
+    allow_writes: Annotated[
+        bool,
+        typer.Option(
+            "--allow-writes",
+            help="Let drafts include non-destructive write tasks (default: read-only only)",
+        ),
+    ] = False,
     http_mode: Annotated[HttpMode, typer.Option("--http-mode")] = HttpMode.STATEFUL,
 ) -> None:
     """Draft eval tasks from the server's tool inventory (review before
     trusting — the header in the output file says the same)."""
     from whetkit.generate import GeneratorConfig, generate_tasks, write_tasks_yaml
 
-    spec = resolve_server_spec(server, http_mode=http_mode)
+    spec = _resolve_server(server, http_mode)
     inventory = asyncio.run(inspect_server(spec))
     tasks, warnings = asyncio.run(
-        generate_tasks(inventory, server, count=count, config=GeneratorConfig(model=model))
+        generate_tasks(
+            inventory,
+            server,
+            count=count,
+            config=GeneratorConfig(model=model),
+            server_context=spec.label(),
+            allow_writes=allow_writes,
+        )
     )
     for warning in warnings:
         typer.echo(f"warning: {warning}", err=True)
@@ -204,6 +255,101 @@ def generate(
     typer.echo(f"  whetkit run --server {server} --tasks {out}")
 
 
+@app.command()
+def export(
+    plan: Annotated[
+        str, typer.Option("--plan", help="Curation plan YAML to export")
+    ] = ".whetkit/curation-plan.yaml",
+    to: Annotated[
+        str,
+        typer.Option(
+            "--to",
+            help=(
+                "'markdown' — a fix report ready for an upstream issue/PR; "
+                "'json' — a neutral override list for gateways and scripts"
+            ),
+        ),
+    ] = "markdown",
+    out: Annotated[str | None, typer.Option("--out", help="Write here instead of stdout")] = None,
+) -> None:
+    """Export a curation plan as a shareable fix report or neutral JSON."""
+    from whetkit.curation import load_plan
+    from whetkit.curation.export import plan_to_json, plan_to_markdown
+
+    if to not in ("markdown", "json"):
+        raise typer.BadParameter("--to must be 'markdown' or 'json'")
+    if not Path(plan).is_file():
+        raise typer.BadParameter(
+            f"no curation plan at {plan} — run 'whetkit curate' first, or pass --plan"
+        )
+    curation_plan = load_plan(plan)
+    if not curation_plan.overrides:
+        typer.echo("plan has no overrides — nothing to export", err=True)
+        raise typer.Exit(code=1)
+
+    rendered = (plan_to_markdown if to == "markdown" else plan_to_json)(curation_plan)
+    if out:
+        Path(out).parent.mkdir(parents=True, exist_ok=True)
+        Path(out).write_text(rendered)
+        typer.echo(f"wrote {out}")
+    else:
+        typer.echo(rendered, nl=False)
+
+
+@app.command()
+def diff(
+    before: Annotated[str, typer.Argument(help="Baseline --summary-json file")],
+    after: Annotated[str, typer.Argument(help="Comparison --summary-json file")],
+) -> None:
+    """Compare two --summary-json files: headline deltas plus per-task
+    transitions. The before/after table without re-running anything."""
+    import json as jsonlib
+
+    docs = []
+    for path in (before, after):
+        if not Path(path).is_file():
+            raise typer.BadParameter(f"no summary file at {path}")
+        docs.append(jsonlib.loads(Path(path).read_text()))
+
+    def mean(doc: dict, key: str) -> float:
+        values = [r[key] for r in doc["runs"] if r.get(key) is not None]
+        return sum(values) / len(values) if values else 0.0
+
+    typer.echo(f"{'metric':<28} {'before':>10} {'after':>10}")
+    for label, key, fmt in (
+        ("Hit-rate", "hit_rate", "{:.0%}"),
+        ("Tool-selection hit-rate", "tool_hit_rate", "{:.0%}"),
+        ("Judge pass-rate", "judge_pass_rate", "{:.0%}"),
+        ("Precision (avg)", "avg_precision", "{:.0%}"),
+        ("Unnecessary calls/task", "avg_extra_calls", "{:.1f}"),
+        ("Tokens in (per run)", "tokens_in", "{:.0f}"),
+    ):
+        b, a = mean(docs[0], key), mean(docs[1], key)
+        typer.echo(f"{label:<28} {fmt.format(b):>10} {fmt.format(a):>10}")
+
+    def outcomes(doc: dict) -> dict[str, list[bool]]:
+        per_task: dict[str, list[bool]] = {}
+        for run_doc in doc["runs"]:
+            for task in run_doc["tasks"]:
+                per_task.setdefault(task["id"], []).append(bool(task["hit"]))
+        return per_task
+
+    before_tasks, after_tasks = outcomes(docs[0]), outcomes(docs[1])
+    typer.echo("")
+    for task_id in sorted(set(before_tasks) | set(after_tasks)):
+
+        def word(hits: list[bool] | None) -> str:
+            if not hits:
+                return "—"
+            if all(hits):
+                return "PASS"
+            return "MISS" if not any(hits) else f"FLAKY {sum(hits)}/{len(hits)}"
+
+        b_word, a_word = word(before_tasks.get(task_id)), word(after_tasks.get(task_id))
+        marker = "" if b_word == a_word else ("  ↑" if a_word == "PASS" else "  ↓")
+        typer.echo(f"  {task_id:<28} {b_word:>10} -> {a_word:<10}{marker}")
+
+
 @app.command()
 def run(
     tasks: Annotated[
@@ -250,21 +396,55 @@ def run(
             help="Completion-token budget per model turn (raise for reasoning models)",
         ),
     ] = 1024,
+    runs: Annotated[
+        int,
+        typer.Option(
+            "--runs",
+            help=(
+                "Repeat the whole task set N times and report mean plus range "
+                "— single runs are noise. Trace groups get a -1..-N suffix."
+            ),
+        ),
+    ] = 1,
     store: Annotated[
         str | None,
         typer.Option("--store", help="Trace SQLite path (default ./.whetkit/traces.sqlite3)"),
     ] = None,
     jsonl: Annotated[
-        str | None, typer.Option("--jsonl", help="Also write traces to this JSONL file")
+        str | None,
+        typer.Option(
+            "--jsonl",
+            help="Also write traces to this JSONL file (suffixed per run when --runs > 1)",
+        ),
+    ] = None,
+    summary_json: Annotated[
+        str | None,
+        typer.Option(
+            "--summary-json",
+            help="Write a machine-readable summary (metrics + per-task outcomes) to this path",
+        ),
+    ] = None,
+    reset_cmd: Annotated[
+        str | None,
+        typer.Option(
+            "--reset-cmd",
+            help=(
+                "Shell command run before each repetition (and the first run) to "
+                "reset server-side fixtures — makes --runs honest on stateful servers"
+            ),
+        ),
     ] = None,
     http_mode: Annotated[HttpMode, typer.Option("--http-mode")] = HttpMode.STATEFUL,
 ) -> None:
     """Run eval tasks against an MCP server and print scored results."""
     from whetkit.datasets import load_tasks
     from whetkit.runner import RunConfig, run_task
-    from whetkit.scoring import JudgeCache, JudgeConfig, MatchMode, score_runs
+    from whetkit.scoring import JudgeCache, JudgeConfig, MatchMode, MultiRunSummary, score_runs
     from whetkit.tracing import TraceStore, default_store_path, write_jsonl
 
+    if runs < 1:
+        raise typer.BadParameter("--runs must be at least 1")
+
     task_list = load_tasks(tasks)
     servers = _resolve_task_servers(task_list, server, http_mode)
     config = RunConfig(model=model, max_turns=max_turns, max_tokens=max_tokens)
@@ -286,34 +466,28 @@ def run(
         client_factory = partial(CuratedMCPClient, plan=curation_plan)
         name_map = curation_plan.rename_map()
 
-    async def _run() -> None:
-        runs = []
+    async def _run_once(group_name: str, cache: JudgeCache):
+        task_runs = []
         for task in task_list:
             typer.echo(f"running {task.id} ...", err=True)
-            runs.append(
+            task_runs.append(
                 await run_task(task, servers[task.server], config, client_factory=client_factory)
             )
 
         with TraceStore(store_path) as trace_store:
-            trace_store.save_runs(runs, run_group=group)
-        if jsonl:
-            write_jsonl(runs, jsonl)
+            trace_store.save_runs(task_runs, run_group=group_name)
 
-        cache = JudgeCache(default_store_path().parent / "judge_cache.sqlite3")
-        try:
-            summary = await score_runs(
-                task_list,
-                runs,
-                mode=MatchMode(match_mode),
-                judge_config=JudgeConfig(model=judge_model),
-                judge_cache=cache,
-                use_judge=use_judge,
-                name_map=name_map,
-            )
-        finally:
-            cache.close()
+        summary = await score_runs(
+            task_list,
+            task_runs,
+            mode=MatchMode(match_mode),
+            judge_config=JudgeConfig(model=judge_model),
+            judge_cache=cache,
+            use_judge=use_judge,
+            name_map=name_map,
+        )
 
-        typer.echo(f"\nResults (group '{group}', model {model}):")
+        typer.echo(f"\nResults (group '{group_name}', model {model}):")
         for score in summary.scores:
             mark = "PASS" if score.hit else "MISS"
             tools = " -> ".join(score.tool_match.called) or "(no tool calls)"
@@ -324,19 +498,89 @@ def run(
                 line += f"  judge={'pass' if score.judge.passed else 'FAIL'}"
             typer.echo(line)
         typer.echo("")
+        for score in summary.scores:
+            if score.spec_gap:
+                called = ", ".join(score.tool_match.called) or "(none)"
+                typer.echo(
+                    f"⚠ possible task-spec gap: '{score.task_id}' reached a correct "
+                    f"outcome (judge passed) via tools not listed in expected_tools "
+                    f"— called: {called}"
+                )
         for line in summary.summary_lines():
             typer.echo(line)
-        tokens_in = sum(r.total_usage.input_tokens for r in runs)
-        tokens_out = sum(r.total_usage.output_tokens for r in runs)
+        tokens_in = sum(r.total_usage.input_tokens for r in task_runs)
+        tokens_out = sum(r.total_usage.output_tokens for r in task_runs)
+        from whetkit.llm import parse_model
+        from whetkit.llm.pricing import estimate_cost_usd
+
+        cost = estimate_cost_usd(parse_model(model)[1], tokens_in, tokens_out)
+        cost_note = f"   ≈ ${cost:.4f} (est.)" if cost is not None else ""
         typer.echo(
             f"Tokens in/out: {tokens_in}/{tokens_out}   "
-            f"Total latency: {sum(r.total_latency_ms for r in runs) / 1000:.1f}s"
+            f"Total latency: {sum(r.total_latency_ms for r in task_runs) / 1000:.1f}s"
+            f"{cost_note}"
         )
+        return task_runs, summary
+
+    async def _run() -> None:
+        summaries = []
+        payloads = []
+        cache = JudgeCache(default_store_path().parent / "judge_cache.sqlite3")
+        try:
+            for run_index in range(1, runs + 1):
+                if reset_cmd:
+                    import subprocess
+
+                    typer.echo(f"reset: {reset_cmd}", err=True)
+                    proc = subprocess.run(reset_cmd, shell=True)
+                    if proc.returncode != 0:
+                        typer.echo(
+                            f"error: --reset-cmd failed with exit code {proc.returncode}",
+                            err=True,
+                        )
+                        raise typer.Exit(code=1)
+                group_name = group if runs == 1 else f"{group}-{run_index}"
+                task_runs, summary = await _run_once(group_name, cache)
+                summaries.append(summary)
+                payloads.append(_summary_payload(group_name, summary, task_runs))
+                if jsonl:
+                    write_jsonl(task_runs, jsonl if runs == 1 else f"{jsonl}.{run_index}")
+        finally:
+            cache.close()
+
+        multi = MultiRunSummary(summaries=summaries)
+        if runs > 1:
+            typer.echo(f"\n== across {runs} runs ==")
+            for line in multi.summary_lines():
+                typer.echo(line)
         if judge == "auto" and not use_judge:
             typer.echo(
                 "(LLM judge skipped: no API key found — set ANTHROPIC_API_KEY or pass --judge on)"
             )
-        typer.echo(f"Traces saved to {store_path} (group '{group}')")
+        suffix = f" (groups '{group}-1'..'-{runs}')" if runs > 1 else f" (group '{group}')"
+        typer.echo(f"Traces saved to {store_path}{suffix}")
+
+        if summary_json:
+            import json as jsonlib
+
+            document = {
+                "model": model,
+                "judge_model": judge_model if use_judge else None,
+                "match_mode": match_mode,
+                "plan": plan,
+                "runs": payloads,
+            }
+            if runs > 1:
+                document["aggregate"] = {
+                    "n": multi.n,
+                    "hit_rate_mean": sum(s.hit_rate for s in summaries) / len(summaries),
+                    "hit_rate_min": min(s.hit_rate for s in summaries),
+                    "hit_rate_max": max(s.hit_rate for s in summaries),
+                    "flaky_tasks": multi.flaky_tasks(),
+                }
+            Path(summary_json).parent.mkdir(parents=True, exist_ok=True)
+            Path(summary_json).write_text(jsonlib.dumps(document, indent=2) + "\n")
+            typer.echo(f"Summary JSON: {summary_json}")
 
     asyncio.run(_run())
 
@@ -597,7 +841,7 @@ def overlay(
     the origin is never modified)."""
     from whetkit.curation import load_plan, serve_overlay
 
-    origin = resolve_server_spec(server, http_mode=http_mode)
+    origin = _resolve_server(server, http_mode)
     asyncio.run(serve_overlay(origin, load_plan(plan)))
 
 

whetkit-0.3.0/src/whetkit/curation/export.py

@@ -0,0 +1,73 @@
+"""Export a curation plan into shareable formats.
+
+A plan is most useful when it leaves the machine it was made on: as a
+Markdown fix report you can paste into an upstream issue or PR, or as
+neutral JSON for gateways and scripts that apply metadata overrides
+themselves.
+"""
+
+import json
+
+from whetkit.curation.plan import CurationPlan, ToolOverride
+
+
+def _action(override: ToolOverride) -> str:
+    if override.hidden:
+        return "hide"
+    if override.new_name and override.new_description:
+        return "rename + rewrite"
+    if override.new_name:
+        return "rename"
+    return "rewrite"
+
+
+def plan_to_markdown(plan: CurationPlan) -> str:
+    """A fix report for humans — upstream-issue / PR-description ready."""
+    lines = [
+        "## Proposed tool-surface fixes",
+        "",
+        "Metadata-only changes — no schemas or behavior are touched. "
+        "Measured with [whetkit](https://github.com/benlamlih/whetkit); "
+        "each change is reversible (it can be served as an overlay without "
+        "modifying the server).",
+        "",
+    ]
+    if plan.notes:
+        lines += [f"> {plan.notes.strip()}", ""]
+    lines += [
+        "| Tool | Change | Proposal | Why |",
+        "|---|---|---|---|",
+    ]
+    for override in plan.overrides:
+        if override.hidden:
+            proposal = "hide from the tool list"
+        else:
+            parts = []
+            if override.new_name:
+                parts.append(f"rename to `{override.new_name}`")
+            if override.new_description:
+                parts.append(f'description: "{override.new_description.strip()}"')
+            proposal = "; ".join(parts)
+        cell = lambda s: " ".join(str(s).split()).replace("|", "\\|")  # noqa: E731
+        lines.append(
+            f"| `{override.original_name}` | {_action(override)} "
+            f"| {cell(proposal)} | {cell(override.reason) or '—'} |"
+        )
+    return "\n".join(lines) + "\n"
+
+
+def plan_to_json(plan: CurationPlan) -> str:
+    """Neutral override list for gateways/scripts that apply their own
+    metadata overrides (Cloudflare portal aliases, MetaMCP namespaces, …)."""
+    overrides = [
+        {
+            "original_name": o.original_name,
+            "action": _action(o),
+            "new_name": o.new_name,
+            "new_description": o.new_description,
+            "hidden": o.hidden,
+            "reason": o.reason,
+        }
+        for o in plan.overrides
+    ]
+    return json.dumps({"notes": plan.notes, "overrides": overrides}, indent=2) + "\n"

{whetkit-0.1.1 → whetkit-0.3.0}/src/whetkit/generate.py

@@ -18,7 +18,7 @@ from whetkit.mcp.introspect import ServerInventory
 
 DEFAULT_GENERATOR_MODEL = "anthropic:claude-sonnet-5"
 
-GENERATOR_SYSTEM_PROMPT = """\
+GENERATOR_SYSTEM_PROMPT_TEMPLATE = """\
 You write eval tasks for an AI-agent benchmark. You are given the tools an
 MCP server exposes. Draft tasks that measure whether an agent picks the
 right tools for realistic user requests.
@@ -32,9 +32,7 @@ Rules for every task:
   alternatives for that step. Use ONLY tool names from the given list.
 - success_criteria is one or two concrete, checkable sentences a grader can
   verify from the agent's final answer alone.
-- Prefer read-only tasks. Include a write task only when the tool set is
-  clearly built for writes, and never anything destructive or irreversible
-  (no deletes, resets, or purges).
+{writes_rule}
 - Set "ordered": true only when the steps must happen in sequence.
 - ids are short kebab-case slugs, unique across the set.
 
@@ -59,9 +57,37 @@ def _inventory_block(inventory: ServerInventory) -> str:
     return "\n".join(lines)
 
 
-def build_generator_prompt(inventory: ServerInventory, count: int) -> str:
+READ_ONLY_RULE = (
+    "- Draft ONLY read-only tasks: nothing that creates, modifies, deletes,\n"
+    "  or sends anything. If the tool set is write-heavy, still restrict\n"
+    "  yourself to its read/query tools."
+)
+WRITES_ALLOWED_RULE = (
+    "- Prefer read-only tasks. Write tasks are allowed when the tool set is\n"
+    "  clearly built for writes, but never anything destructive or\n"
+    "  irreversible (no deletes, resets, or purges)."
+)
+
+
+def generator_system_prompt(allow_writes: bool) -> str:
+    rule = WRITES_ALLOWED_RULE if allow_writes else READ_ONLY_RULE
+    # plain replace, not str.format — the template contains JSON braces
+    return GENERATOR_SYSTEM_PROMPT_TEMPLATE.replace("{writes_rule}", rule)
+
+
+def build_generator_prompt(inventory: ServerInventory, count: int, server_context: str = "") -> str:
+    context_block = ""
+    if server_context:
+        context_block = (
+            "## Execution context\n"
+            f"These tasks will execute against: {server_context}\n"
+            "Write prompts whose arguments fit THAT context — the real paths,\n"
+            "repositories, or resources it implies. Never invent placeholder\n"
+            "examples like /home/user/project or acme-corp/website.\n\n"
+        )
     return (
         f"## Tools exposed by the server\n{_inventory_block(inventory)}\n\n"
+        f"{context_block}"
         f"Draft exactly {count} eval tasks now."
     )
 
@@ -102,6 +128,8 @@ async def generate_tasks(
     count: int = 5,
     config: GeneratorConfig | None = None,
     provider: LLMProvider | None = None,
+    server_context: str = "",
+    allow_writes: bool = False,
 ) -> tuple[list[TaskSpec], list[str]]:
     """Draft ``count`` tasks for ``server``. Returns (tasks, warnings);
     drafts that fail validation are dropped, never written."""
@@ -109,12 +137,12 @@ async def generate_tasks(
     provider_name, model_id = parse_model(config.model)
     provider = provider or get_provider(provider_name)
 
-    prompt = build_generator_prompt(inventory, count)
+    prompt = build_generator_prompt(inventory, count, server_context=server_context)
     drafts = None
     for _attempt in range(2):
         turn = await provider.complete(
             model=model_id,
-            system=GENERATOR_SYSTEM_PROMPT,
+            system=generator_system_prompt(allow_writes),
             messages=[ChatMessage(role="user", content=prompt)],
             tools=[],
             max_tokens=config.max_tokens,

whetkit-0.3.0/src/whetkit/llm/pricing.py

@@ -0,0 +1,27 @@
+"""Rough USD cost estimates for known models.
+
+Prices are per million tokens (input, output), hand-checked 2026-07-08
+against the providers' public pricing pages. They rot — treat every
+number this module produces as an estimate, which is how the CLI labels
+it. Unknown models simply get no estimate.
+"""
+
+PRICES_PER_MTOK: dict[str, tuple[float, float]] = {
+    "claude-sonnet-5": (3.00, 15.00),
+    "claude-haiku-4-5": (1.00, 5.00),
+    "gpt-4.1": (2.00, 8.00),
+    "gpt-4.1-mini": (0.40, 1.60),
+    "gpt-4.1-nano": (0.10, 0.40),
+    "gpt-5-mini": (0.25, 2.00),
+}
+
+
+def estimate_cost_usd(model_id: str, tokens_in: int, tokens_out: int) -> float | None:
+    """Best-effort estimate; None when the model isn't in the table.
+    Dated model ids (claude-haiku-4-5-20251001) match their base entry."""
+    # longest key first, so gpt-4.1-mini never matches the gpt-4.1 entry
+    for known in sorted(PRICES_PER_MTOK, key=len, reverse=True):
+        price_in, price_out = PRICES_PER_MTOK[known]
+        if model_id == known or model_id.startswith(f"{known}-"):
+            return (tokens_in * price_in + tokens_out * price_out) / 1_000_000
+    return None

{whetkit-0.1.1 → whetkit-0.3.0}/src/whetkit/mcp/transport.py

@@ -17,6 +17,7 @@ Three connection modes are supported, because real-world servers are mixed:
 
 import json
 import os
+import re
 import shutil
 import sys
 from collections.abc import AsyncIterator
@@ -105,8 +106,33 @@ def resolve_server_spec(value: str, http_mode: HttpMode = HttpMode.STATEFUL) ->
     raise ValueError(f"{value}: not a URL, directory, .json spec, or .py server")
 
 
+_ENV_REF_RE = re.compile(r"\$\{(\w+)\}")
+
+
+def _expand_env(value):
+    """Substitute ``${VAR}`` references from the environment in every string
+    of a spec document — so credentials (HTTP headers, child env) never have
+    to be written into server.json itself."""
+    if isinstance(value, str):
+
+        def sub(match: re.Match) -> str:
+            name = match.group(1)
+            if (resolved := os.environ.get(name)) is None:
+                raise ValueError(
+                    f"server spec references ${{{name}}} but it is not set in the environment"
+                )
+            return resolved
+
+        return _ENV_REF_RE.sub(sub, value)
+    if isinstance(value, dict):
+        return {k: _expand_env(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_expand_env(v) for v in value]
+    return value
+
+
 def _spec_from_json(path: Path) -> ServerSpec:
-    data = json.loads(path.read_text())
+    data = _expand_env(json.loads(path.read_text()))
     spec = spec_from_dict(data)
     if isinstance(spec, StdioSpec):
         spec.command = _python_command(spec.command)

{whetkit-0.1.1 → whetkit-0.3.0}/src/whetkit/scoring/__init__.py

@@ -1,11 +1,12 @@
 """Scoring: deterministic tool-selection matching + LLM-as-judge grading."""
 
-from whetkit.scoring.aggregate import EvalSummary, TaskScore, score_runs
+from whetkit.scoring.aggregate import EvalSummary, MultiRunSummary, TaskScore, score_runs
 from whetkit.scoring.deterministic import MatchMode, ToolMatchResult, score_tool_match
 from whetkit.scoring.judge import JudgeCache, JudgeConfig, JudgeVerdict, judge_run
 
 __all__ = [
     "EvalSummary",
+    "MultiRunSummary",
     "JudgeCache",
     "JudgeConfig",
     "JudgeVerdict",

{whetkit-0.1.1 → whetkit-0.3.0}/src/whetkit/scoring/aggregate.py

@@ -27,6 +27,14 @@ class TaskScore(BaseModel):
             return False
         return self.tool_hit
 
+    @property
+    def spec_gap(self) -> bool:
+        """Judge passed but the tool match failed: the agent reached a
+        correct outcome via tools the task never listed. Nine times out of
+        ten that means expected_tools is incomplete, not that the agent
+        failed."""
+        return self.judge is not None and self.judge.passed and not self.tool_hit
+
 
 class EvalSummary(BaseModel):
     scores: list[TaskScore]
@@ -88,6 +96,51 @@ class EvalSummary(BaseModel):
         return lines
 
 
+class MultiRunSummary(BaseModel):
+    """Aggregate over repeated executions of the same task set.
+
+    Single runs are noise: the same server, tasks, and model can flip a
+    task between runs. The honest headline is mean plus range."""
+
+    summaries: list[EvalSummary]
+
+    @property
+    def n(self) -> int:
+        return len(self.summaries)
+
+    def _spread(self, values: list[float]) -> str:
+        mean = sum(values) / len(values)
+        low, high = min(values), max(values)
+        if low == high:
+            return f"{mean:.0%}"
+        return f"{mean:.0%} [{low:.0%}–{high:.0%}]"
+
+    def summary_lines(self) -> list[str]:
+        lines = [
+            f"Runs: {self.n} × {self.summaries[0].task_count} task(s)",
+            f"Hit-rate: {self._spread([s.hit_rate for s in self.summaries])}",
+            f"Tool-selection hit-rate: {self._spread([s.tool_hit_rate for s in self.summaries])}",
+            f"Tool precision (avg): {self._spread([s.avg_precision for s in self.summaries])}",
+        ]
+        judged = [s.judge_pass_rate for s in self.summaries if s.judge_pass_rate is not None]
+        if judged:
+            lines.insert(2, f"Judge pass-rate: {self._spread(judged)}")
+        extras = [s.avg_extra_calls for s in self.summaries]
+        lines.append(f"Unnecessary calls (avg): {sum(extras) / len(extras):.1f}/task")
+        flaky = self.flaky_tasks()
+        if flaky:
+            lines.append(f"Flaky tasks (hit in some runs, missed in others): {', '.join(flaky)}")
+        return lines
+
+    def flaky_tasks(self) -> list[str]:
+        """Task ids whose hit outcome differs across runs."""
+        outcomes: dict[str, set[bool]] = {}
+        for summary in self.summaries:
+            for score in summary.scores:
+                outcomes.setdefault(score.task_id, set()).add(score.hit)
+        return sorted(task_id for task_id, seen in outcomes.items() if len(seen) > 1)
+
+
 async def score_runs(
     tasks: list[TaskSpec],
     runs: list[TaskRun],