whetkit 0.2.0__tar.gz → 0.4.0__tar.gz

{whetkit-0.2.0 → whetkit-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whetkit
-Version: 0.2.0
+Version: 0.4.0
 Summary: Measure and improve how well LLM agents select and use the tools exposed by an MCP server
 Keywords: mcp,model-context-protocol,llm,agents,evals,tool-use,tool-selection,benchmark
 Author: Mohammed Benlamlih

{whetkit-0.2.0 → whetkit-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "whetkit"
-version = "0.2.0"
+version = "0.4.0"
 description = "Measure and improve how well LLM agents select and use the tools exposed by an MCP server"
 readme = "README.md"
 license = "Apache-2.0"

{whetkit-0.2.0 → whetkit-0.4.0}/src/whetkit/cli.py

@@ -16,6 +16,14 @@ app = typer.Typer(
 )
 
 
+def _resolve_server(value: str, http_mode: HttpMode) -> ServerSpec:
+    """resolve_server_spec with a CLI-friendly error instead of a traceback."""
+    try:
+        return resolve_server_spec(value, http_mode=http_mode)
+    except ValueError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+
+
 def _judge_enabled(judge: str, judge_model: str) -> bool:
     """--judge auto: grade with the LLM judge only when its API key is set."""
     if judge in ("on", "off"):
@@ -31,10 +39,10 @@ def _resolve_task_servers(
     tasks: list, server_override: str | None, http_mode: HttpMode
 ) -> dict[str, ServerSpec]:
     if server_override is not None:
-        spec = resolve_server_spec(server_override, http_mode=http_mode)
+        spec = _resolve_server(server_override, http_mode)
         return {task.server: spec for task in tasks}
     return {
-        task.server: resolve_server_spec(task.server, http_mode=http_mode)
+        task.server: _resolve_server(task.server, http_mode)
         for task in {t.server: t for t in tasks}.values()
     }
 
@@ -70,6 +78,7 @@ def _summary_payload(group_name: str, summary, task_runs: list) -> dict:
                 "hit": score.hit,
                 "tool_hit": score.tool_hit,
                 "judge_passed": score.judge.passed if score.judge else None,
+                "spec_gap": score.spec_gap,
                 "called": score.tool_match.called,
                 "missing": score.tool_match.missing_slots,
                 "extra_calls": score.tool_match.extra_calls,
@@ -117,7 +126,7 @@ def inspect(
     ] = HttpMode.STATEFUL,
 ) -> None:
     """Connect to an MCP server and print its tool inventory."""
-    spec = resolve_server_spec(server, http_mode=http_mode)
+    spec = _resolve_server(server, http_mode)
     inventory = asyncio.run(inspect_server(spec))
 
     for line in inventory.summary_lines():
@@ -164,7 +173,7 @@ def doctor(
     if fail_on not in ("error", "warn", "never"):
         raise typer.BadParameter("--fail-on must be 'error', 'warn', or 'never'")
 
-    spec = resolve_server_spec(server, http_mode=http_mode)
+    spec = _resolve_server(server, http_mode)
     inventory = asyncio.run(inspect_server(spec))
     findings = diagnose(inventory)
 
@@ -209,16 +218,30 @@ def generate(
     model: Annotated[
         str, typer.Option("--model", help="Generator model as provider:model_id")
     ] = "anthropic:claude-sonnet-5",
+    allow_writes: Annotated[
+        bool,
+        typer.Option(
+            "--allow-writes",
+            help="Let drafts include non-destructive write tasks (default: read-only only)",
+        ),
+    ] = False,
     http_mode: Annotated[HttpMode, typer.Option("--http-mode")] = HttpMode.STATEFUL,
 ) -> None:
     """Draft eval tasks from the server's tool inventory (review before
     trusting — the header in the output file says the same)."""
     from whetkit.generate import GeneratorConfig, generate_tasks, write_tasks_yaml
 
-    spec = resolve_server_spec(server, http_mode=http_mode)
+    spec = _resolve_server(server, http_mode)
     inventory = asyncio.run(inspect_server(spec))
     tasks, warnings = asyncio.run(
-        generate_tasks(inventory, server, count=count, config=GeneratorConfig(model=model))
+        generate_tasks(
+            inventory,
+            server,
+            count=count,
+            config=GeneratorConfig(model=model),
+            server_context=spec.label(),
+            allow_writes=allow_writes,
+        )
     )
     for warning in warnings:
         typer.echo(f"warning: {warning}", err=True)
@@ -232,6 +255,62 @@ def generate(
     typer.echo(f"  whetkit run --server {server} --tasks {out}")
 
 
+@app.command(name="plan-init")
+def plan_init(
+    server: Annotated[
+        str,
+        typer.Option("--server", help="MCP server: URL, directory, server.json, or server.py"),
+    ],
+    keep: Annotated[
+        str, typer.Option("--keep", help="Comma-separated tool names to keep visible")
+    ] = "",
+    from_tasks: Annotated[
+        str | None,
+        typer.Option(
+            "--from-tasks",
+            help="Also keep every tool referenced by these tasks' expected_tools",
+        ),
+    ] = None,
+    out: Annotated[
+        str, typer.Option("--out", help="Where to write the plan YAML")
+    ] = ".whetkit/curation-plan.yaml",
+    http_mode: Annotated[HttpMode, typer.Option("--http-mode")] = HttpMode.STATEFUL,
+) -> None:
+    """Scaffold a view plan: keep the named tools, hide everything else.
+    The fastest way to serve a lean read-only slice of a big server."""
+    from whetkit.curation import CurationPlan, ToolOverride, save_plan
+    from whetkit.datasets import load_tasks
+
+    keep_set = {name.strip() for name in keep.split(",") if name.strip()}
+    if from_tasks:
+        for task in load_tasks(from_tasks):
+            for slot in task.expected_tool_slots:
+                keep_set.update(slot)
+    if not keep_set:
+        raise typer.BadParameter("nothing to keep — pass --keep and/or --from-tasks")
+
+    spec = _resolve_server(server, http_mode)
+    inventory = asyncio.run(inspect_server(spec))
+    names = {t.name for t in inventory.tools}
+    if unknown := sorted(keep_set - names):
+        typer.echo(f"warning: not on the server, ignoring: {', '.join(unknown)}", err=True)
+
+    hidden = [
+        ToolOverride(original_name=t.name, hidden=True, reason="Not part of this view's workflows.")
+        for t in inventory.tools
+        if t.name not in keep_set
+    ]
+    plan = CurationPlan(
+        server=spec.label(),
+        notes=f"View plan: keep {len(names & keep_set)} tool(s), hide the rest.",
+        overrides=hidden,
+    )
+    Path(out).parent.mkdir(parents=True, exist_ok=True)
+    save_plan(plan, out)
+    typer.echo(f"kept {len(names & keep_set)}, hidden {len(hidden)} — wrote {out}")
+    typer.echo(f"score it:  whetkit run --server {server} --tasks <tasks> --plan {out}")
+
+
 @app.command()
 def export(
     plan: Annotated[
@@ -273,6 +352,60 @@ def export(
         typer.echo(rendered, nl=False)
 
 
+@app.command()
+def diff(
+    before: Annotated[str, typer.Argument(help="Baseline --summary-json file")],
+    after: Annotated[str, typer.Argument(help="Comparison --summary-json file")],
+) -> None:
+    """Compare two --summary-json files: headline deltas plus per-task
+    transitions. The before/after table without re-running anything."""
+    import json as jsonlib
+
+    docs = []
+    for path in (before, after):
+        if not Path(path).is_file():
+            raise typer.BadParameter(f"no summary file at {path}")
+        docs.append(jsonlib.loads(Path(path).read_text()))
+
+    def mean(doc: dict, key: str) -> float:
+        values = [r[key] for r in doc["runs"] if r.get(key) is not None]
+        return sum(values) / len(values) if values else 0.0
+
+    typer.echo(f"{'metric':<28} {'before':>10} {'after':>10}")
+    for label, key, fmt in (
+        ("Hit-rate", "hit_rate", "{:.0%}"),
+        ("Tool-selection hit-rate", "tool_hit_rate", "{:.0%}"),
+        ("Judge pass-rate", "judge_pass_rate", "{:.0%}"),
+        ("Precision (avg)", "avg_precision", "{:.0%}"),
+        ("Unnecessary calls/task", "avg_extra_calls", "{:.1f}"),
+        ("Tokens in (per run)", "tokens_in", "{:.0f}"),
+    ):
+        b, a = mean(docs[0], key), mean(docs[1], key)
+        typer.echo(f"{label:<28} {fmt.format(b):>10} {fmt.format(a):>10}")
+
+    def outcomes(doc: dict) -> dict[str, list[bool]]:
+        per_task: dict[str, list[bool]] = {}
+        for run_doc in doc["runs"]:
+            for task in run_doc["tasks"]:
+                per_task.setdefault(task["id"], []).append(bool(task["hit"]))
+        return per_task
+
+    before_tasks, after_tasks = outcomes(docs[0]), outcomes(docs[1])
+    typer.echo("")
+    for task_id in sorted(set(before_tasks) | set(after_tasks)):
+
+        def word(hits: list[bool] | None) -> str:
+            if not hits:
+                return "—"
+            if all(hits):
+                return "PASS"
+            return "MISS" if not any(hits) else f"FLAKY {sum(hits)}/{len(hits)}"
+
+        b_word, a_word = word(before_tasks.get(task_id)), word(after_tasks.get(task_id))
+        marker = "" if b_word == a_word else ("  ↑" if a_word == "PASS" else "  ↓")
+        typer.echo(f"  {task_id:<28} {b_word:>10} -> {a_word:<10}{marker}")
+
+
 @app.command()
 def run(
     tasks: Annotated[
@@ -347,6 +480,16 @@ def run(
             help="Write a machine-readable summary (metrics + per-task outcomes) to this path",
         ),
     ] = None,
+    reset_cmd: Annotated[
+        str | None,
+        typer.Option(
+            "--reset-cmd",
+            help=(
+                "Shell command run before each repetition (and the first run) to "
+                "reset server-side fixtures — makes --runs honest on stateful servers"
+            ),
+        ),
+    ] = None,
     http_mode: Annotated[HttpMode, typer.Option("--http-mode")] = HttpMode.STATEFUL,
 ) -> None:
     """Run eval tasks against an MCP server and print scored results."""
@@ -411,13 +554,27 @@ def run(
                 line += f"  judge={'pass' if score.judge.passed else 'FAIL'}"
             typer.echo(line)
         typer.echo("")
+        for score in summary.scores:
+            if score.spec_gap:
+                called = ", ".join(score.tool_match.called) or "(none)"
+                typer.echo(
+                    f"⚠ possible task-spec gap: '{score.task_id}' reached a correct "
+                    f"outcome (judge passed) via tools not listed in expected_tools "
+                    f"— called: {called}"
+                )
         for line in summary.summary_lines():
             typer.echo(line)
         tokens_in = sum(r.total_usage.input_tokens for r in task_runs)
         tokens_out = sum(r.total_usage.output_tokens for r in task_runs)
+        from whetkit.llm import parse_model
+        from whetkit.llm.pricing import estimate_cost_usd
+
+        cost = estimate_cost_usd(parse_model(model)[1], tokens_in, tokens_out)
+        cost_note = f"   ≈ ${cost:.4f} (est.)" if cost is not None else ""
         typer.echo(
             f"Tokens in/out: {tokens_in}/{tokens_out}   "
             f"Total latency: {sum(r.total_latency_ms for r in task_runs) / 1000:.1f}s"
+            f"{cost_note}"
         )
         return task_runs, summary
 
@@ -427,6 +584,17 @@ def run(
         cache = JudgeCache(default_store_path().parent / "judge_cache.sqlite3")
         try:
             for run_index in range(1, runs + 1):
+                if reset_cmd:
+                    import subprocess
+
+                    typer.echo(f"reset: {reset_cmd}", err=True)
+                    proc = subprocess.run(reset_cmd, shell=True)
+                    if proc.returncode != 0:
+                        typer.echo(
+                            f"error: --reset-cmd failed with exit code {proc.returncode}",
+                            err=True,
+                        )
+                        raise typer.Exit(code=1)
                 group_name = group if runs == 1 else f"{group}-{run_index}"
                 task_runs, summary = await _run_once(group_name, cache)
                 summaries.append(summary)
@@ -505,6 +673,17 @@ def curate(
             help="Completion-token budget per model turn (raise for reasoning models)",
         ),
     ] = 1024,
+    prune_unused: Annotated[
+        bool,
+        typer.Option(
+            "--prune-unused",
+            help=(
+                "Additionally hide every tool the eval never touched — the cost "
+                "play for big servers. Only sound when the tasks cover all the "
+                "workflows the curated view will serve."
+            ),
+        ),
+    ] = False,
     store: Annotated[str | None, typer.Option("--store")] = None,
     http_mode: Annotated[HttpMode, typer.Option("--http-mode")] = HttpMode.STATEFUL,
 ) -> None:
@@ -512,6 +691,7 @@ def curate(
     it, and show the before/after hit-rate."""
     from whetkit.curation import CuratedMCPClient, propose_plan, save_plan
     from whetkit.curation.optimizer import OptimizerConfig
+    from whetkit.curation.optimizer import prune_unused as apply_prune_unused
     from whetkit.datasets import load_tasks
     from whetkit.mcp import inspect_server
     from whetkit.runner import RunConfig, run_task
@@ -554,6 +734,9 @@ def curate(
             )
             for warning in warnings:
                 typer.echo(f"warning: {warning}", err=True)
+            if prune_unused:
+                pruned = apply_prune_unused(plan, inventory, task_list, baseline_runs)
+                typer.echo(f"--prune-unused: hid {pruned} untouched tool(s)", err=True)
             save_plan(plan, plan_path)
             typer.echo(f"curation plan written to {plan_path}", err=True)
 
@@ -630,6 +813,159 @@ def curate(
     asyncio.run(_curate())
 
 
+@app.command()
+def fix(
+    tasks: Annotated[
+        str, typer.Option("--tasks", help="Task YAML file or directory of task files")
+    ],
+    server: Annotated[
+        str | None, typer.Option("--server", help="Override the server every task runs against")
+    ] = None,
+    model: Annotated[
+        str, typer.Option("--model", help="Agent model as provider:model_id")
+    ] = "anthropic:claude-sonnet-5",
+    optimizer_model: Annotated[
+        str, typer.Option("--optimizer-model")
+    ] = "anthropic:claude-sonnet-5",
+    judge: Annotated[str, typer.Option("--judge", help="'auto', 'on', or 'off'")] = "auto",
+    judge_model: Annotated[str, typer.Option("--judge-model")] = "anthropic:claude-sonnet-5",
+    max_iterations: Annotated[
+        int, typer.Option("--max-iterations", help="Propose→eval→revise rounds (≥1)")
+    ] = 3,
+    plan_path: Annotated[
+        str, typer.Option("--plan", help="Where to write the best plan YAML")
+    ] = ".whetkit/curation-plan.yaml",
+    match_mode: Annotated[str, typer.Option("--match-mode")] = "order_tolerant",
+    max_turns: Annotated[int, typer.Option("--max-turns")] = 10,
+    max_tokens: Annotated[int, typer.Option("--max-tokens")] = 1024,
+    store: Annotated[str | None, typer.Option("--store")] = None,
+    http_mode: Annotated[HttpMode, typer.Option("--http-mode")] = HttpMode.STATEFUL,
+) -> None:
+    """Self-correcting curation: propose a plan, eval through it, feed the
+    regressions and remaining waste back to the optimizer, revise — up to
+    --max-iterations — and keep the best plan by measured results."""
+    from functools import partial
+
+    from whetkit.curation import CuratedMCPClient, propose_plan, save_plan
+    from whetkit.curation.optimizer import OptimizerConfig, propose_revision
+    from whetkit.datasets import load_tasks
+    from whetkit.runner import RunConfig, run_task
+    from whetkit.scoring import JudgeCache, JudgeConfig, MatchMode, score_runs
+    from whetkit.tracing import TraceStore, default_store_path
+
+    if max_iterations < 1:
+        raise typer.BadParameter("--max-iterations must be at least 1")
+    task_list = load_tasks(tasks)
+    servers = _resolve_task_servers(task_list, server, http_mode)
+    config = RunConfig(model=model, max_turns=max_turns, max_tokens=max_tokens)
+    use_judge = _judge_enabled(judge, judge_model)
+    judge_config = JudgeConfig(model=judge_model)
+    mode = MatchMode(match_mode)
+    store_path = store or str(default_store_path())
+    optimizer_config = OptimizerConfig(model=optimizer_model)
+
+    from whetkit.mcp import MCPClient
+
+    async def _eval(group: str, cache, client_factory=MCPClient):
+        runs = []
+        for task in task_list:
+            typer.echo(f"running {task.id} ...", err=True)
+            runs.append(
+                await run_task(task, servers[task.server], config, client_factory=client_factory)
+            )
+        with TraceStore(store_path) as trace_store:
+            trace_store.save_runs(runs, run_group=group)
+        return runs
+
+    async def _fix() -> None:
+        from whetkit.mcp import inspect_server as _inspect
+
+        cache = JudgeCache(default_store_path().parent / "judge_cache.sqlite3")
+        try:
+            typer.echo("== baseline ==", err=True)
+            baseline_runs = await _eval("baseline", cache)
+            baseline = await score_runs(
+                task_list,
+                baseline_runs,
+                mode=mode,
+                judge_config=judge_config,
+                judge_cache=cache,
+                use_judge=use_judge,
+            )
+            inventory = await _inspect(next(iter(servers.values())))
+
+            typer.echo("== proposing plan (iteration 1) ==", err=True)
+            plan, warnings = await propose_plan(
+                inventory, task_list, baseline_runs, baseline.scores, optimizer_config
+            )
+            for w in warnings:
+                typer.echo(f"warning: {w}", err=True)
+
+            def metric(summary):
+                return (summary.hit_rate, -summary.avg_extra_calls, summary.avg_precision)
+
+            best_plan, best_summary = None, None
+            for iteration in range(1, max_iterations + 1):
+                typer.echo(f"== eval through plan (iteration {iteration}) ==", err=True)
+                curated_runs = await _eval(
+                    f"fix-{iteration}",
+                    cache,
+                    client_factory=partial(CuratedMCPClient, plan=plan),
+                )
+                curated = await score_runs(
+                    task_list,
+                    curated_runs,
+                    mode=mode,
+                    judge_config=judge_config,
+                    judge_cache=cache,
+                    use_judge=use_judge,
+                    name_map=plan.rename_map(),
+                )
+                typer.echo(
+                    f"iteration {iteration}: hit {curated.hit_rate:.0%} "
+                    f"(baseline {baseline.hit_rate:.0%}), "
+                    f"extra calls {curated.avg_extra_calls:.1f}/task",
+                    err=True,
+                )
+                if best_summary is None or metric(curated) > metric(best_summary):
+                    best_plan, best_summary = plan, curated
+
+                converged = (
+                    curated.hit_rate >= baseline.hit_rate and curated.avg_extra_calls <= 0.25
+                )
+                if converged or iteration == max_iterations:
+                    if converged:
+                        typer.echo("converged — no regressions, negligible waste", err=True)
+                    break
+
+                typer.echo("== revising plan ==", err=True)
+                plan, warnings = await propose_revision(
+                    plan,
+                    inventory,
+                    task_list,
+                    baseline_runs,
+                    baseline.scores,
+                    curated_runs,
+                    curated.scores,
+                    optimizer_config,
+                )
+                for w in warnings:
+                    typer.echo(f"warning: {w}", err=True)
+        finally:
+            cache.close()
+
+        save_plan(best_plan, plan_path)
+        typer.echo(f"\nbest plan (of {max_iterations} max iterations) -> {plan_path}")
+        typer.echo(
+            f"Hit-rate: {baseline.hit_rate:.0%} -> {best_summary.hit_rate:.0%}   "
+            f"Extra calls: {baseline.avg_extra_calls:.1f} -> "
+            f"{best_summary.avg_extra_calls:.1f}/task"
+        )
+        typer.echo(f"serve it:  whetkit overlay --server <origin> --plan {plan_path}")
+
+    asyncio.run(_fix())
+
+
 @app.command()
 def report(
     tasks: Annotated[
@@ -729,7 +1065,7 @@ def overlay(
     the origin is never modified)."""
     from whetkit.curation import load_plan, serve_overlay
 
-    origin = resolve_server_spec(server, http_mode=http_mode)
+    origin = _resolve_server(server, http_mode)
     asyncio.run(serve_overlay(origin, load_plan(plan)))
 
 

{whetkit-0.2.0 → whetkit-0.4.0}/src/whetkit/curation/optimizer.py

@@ -197,3 +197,132 @@ async def propose_plan(
             plan.overrides = []
             break
     return plan, warnings
+
+
+def prune_unused(
+    plan: CurationPlan,
+    inventory: ServerInventory,
+    tasks: list[TaskSpec],
+    runs: list[TaskRun],
+) -> int:
+    """Hide every tool the eval never touched: not called in any baseline
+    run, not accepted by any task's expected_tools, and not already
+    overridden. The cost play for big servers — only sound when the task
+    set really covers the workflows the curated view will serve, which is
+    why it's opt-in. Returns how many tools were hidden."""
+    used = {name for run in runs for name in run.called_tool_names}
+    for task in tasks:
+        for slot in task.expected_tool_slots:
+            used.update(slot)
+    overridden = {o.original_name for o in plan.overrides}
+    added = 0
+    for tool in inventory.tools:
+        if tool.name in used or tool.name in overridden:
+            continue
+        plan.overrides.append(
+            ToolOverride(
+                original_name=tool.name,
+                hidden=True,
+                reason="Unused by the eval workflows (pruned by --prune-unused).",
+            )
+        )
+        added += 1
+    return added
+
+
+REVISION_INSTRUCTIONS = """\
+You previously proposed the curation plan below, and it was re-evaluated.
+Revise it: KEEP the changes that worked, FIX or DROP the ones implicated in
+regressions or remaining waste. A rename the agent still misused needs a
+better name or a sharper description; a prune that broke a task must be
+un-hidden. Respond with the SAME JSON shape as before — the FULL revised
+plan, not a delta.
+"""
+
+
+def build_revision_prompt(
+    plan: CurationPlan,
+    inventory: ServerInventory,
+    tasks: list[TaskSpec],
+    baseline_runs: list[TaskRun],
+    baseline_scores: list[TaskScore],
+    curated_runs: list[TaskRun],
+    curated_scores: list[TaskScore],
+) -> str:
+    import yaml
+
+    baseline_by_id = {s.task_id: s for s in baseline_scores}
+    regressed = [
+        s.task_id
+        for s in curated_scores
+        if not s.hit and (b := baseline_by_id.get(s.task_id)) is not None and b.hit
+    ]
+    regression_line = (
+        f"REGRESSIONS your plan caused (fix these first): {', '.join(regressed)}"
+        if regressed
+        else "No regressions — focus on remaining waste (unnecessary calls)."
+    )
+    plan_yaml = yaml.safe_dump(plan.model_dump(exclude_defaults=True), sort_keys=False)
+    return (
+        f"## Tools exposed by the server\n{_inventory_block(inventory)}\n\n"
+        f"## Baseline eval (no plan)\n{_trace_block(tasks, baseline_runs, baseline_scores)}\n\n"
+        f"## Your previous plan\n```yaml\n{plan_yaml}```\n\n"
+        f"## Eval THROUGH your plan\n{_trace_block(tasks, curated_runs, curated_scores)}\n\n"
+        f"{regression_line}\n\nRevise the plan now."
+    )
+
+
+async def propose_revision(
+    plan: CurationPlan,
+    inventory: ServerInventory,
+    tasks: list[TaskSpec],
+    baseline_runs: list[TaskRun],
+    baseline_scores: list[TaskScore],
+    curated_runs: list[TaskRun],
+    curated_scores: list[TaskScore],
+    config: OptimizerConfig | None = None,
+    provider: LLMProvider | None = None,
+) -> tuple[CurationPlan, list[str]]:
+    """One revision round: same parsing/validation as propose_plan, but the
+    model sees its previous plan and what happened through it."""
+    config = config or OptimizerConfig()
+    provider_name, model_id = parse_model(config.model)
+    provider = provider or get_provider(provider_name)
+
+    prompt = build_revision_prompt(
+        plan, inventory, tasks, baseline_runs, baseline_scores, curated_runs, curated_scores
+    )
+    parsed = None
+    for _attempt in range(2):
+        turn = await provider.complete(
+            model=model_id,
+            system=OPTIMIZER_SYSTEM_PROMPT + "\n" + REVISION_INSTRUCTIONS,
+            messages=[ChatMessage(role="user", content=prompt)],
+            tools=[],
+            max_tokens=config.max_tokens,
+        )
+        parsed = _parse_overrides(turn.text or "")
+        if parsed is not None:
+            break
+    if parsed is None:
+        return plan, ["revision output was not valid JSON after 2 attempts; keeping previous plan"]
+
+    notes, overrides = parsed
+    revised = CurationPlan(server=inventory.server, notes=notes, overrides=overrides)
+    origin_names = {t.name for t in inventory.tools}
+    warnings: list[str] = []
+    while problems := revised.validate_against(origin_names):
+        problem = problems[0]
+        name_match = re.search(r"'([^']+)'", problem)
+        offender = name_match.group(1) if name_match else None
+        before = len(revised.overrides)
+        revised.overrides = [
+            o
+            for o in revised.overrides
+            if offender not in (o.original_name, o.new_name, o.presented_name)
+        ]
+        warnings.append(f"dropped unsafe override(s): {problem}")
+        if len(revised.overrides) == before:
+            warnings.append("could not repair revised plan; keeping previous plan")
+            return plan, warnings
+    return revised, warnings

{whetkit-0.2.0 → whetkit-0.4.0}/src/whetkit/generate.py

@@ -18,7 +18,7 @@ from whetkit.mcp.introspect import ServerInventory
 
 DEFAULT_GENERATOR_MODEL = "anthropic:claude-sonnet-5"
 
-GENERATOR_SYSTEM_PROMPT = """\
+GENERATOR_SYSTEM_PROMPT_TEMPLATE = """\
 You write eval tasks for an AI-agent benchmark. You are given the tools an
 MCP server exposes. Draft tasks that measure whether an agent picks the
 right tools for realistic user requests.
@@ -32,9 +32,7 @@ Rules for every task:
   alternatives for that step. Use ONLY tool names from the given list.
 - success_criteria is one or two concrete, checkable sentences a grader can
   verify from the agent's final answer alone.
-- Prefer read-only tasks. Include a write task only when the tool set is
-  clearly built for writes, and never anything destructive or irreversible
-  (no deletes, resets, or purges).
+{writes_rule}
 - Set "ordered": true only when the steps must happen in sequence.
 - ids are short kebab-case slugs, unique across the set.
 
@@ -59,9 +57,37 @@ def _inventory_block(inventory: ServerInventory) -> str:
     return "\n".join(lines)
 
 
-def build_generator_prompt(inventory: ServerInventory, count: int) -> str:
+READ_ONLY_RULE = (
+    "- Draft ONLY read-only tasks: nothing that creates, modifies, deletes,\n"
+    "  or sends anything. If the tool set is write-heavy, still restrict\n"
+    "  yourself to its read/query tools."
+)
+WRITES_ALLOWED_RULE = (
+    "- Prefer read-only tasks. Write tasks are allowed when the tool set is\n"
+    "  clearly built for writes, but never anything destructive or\n"
+    "  irreversible (no deletes, resets, or purges)."
+)
+
+
+def generator_system_prompt(allow_writes: bool) -> str:
+    rule = WRITES_ALLOWED_RULE if allow_writes else READ_ONLY_RULE
+    # plain replace, not str.format — the template contains JSON braces
+    return GENERATOR_SYSTEM_PROMPT_TEMPLATE.replace("{writes_rule}", rule)
+
+
+def build_generator_prompt(inventory: ServerInventory, count: int, server_context: str = "") -> str:
+    context_block = ""
+    if server_context:
+        context_block = (
+            "## Execution context\n"
+            f"These tasks will execute against: {server_context}\n"
+            "Write prompts whose arguments fit THAT context — the real paths,\n"
+            "repositories, or resources it implies. Never invent placeholder\n"
+            "examples like /home/user/project or acme-corp/website.\n\n"
+        )
     return (
         f"## Tools exposed by the server\n{_inventory_block(inventory)}\n\n"
+        f"{context_block}"
         f"Draft exactly {count} eval tasks now."
     )
 
@@ -102,6 +128,8 @@ async def generate_tasks(
     count: int = 5,
     config: GeneratorConfig | None = None,
     provider: LLMProvider | None = None,
+    server_context: str = "",
+    allow_writes: bool = False,
 ) -> tuple[list[TaskSpec], list[str]]:
     """Draft ``count`` tasks for ``server``. Returns (tasks, warnings);
     drafts that fail validation are dropped, never written."""
@@ -109,12 +137,12 @@ async def generate_tasks(
     provider_name, model_id = parse_model(config.model)
     provider = provider or get_provider(provider_name)
 
-    prompt = build_generator_prompt(inventory, count)
+    prompt = build_generator_prompt(inventory, count, server_context=server_context)
     drafts = None
     for _attempt in range(2):
         turn = await provider.complete(
             model=model_id,
-            system=GENERATOR_SYSTEM_PROMPT,
+            system=generator_system_prompt(allow_writes),
             messages=[ChatMessage(role="user", content=prompt)],
             tools=[],
             max_tokens=config.max_tokens,

whetkit-0.4.0/src/whetkit/llm/pricing.py

@@ -0,0 +1,27 @@
+"""Rough USD cost estimates for known models.
+
+Prices are per million tokens (input, output), hand-checked 2026-07-08
+against the providers' public pricing pages. They rot — treat every
+number this module produces as an estimate, which is how the CLI labels
+it. Unknown models simply get no estimate.
+"""
+
+PRICES_PER_MTOK: dict[str, tuple[float, float]] = {
+    "claude-sonnet-5": (3.00, 15.00),
+    "claude-haiku-4-5": (1.00, 5.00),
+    "gpt-4.1": (2.00, 8.00),
+    "gpt-4.1-mini": (0.40, 1.60),
+    "gpt-4.1-nano": (0.10, 0.40),
+    "gpt-5-mini": (0.25, 2.00),
+}
+
+
+def estimate_cost_usd(model_id: str, tokens_in: int, tokens_out: int) -> float | None:
+    """Best-effort estimate; None when the model isn't in the table.
+    Dated model ids (claude-haiku-4-5-20251001) match their base entry."""
+    # longest key first, so gpt-4.1-mini never matches the gpt-4.1 entry
+    for known in sorted(PRICES_PER_MTOK, key=len, reverse=True):
+        price_in, price_out = PRICES_PER_MTOK[known]
+        if model_id == known or model_id.startswith(f"{known}-"):
+            return (tokens_in * price_in + tokens_out * price_out) / 1_000_000
+    return None

{whetkit-0.2.0 → whetkit-0.4.0}/src/whetkit/mcp/transport.py

@@ -17,6 +17,7 @@ Three connection modes are supported, because real-world servers are mixed:
 
 import json
 import os
+import re
 import shutil
 import sys
 from collections.abc import AsyncIterator
@@ -105,8 +106,33 @@ def resolve_server_spec(value: str, http_mode: HttpMode = HttpMode.STATEFUL) ->
     raise ValueError(f"{value}: not a URL, directory, .json spec, or .py server")
 
 
+_ENV_REF_RE = re.compile(r"\$\{(\w+)\}")
+
+
+def _expand_env(value):
+    """Substitute ``${VAR}`` references from the environment in every string
+    of a spec document — so credentials (HTTP headers, child env) never have
+    to be written into server.json itself."""
+    if isinstance(value, str):
+
+        def sub(match: re.Match) -> str:
+            name = match.group(1)
+            if (resolved := os.environ.get(name)) is None:
+                raise ValueError(
+                    f"server spec references ${{{name}}} but it is not set in the environment"
+                )
+            return resolved
+
+        return _ENV_REF_RE.sub(sub, value)
+    if isinstance(value, dict):
+        return {k: _expand_env(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_expand_env(v) for v in value]
+    return value
+
+
 def _spec_from_json(path: Path) -> ServerSpec:
-    data = json.loads(path.read_text())
+    data = _expand_env(json.loads(path.read_text()))
     spec = spec_from_dict(data)
     if isinstance(spec, StdioSpec):
         spec.command = _python_command(spec.command)

{whetkit-0.2.0 → whetkit-0.4.0}/src/whetkit/scoring/aggregate.py

@@ -27,6 +27,14 @@ class TaskScore(BaseModel):
             return False
         return self.tool_hit
 
+    @property
+    def spec_gap(self) -> bool:
+        """Judge passed but the tool match failed: the agent reached a
+        correct outcome via tools the task never listed. Nine times out of
+        ten that means expected_tools is incomplete, not that the agent
+        failed."""
+        return self.judge is not None and self.judge.passed and not self.tool_hit
+
 
 class EvalSummary(BaseModel):
     scores: list[TaskScore]