whetkit 0.2.0__tar.gz → 0.3.0__tar.gz

{whetkit-0.2.0 → whetkit-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whetkit
-Version: 0.2.0
+Version: 0.3.0
 Summary: Measure and improve how well LLM agents select and use the tools exposed by an MCP server
 Keywords: mcp,model-context-protocol,llm,agents,evals,tool-use,tool-selection,benchmark
 Author: Mohammed Benlamlih

{whetkit-0.2.0 → whetkit-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "whetkit"
-version = "0.2.0"
+version = "0.3.0"
 description = "Measure and improve how well LLM agents select and use the tools exposed by an MCP server"
 readme = "README.md"
 license = "Apache-2.0"

{whetkit-0.2.0 → whetkit-0.3.0}/src/whetkit/cli.py

@@ -16,6 +16,14 @@ app = typer.Typer(
 )
 
 
+def _resolve_server(value: str, http_mode: HttpMode) -> ServerSpec:
+    """resolve_server_spec with a CLI-friendly error instead of a traceback."""
+    try:
+        return resolve_server_spec(value, http_mode=http_mode)
+    except ValueError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+
+
 def _judge_enabled(judge: str, judge_model: str) -> bool:
     """--judge auto: grade with the LLM judge only when its API key is set."""
     if judge in ("on", "off"):
@@ -31,10 +39,10 @@ def _resolve_task_servers(
     tasks: list, server_override: str | None, http_mode: HttpMode
 ) -> dict[str, ServerSpec]:
     if server_override is not None:
-        spec = resolve_server_spec(server_override, http_mode=http_mode)
+        spec = _resolve_server(server_override, http_mode)
         return {task.server: spec for task in tasks}
     return {
-        task.server: resolve_server_spec(task.server, http_mode=http_mode)
+        task.server: _resolve_server(task.server, http_mode)
         for task in {t.server: t for t in tasks}.values()
     }
 
@@ -70,6 +78,7 @@ def _summary_payload(group_name: str, summary, task_runs: list) -> dict:
                 "hit": score.hit,
                 "tool_hit": score.tool_hit,
                 "judge_passed": score.judge.passed if score.judge else None,
+                "spec_gap": score.spec_gap,
                 "called": score.tool_match.called,
                 "missing": score.tool_match.missing_slots,
                 "extra_calls": score.tool_match.extra_calls,
@@ -117,7 +126,7 @@ def inspect(
     ] = HttpMode.STATEFUL,
 ) -> None:
     """Connect to an MCP server and print its tool inventory."""
-    spec = resolve_server_spec(server, http_mode=http_mode)
+    spec = _resolve_server(server, http_mode)
     inventory = asyncio.run(inspect_server(spec))
 
     for line in inventory.summary_lines():
@@ -164,7 +173,7 @@ def doctor(
     if fail_on not in ("error", "warn", "never"):
         raise typer.BadParameter("--fail-on must be 'error', 'warn', or 'never'")
 
-    spec = resolve_server_spec(server, http_mode=http_mode)
+    spec = _resolve_server(server, http_mode)
     inventory = asyncio.run(inspect_server(spec))
     findings = diagnose(inventory)
 
@@ -209,16 +218,30 @@ def generate(
     model: Annotated[
         str, typer.Option("--model", help="Generator model as provider:model_id")
     ] = "anthropic:claude-sonnet-5",
+    allow_writes: Annotated[
+        bool,
+        typer.Option(
+            "--allow-writes",
+            help="Let drafts include non-destructive write tasks (default: read-only only)",
+        ),
+    ] = False,
     http_mode: Annotated[HttpMode, typer.Option("--http-mode")] = HttpMode.STATEFUL,
 ) -> None:
     """Draft eval tasks from the server's tool inventory (review before
     trusting — the header in the output file says the same)."""
     from whetkit.generate import GeneratorConfig, generate_tasks, write_tasks_yaml
 
-    spec = resolve_server_spec(server, http_mode=http_mode)
+    spec = _resolve_server(server, http_mode)
     inventory = asyncio.run(inspect_server(spec))
     tasks, warnings = asyncio.run(
-        generate_tasks(inventory, server, count=count, config=GeneratorConfig(model=model))
+        generate_tasks(
+            inventory,
+            server,
+            count=count,
+            config=GeneratorConfig(model=model),
+            server_context=spec.label(),
+            allow_writes=allow_writes,
+        )
     )
     for warning in warnings:
         typer.echo(f"warning: {warning}", err=True)
@@ -273,6 +296,60 @@ def export(
         typer.echo(rendered, nl=False)
 
 
+@app.command()
+def diff(
+    before: Annotated[str, typer.Argument(help="Baseline --summary-json file")],
+    after: Annotated[str, typer.Argument(help="Comparison --summary-json file")],
+) -> None:
+    """Compare two --summary-json files: headline deltas plus per-task
+    transitions. The before/after table without re-running anything."""
+    import json as jsonlib
+
+    docs = []
+    for path in (before, after):
+        if not Path(path).is_file():
+            raise typer.BadParameter(f"no summary file at {path}")
+        docs.append(jsonlib.loads(Path(path).read_text()))
+
+    def mean(doc: dict, key: str) -> float:
+        values = [r[key] for r in doc["runs"] if r.get(key) is not None]
+        return sum(values) / len(values) if values else 0.0
+
+    typer.echo(f"{'metric':<28} {'before':>10} {'after':>10}")
+    for label, key, fmt in (
+        ("Hit-rate", "hit_rate", "{:.0%}"),
+        ("Tool-selection hit-rate", "tool_hit_rate", "{:.0%}"),
+        ("Judge pass-rate", "judge_pass_rate", "{:.0%}"),
+        ("Precision (avg)", "avg_precision", "{:.0%}"),
+        ("Unnecessary calls/task", "avg_extra_calls", "{:.1f}"),
+        ("Tokens in (per run)", "tokens_in", "{:.0f}"),
+    ):
+        b, a = mean(docs[0], key), mean(docs[1], key)
+        typer.echo(f"{label:<28} {fmt.format(b):>10} {fmt.format(a):>10}")
+
+    def outcomes(doc: dict) -> dict[str, list[bool]]:
+        per_task: dict[str, list[bool]] = {}
+        for run_doc in doc["runs"]:
+            for task in run_doc["tasks"]:
+                per_task.setdefault(task["id"], []).append(bool(task["hit"]))
+        return per_task
+
+    before_tasks, after_tasks = outcomes(docs[0]), outcomes(docs[1])
+    typer.echo("")
+    for task_id in sorted(set(before_tasks) | set(after_tasks)):
+
+        def word(hits: list[bool] | None) -> str:
+            if not hits:
+                return "—"
+            if all(hits):
+                return "PASS"
+            return "MISS" if not any(hits) else f"FLAKY {sum(hits)}/{len(hits)}"
+
+        b_word, a_word = word(before_tasks.get(task_id)), word(after_tasks.get(task_id))
+        marker = "" if b_word == a_word else ("  ↑" if a_word == "PASS" else "  ↓")
+        typer.echo(f"  {task_id:<28} {b_word:>10} -> {a_word:<10}{marker}")
+
+
 @app.command()
 def run(
     tasks: Annotated[
@@ -347,6 +424,16 @@ def run(
             help="Write a machine-readable summary (metrics + per-task outcomes) to this path",
         ),
     ] = None,
+    reset_cmd: Annotated[
+        str | None,
+        typer.Option(
+            "--reset-cmd",
+            help=(
+                "Shell command run before each repetition (and the first run) to "
+                "reset server-side fixtures — makes --runs honest on stateful servers"
+            ),
+        ),
+    ] = None,
     http_mode: Annotated[HttpMode, typer.Option("--http-mode")] = HttpMode.STATEFUL,
 ) -> None:
     """Run eval tasks against an MCP server and print scored results."""
@@ -411,13 +498,27 @@ def run(
                 line += f"  judge={'pass' if score.judge.passed else 'FAIL'}"
             typer.echo(line)
         typer.echo("")
+        for score in summary.scores:
+            if score.spec_gap:
+                called = ", ".join(score.tool_match.called) or "(none)"
+                typer.echo(
+                    f"⚠ possible task-spec gap: '{score.task_id}' reached a correct "
+                    f"outcome (judge passed) via tools not listed in expected_tools "
+                    f"— called: {called}"
+                )
         for line in summary.summary_lines():
             typer.echo(line)
         tokens_in = sum(r.total_usage.input_tokens for r in task_runs)
         tokens_out = sum(r.total_usage.output_tokens for r in task_runs)
+        from whetkit.llm import parse_model
+        from whetkit.llm.pricing import estimate_cost_usd
+
+        cost = estimate_cost_usd(parse_model(model)[1], tokens_in, tokens_out)
+        cost_note = f"   ≈ ${cost:.4f} (est.)" if cost is not None else ""
         typer.echo(
             f"Tokens in/out: {tokens_in}/{tokens_out}   "
             f"Total latency: {sum(r.total_latency_ms for r in task_runs) / 1000:.1f}s"
+            f"{cost_note}"
         )
         return task_runs, summary
 
@@ -427,6 +528,17 @@ def run(
         cache = JudgeCache(default_store_path().parent / "judge_cache.sqlite3")
         try:
             for run_index in range(1, runs + 1):
+                if reset_cmd:
+                    import subprocess
+
+                    typer.echo(f"reset: {reset_cmd}", err=True)
+                    proc = subprocess.run(reset_cmd, shell=True)
+                    if proc.returncode != 0:
+                        typer.echo(
+                            f"error: --reset-cmd failed with exit code {proc.returncode}",
+                            err=True,
+                        )
+                        raise typer.Exit(code=1)
                 group_name = group if runs == 1 else f"{group}-{run_index}"
                 task_runs, summary = await _run_once(group_name, cache)
                 summaries.append(summary)
@@ -729,7 +841,7 @@ def overlay(
     the origin is never modified)."""
     from whetkit.curation import load_plan, serve_overlay
 
-    origin = resolve_server_spec(server, http_mode=http_mode)
+    origin = _resolve_server(server, http_mode)
     asyncio.run(serve_overlay(origin, load_plan(plan)))
 
 

{whetkit-0.2.0 → whetkit-0.3.0}/src/whetkit/generate.py

@@ -18,7 +18,7 @@ from whetkit.mcp.introspect import ServerInventory
 
 DEFAULT_GENERATOR_MODEL = "anthropic:claude-sonnet-5"
 
-GENERATOR_SYSTEM_PROMPT = """\
+GENERATOR_SYSTEM_PROMPT_TEMPLATE = """\
 You write eval tasks for an AI-agent benchmark. You are given the tools an
 MCP server exposes. Draft tasks that measure whether an agent picks the
 right tools for realistic user requests.
@@ -32,9 +32,7 @@ Rules for every task:
   alternatives for that step. Use ONLY tool names from the given list.
 - success_criteria is one or two concrete, checkable sentences a grader can
   verify from the agent's final answer alone.
-- Prefer read-only tasks. Include a write task only when the tool set is
-  clearly built for writes, and never anything destructive or irreversible
-  (no deletes, resets, or purges).
+{writes_rule}
 - Set "ordered": true only when the steps must happen in sequence.
 - ids are short kebab-case slugs, unique across the set.
 
@@ -59,9 +57,37 @@ def _inventory_block(inventory: ServerInventory) -> str:
     return "\n".join(lines)
 
 
-def build_generator_prompt(inventory: ServerInventory, count: int) -> str:
+READ_ONLY_RULE = (
+    "- Draft ONLY read-only tasks: nothing that creates, modifies, deletes,\n"
+    "  or sends anything. If the tool set is write-heavy, still restrict\n"
+    "  yourself to its read/query tools."
+)
+WRITES_ALLOWED_RULE = (
+    "- Prefer read-only tasks. Write tasks are allowed when the tool set is\n"
+    "  clearly built for writes, but never anything destructive or\n"
+    "  irreversible (no deletes, resets, or purges)."
+)
+
+
+def generator_system_prompt(allow_writes: bool) -> str:
+    rule = WRITES_ALLOWED_RULE if allow_writes else READ_ONLY_RULE
+    # plain replace, not str.format — the template contains JSON braces
+    return GENERATOR_SYSTEM_PROMPT_TEMPLATE.replace("{writes_rule}", rule)
+
+
+def build_generator_prompt(inventory: ServerInventory, count: int, server_context: str = "") -> str:
+    context_block = ""
+    if server_context:
+        context_block = (
+            "## Execution context\n"
+            f"These tasks will execute against: {server_context}\n"
+            "Write prompts whose arguments fit THAT context — the real paths,\n"
+            "repositories, or resources it implies. Never invent placeholder\n"
+            "examples like /home/user/project or acme-corp/website.\n\n"
+        )
     return (
         f"## Tools exposed by the server\n{_inventory_block(inventory)}\n\n"
+        f"{context_block}"
         f"Draft exactly {count} eval tasks now."
     )
 
@@ -102,6 +128,8 @@ async def generate_tasks(
     count: int = 5,
     config: GeneratorConfig | None = None,
     provider: LLMProvider | None = None,
+    server_context: str = "",
+    allow_writes: bool = False,
 ) -> tuple[list[TaskSpec], list[str]]:
     """Draft ``count`` tasks for ``server``. Returns (tasks, warnings);
     drafts that fail validation are dropped, never written."""
@@ -109,12 +137,12 @@ async def generate_tasks(
     provider_name, model_id = parse_model(config.model)
     provider = provider or get_provider(provider_name)
 
-    prompt = build_generator_prompt(inventory, count)
+    prompt = build_generator_prompt(inventory, count, server_context=server_context)
     drafts = None
     for _attempt in range(2):
         turn = await provider.complete(
             model=model_id,
-            system=GENERATOR_SYSTEM_PROMPT,
+            system=generator_system_prompt(allow_writes),
             messages=[ChatMessage(role="user", content=prompt)],
             tools=[],
             max_tokens=config.max_tokens,

whetkit-0.3.0/src/whetkit/llm/pricing.py

@@ -0,0 +1,27 @@
+"""Rough USD cost estimates for known models.
+
+Prices are per million tokens (input, output), hand-checked 2026-07-08
+against the providers' public pricing pages. They rot — treat every
+number this module produces as an estimate, which is how the CLI labels
+it. Unknown models simply get no estimate.
+"""
+
+PRICES_PER_MTOK: dict[str, tuple[float, float]] = {
+    "claude-sonnet-5": (3.00, 15.00),
+    "claude-haiku-4-5": (1.00, 5.00),
+    "gpt-4.1": (2.00, 8.00),
+    "gpt-4.1-mini": (0.40, 1.60),
+    "gpt-4.1-nano": (0.10, 0.40),
+    "gpt-5-mini": (0.25, 2.00),
+}
+
+
+def estimate_cost_usd(model_id: str, tokens_in: int, tokens_out: int) -> float | None:
+    """Best-effort estimate; None when the model isn't in the table.
+    Dated model ids (claude-haiku-4-5-20251001) match their base entry."""
+    # longest key first, so gpt-4.1-mini never matches the gpt-4.1 entry
+    for known in sorted(PRICES_PER_MTOK, key=len, reverse=True):
+        price_in, price_out = PRICES_PER_MTOK[known]
+        if model_id == known or model_id.startswith(f"{known}-"):
+            return (tokens_in * price_in + tokens_out * price_out) / 1_000_000
+    return None

{whetkit-0.2.0 → whetkit-0.3.0}/src/whetkit/mcp/transport.py

@@ -17,6 +17,7 @@ Three connection modes are supported, because real-world servers are mixed:
 
 import json
 import os
+import re
 import shutil
 import sys
 from collections.abc import AsyncIterator
@@ -105,8 +106,33 @@ def resolve_server_spec(value: str, http_mode: HttpMode = HttpMode.STATEFUL) ->
     raise ValueError(f"{value}: not a URL, directory, .json spec, or .py server")
 
 
+_ENV_REF_RE = re.compile(r"\$\{(\w+)\}")
+
+
+def _expand_env(value):
+    """Substitute ``${VAR}`` references from the environment in every string
+    of a spec document — so credentials (HTTP headers, child env) never have
+    to be written into server.json itself."""
+    if isinstance(value, str):
+
+        def sub(match: re.Match) -> str:
+            name = match.group(1)
+            if (resolved := os.environ.get(name)) is None:
+                raise ValueError(
+                    f"server spec references ${{{name}}} but it is not set in the environment"
+                )
+            return resolved
+
+        return _ENV_REF_RE.sub(sub, value)
+    if isinstance(value, dict):
+        return {k: _expand_env(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_expand_env(v) for v in value]
+    return value
+
+
 def _spec_from_json(path: Path) -> ServerSpec:
-    data = json.loads(path.read_text())
+    data = _expand_env(json.loads(path.read_text()))
     spec = spec_from_dict(data)
     if isinstance(spec, StdioSpec):
         spec.command = _python_command(spec.command)

{whetkit-0.2.0 → whetkit-0.3.0}/src/whetkit/scoring/aggregate.py

@@ -27,6 +27,14 @@ class TaskScore(BaseModel):
             return False
         return self.tool_hit
 
+    @property
+    def spec_gap(self) -> bool:
+        """Judge passed but the tool match failed: the agent reached a
+        correct outcome via tools the task never listed. Nine times out of
+        ten that means expected_tools is incomplete, not that the agent
+        failed."""
+        return self.judge is not None and self.judge.passed and not self.tool_hit
+
 
 class EvalSummary(BaseModel):
     scores: list[TaskScore]