whetkit 0.1.1__tar.gz → 0.2.0__tar.gz

{whetkit-0.1.1 → whetkit-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whetkit
-Version: 0.1.1
+Version: 0.2.0
 Summary: Measure and improve how well LLM agents select and use the tools exposed by an MCP server
 Keywords: mcp,model-context-protocol,llm,agents,evals,tool-use,tool-selection,benchmark
 Author: Mohammed Benlamlih

{whetkit-0.1.1 → whetkit-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "whetkit"
-version = "0.1.1"
+version = "0.2.0"
 description = "Measure and improve how well LLM agents select and use the tools exposed by an MCP server"
 readme = "README.md"
 license = "Apache-2.0"

{whetkit-0.1.1 → whetkit-0.2.0}/src/whetkit/cli.py

@@ -51,6 +51,34 @@ def _write_report(report, out_dir: str) -> tuple[str, str]:
     return str(html_path), str(json_path)
 
 
+def _summary_payload(group_name: str, summary, task_runs: list) -> dict:
+    """One run's metrics as plain data — what --summary-json emits per run."""
+    return {
+        "group": group_name,
+        "hit_rate": summary.hit_rate,
+        "tool_hit_rate": summary.tool_hit_rate,
+        "judge_pass_rate": summary.judge_pass_rate,
+        "avg_precision": summary.avg_precision,
+        "avg_recall": summary.avg_recall,
+        "avg_extra_calls": summary.avg_extra_calls,
+        "tokens_in": sum(r.total_usage.input_tokens for r in task_runs),
+        "tokens_out": sum(r.total_usage.output_tokens for r in task_runs),
+        "latency_ms": sum(r.total_latency_ms for r in task_runs),
+        "tasks": [
+            {
+                "id": score.task_id,
+                "hit": score.hit,
+                "tool_hit": score.tool_hit,
+                "judge_passed": score.judge.passed if score.judge else None,
+                "called": score.tool_match.called,
+                "missing": score.tool_match.missing_slots,
+                "extra_calls": score.tool_match.extra_calls,
+            }
+            for score in summary.scores
+        ],
+    }
+
+
 def _version_callback(value: bool) -> None:
     if value:
         from importlib.metadata import version
@@ -204,6 +232,47 @@ def generate(
     typer.echo(f"  whetkit run --server {server} --tasks {out}")
 
 
+@app.command()
+def export(
+    plan: Annotated[
+        str, typer.Option("--plan", help="Curation plan YAML to export")
+    ] = ".whetkit/curation-plan.yaml",
+    to: Annotated[
+        str,
+        typer.Option(
+            "--to",
+            help=(
+                "'markdown' — a fix report ready for an upstream issue/PR; "
+                "'json' — a neutral override list for gateways and scripts"
+            ),
+        ),
+    ] = "markdown",
+    out: Annotated[str | None, typer.Option("--out", help="Write here instead of stdout")] = None,
+) -> None:
+    """Export a curation plan as a shareable fix report or neutral JSON."""
+    from whetkit.curation import load_plan
+    from whetkit.curation.export import plan_to_json, plan_to_markdown
+
+    if to not in ("markdown", "json"):
+        raise typer.BadParameter("--to must be 'markdown' or 'json'")
+    if not Path(plan).is_file():
+        raise typer.BadParameter(
+            f"no curation plan at {plan} — run 'whetkit curate' first, or pass --plan"
+        )
+    curation_plan = load_plan(plan)
+    if not curation_plan.overrides:
+        typer.echo("plan has no overrides — nothing to export", err=True)
+        raise typer.Exit(code=1)
+
+    rendered = (plan_to_markdown if to == "markdown" else plan_to_json)(curation_plan)
+    if out:
+        Path(out).parent.mkdir(parents=True, exist_ok=True)
+        Path(out).write_text(rendered)
+        typer.echo(f"wrote {out}")
+    else:
+        typer.echo(rendered, nl=False)
+
+
 @app.command()
 def run(
     tasks: Annotated[
@@ -250,21 +319,45 @@ def run(
             help="Completion-token budget per model turn (raise for reasoning models)",
         ),
     ] = 1024,
+    runs: Annotated[
+        int,
+        typer.Option(
+            "--runs",
+            help=(
+                "Repeat the whole task set N times and report mean plus range "
+                "— single runs are noise. Trace groups get a -1..-N suffix."
+            ),
+        ),
+    ] = 1,
     store: Annotated[
         str | None,
         typer.Option("--store", help="Trace SQLite path (default ./.whetkit/traces.sqlite3)"),
     ] = None,
     jsonl: Annotated[
-        str | None, typer.Option("--jsonl", help="Also write traces to this JSONL file")
+        str | None,
+        typer.Option(
+            "--jsonl",
+            help="Also write traces to this JSONL file (suffixed per run when --runs > 1)",
+        ),
+    ] = None,
+    summary_json: Annotated[
+        str | None,
+        typer.Option(
+            "--summary-json",
+            help="Write a machine-readable summary (metrics + per-task outcomes) to this path",
+        ),
     ] = None,
     http_mode: Annotated[HttpMode, typer.Option("--http-mode")] = HttpMode.STATEFUL,
 ) -> None:
     """Run eval tasks against an MCP server and print scored results."""
     from whetkit.datasets import load_tasks
     from whetkit.runner import RunConfig, run_task
-    from whetkit.scoring import JudgeCache, JudgeConfig, MatchMode, score_runs
+    from whetkit.scoring import JudgeCache, JudgeConfig, MatchMode, MultiRunSummary, score_runs
     from whetkit.tracing import TraceStore, default_store_path, write_jsonl
 
+    if runs < 1:
+        raise typer.BadParameter("--runs must be at least 1")
+
     task_list = load_tasks(tasks)
     servers = _resolve_task_servers(task_list, server, http_mode)
     config = RunConfig(model=model, max_turns=max_turns, max_tokens=max_tokens)
@@ -286,34 +379,28 @@ def run(
         client_factory = partial(CuratedMCPClient, plan=curation_plan)
         name_map = curation_plan.rename_map()
 
-    async def _run() -> None:
-        runs = []
+    async def _run_once(group_name: str, cache: JudgeCache):
+        task_runs = []
         for task in task_list:
             typer.echo(f"running {task.id} ...", err=True)
-            runs.append(
+            task_runs.append(
                 await run_task(task, servers[task.server], config, client_factory=client_factory)
             )
 
         with TraceStore(store_path) as trace_store:
-            trace_store.save_runs(runs, run_group=group)
-        if jsonl:
-            write_jsonl(runs, jsonl)
+            trace_store.save_runs(task_runs, run_group=group_name)
 
-        cache = JudgeCache(default_store_path().parent / "judge_cache.sqlite3")
-        try:
-            summary = await score_runs(
-                task_list,
-                runs,
-                mode=MatchMode(match_mode),
-                judge_config=JudgeConfig(model=judge_model),
-                judge_cache=cache,
-                use_judge=use_judge,
-                name_map=name_map,
-            )
-        finally:
-            cache.close()
+        summary = await score_runs(
+            task_list,
+            task_runs,
+            mode=MatchMode(match_mode),
+            judge_config=JudgeConfig(model=judge_model),
+            judge_cache=cache,
+            use_judge=use_judge,
+            name_map=name_map,
+        )
 
-        typer.echo(f"\nResults (group '{group}', model {model}):")
+        typer.echo(f"\nResults (group '{group_name}', model {model}):")
         for score in summary.scores:
             mark = "PASS" if score.hit else "MISS"
             tools = " -> ".join(score.tool_match.called) or "(no tool calls)"
@@ -326,17 +413,62 @@ def run(
         typer.echo("")
         for line in summary.summary_lines():
             typer.echo(line)
-        tokens_in = sum(r.total_usage.input_tokens for r in runs)
-        tokens_out = sum(r.total_usage.output_tokens for r in runs)
+        tokens_in = sum(r.total_usage.input_tokens for r in task_runs)
+        tokens_out = sum(r.total_usage.output_tokens for r in task_runs)
         typer.echo(
             f"Tokens in/out: {tokens_in}/{tokens_out}   "
-            f"Total latency: {sum(r.total_latency_ms for r in runs) / 1000:.1f}s"
+            f"Total latency: {sum(r.total_latency_ms for r in task_runs) / 1000:.1f}s"
         )
+        return task_runs, summary
+
+    async def _run() -> None:
+        summaries = []
+        payloads = []
+        cache = JudgeCache(default_store_path().parent / "judge_cache.sqlite3")
+        try:
+            for run_index in range(1, runs + 1):
+                group_name = group if runs == 1 else f"{group}-{run_index}"
+                task_runs, summary = await _run_once(group_name, cache)
+                summaries.append(summary)
+                payloads.append(_summary_payload(group_name, summary, task_runs))
+                if jsonl:
+                    write_jsonl(task_runs, jsonl if runs == 1 else f"{jsonl}.{run_index}")
+        finally:
+            cache.close()
+
+        multi = MultiRunSummary(summaries=summaries)
+        if runs > 1:
+            typer.echo(f"\n== across {runs} runs ==")
+            for line in multi.summary_lines():
+                typer.echo(line)
         if judge == "auto" and not use_judge:
             typer.echo(
                 "(LLM judge skipped: no API key found — set ANTHROPIC_API_KEY or pass --judge on)"
             )
-        typer.echo(f"Traces saved to {store_path} (group '{group}')")
+        suffix = f" (groups '{group}-1'..'-{runs}')" if runs > 1 else f" (group '{group}')"
+        typer.echo(f"Traces saved to {store_path}{suffix}")
+
+        if summary_json:
+            import json as jsonlib
+
+            document = {
+                "model": model,
+                "judge_model": judge_model if use_judge else None,
+                "match_mode": match_mode,
+                "plan": plan,
+                "runs": payloads,
+            }
+            if runs > 1:
+                document["aggregate"] = {
+                    "n": multi.n,
+                    "hit_rate_mean": sum(s.hit_rate for s in summaries) / len(summaries),
+                    "hit_rate_min": min(s.hit_rate for s in summaries),
+                    "hit_rate_max": max(s.hit_rate for s in summaries),
+                    "flaky_tasks": multi.flaky_tasks(),
+                }
+            Path(summary_json).parent.mkdir(parents=True, exist_ok=True)
+            Path(summary_json).write_text(jsonlib.dumps(document, indent=2) + "\n")
+            typer.echo(f"Summary JSON: {summary_json}")
 
     asyncio.run(_run())
 

whetkit-0.2.0/src/whetkit/curation/export.py

@@ -0,0 +1,73 @@
+"""Export a curation plan into shareable formats.
+
+A plan is most useful when it leaves the machine it was made on: as a
+Markdown fix report you can paste into an upstream issue or PR, or as
+neutral JSON for gateways and scripts that apply metadata overrides
+themselves.
+"""
+
+import json
+
+from whetkit.curation.plan import CurationPlan, ToolOverride
+
+
+def _action(override: ToolOverride) -> str:
+    if override.hidden:
+        return "hide"
+    if override.new_name and override.new_description:
+        return "rename + rewrite"
+    if override.new_name:
+        return "rename"
+    return "rewrite"
+
+
+def plan_to_markdown(plan: CurationPlan) -> str:
+    """A fix report for humans — upstream-issue / PR-description ready."""
+    lines = [
+        "## Proposed tool-surface fixes",
+        "",
+        "Metadata-only changes — no schemas or behavior are touched. "
+        "Measured with [whetkit](https://github.com/benlamlih/whetkit); "
+        "each change is reversible (it can be served as an overlay without "
+        "modifying the server).",
+        "",
+    ]
+    if plan.notes:
+        lines += [f"> {plan.notes.strip()}", ""]
+    lines += [
+        "| Tool | Change | Proposal | Why |",
+        "|---|---|---|---|",
+    ]
+    for override in plan.overrides:
+        if override.hidden:
+            proposal = "hide from the tool list"
+        else:
+            parts = []
+            if override.new_name:
+                parts.append(f"rename to `{override.new_name}`")
+            if override.new_description:
+                parts.append(f'description: "{override.new_description.strip()}"')
+            proposal = "; ".join(parts)
+        cell = lambda s: " ".join(str(s).split()).replace("|", "\\|")  # noqa: E731
+        lines.append(
+            f"| `{override.original_name}` | {_action(override)} "
+            f"| {cell(proposal)} | {cell(override.reason) or '—'} |"
+        )
+    return "\n".join(lines) + "\n"
+
+
+def plan_to_json(plan: CurationPlan) -> str:
+    """Neutral override list for gateways/scripts that apply their own
+    metadata overrides (Cloudflare portal aliases, MetaMCP namespaces, …)."""
+    overrides = [
+        {
+            "original_name": o.original_name,
+            "action": _action(o),
+            "new_name": o.new_name,
+            "new_description": o.new_description,
+            "hidden": o.hidden,
+            "reason": o.reason,
+        }
+        for o in plan.overrides
+    ]
+    return json.dumps({"notes": plan.notes, "overrides": overrides}, indent=2) + "\n"

{whetkit-0.1.1 → whetkit-0.2.0}/src/whetkit/scoring/__init__.py

@@ -1,11 +1,12 @@
 """Scoring: deterministic tool-selection matching + LLM-as-judge grading."""
 
-from whetkit.scoring.aggregate import EvalSummary, TaskScore, score_runs
+from whetkit.scoring.aggregate import EvalSummary, MultiRunSummary, TaskScore, score_runs
 from whetkit.scoring.deterministic import MatchMode, ToolMatchResult, score_tool_match
 from whetkit.scoring.judge import JudgeCache, JudgeConfig, JudgeVerdict, judge_run
 
 __all__ = [
     "EvalSummary",
+    "MultiRunSummary",
     "JudgeCache",
     "JudgeConfig",
     "JudgeVerdict",

{whetkit-0.1.1 → whetkit-0.2.0}/src/whetkit/scoring/aggregate.py

@@ -88,6 +88,51 @@ class EvalSummary(BaseModel):
         return lines
 
 
+class MultiRunSummary(BaseModel):
+    """Aggregate over repeated executions of the same task set.
+
+    Single runs are noise: the same server, tasks, and model can flip a
+    task between runs. The honest headline is mean plus range."""
+
+    summaries: list[EvalSummary]
+
+    @property
+    def n(self) -> int:
+        return len(self.summaries)
+
+    def _spread(self, values: list[float]) -> str:
+        mean = sum(values) / len(values)
+        low, high = min(values), max(values)
+        if low == high:
+            return f"{mean:.0%}"
+        return f"{mean:.0%} [{low:.0%}–{high:.0%}]"
+
+    def summary_lines(self) -> list[str]:
+        lines = [
+            f"Runs: {self.n} × {self.summaries[0].task_count} task(s)",
+            f"Hit-rate: {self._spread([s.hit_rate for s in self.summaries])}",
+            f"Tool-selection hit-rate: {self._spread([s.tool_hit_rate for s in self.summaries])}",
+            f"Tool precision (avg): {self._spread([s.avg_precision for s in self.summaries])}",
+        ]
+        judged = [s.judge_pass_rate for s in self.summaries if s.judge_pass_rate is not None]
+        if judged:
+            lines.insert(2, f"Judge pass-rate: {self._spread(judged)}")
+        extras = [s.avg_extra_calls for s in self.summaries]
+        lines.append(f"Unnecessary calls (avg): {sum(extras) / len(extras):.1f}/task")
+        flaky = self.flaky_tasks()
+        if flaky:
+            lines.append(f"Flaky tasks (hit in some runs, missed in others): {', '.join(flaky)}")
+        return lines
+
+    def flaky_tasks(self) -> list[str]:
+        """Task ids whose hit outcome differs across runs."""
+        outcomes: dict[str, set[bool]] = {}
+        for summary in self.summaries:
+            for score in summary.scores:
+                outcomes.setdefault(score.task_id, set()).add(score.hit)
+        return sorted(task_id for task_id, seen in outcomes.items() if len(seen) > 1)
+
+
 async def score_runs(
     tasks: list[TaskSpec],
     runs: list[TaskRun],