whetkit 0.4.0__tar.gz → 0.6.0__tar.gz

{whetkit-0.4.0 → whetkit-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whetkit
-Version: 0.4.0
+Version: 0.6.0
 Summary: Measure and improve how well LLM agents select and use the tools exposed by an MCP server
 Keywords: mcp,model-context-protocol,llm,agents,evals,tool-use,tool-selection,benchmark
 Author: Mohammed Benlamlih
@@ -163,12 +163,21 @@ back to the original world.
 |---|---|
 | `whetkit doctor` | Lint the tool surface: vague descriptions, cryptic names, near-duplicates, context bloat. `--json`; `--fail-on warn` for CI. |
 | `whetkit inspect` | Tool inventory: names, params, description tokens, schema complexity. |
-| `whetkit generate` | Draft eval tasks from the inventory; validated, review-before-trust YAML. |
-| `whetkit run` | Agentic eval loop with real tool execution; scored results + traces. `--plan` scores a curated view. |
-| `whetkit curate` | Baseline → LLM-proposed overlay plan → curated eval → before/after report. |
+| `whetkit generate` | Draft eval tasks from the inventory (server-context aware, read-only unless `--allow-writes`); review-before-trust YAML. |
+| `whetkit run` | Agentic eval with real tool execution. `--runs N` for mean±range and flaky-task detection, `--concurrency` for independent tasks, `--reset-cmd` for stateful fixtures, `--plan` to score a curated view, `--summary-json` for machine-readable results (with cost estimates). |
+| `whetkit curate` | Baseline → LLM-proposed overlay plan → curated eval → before/after report. `--prune-unused` for the cost play. |
+| `whetkit fix` | Self-correcting curation: propose → eval → feed regressions back → revise, up to `--max-iterations`; keeps the best plan by measured results. |
+| `whetkit plan-init` | Scaffold a view plan: `--keep a,b`, `--from-tasks tasks/`, `--from-traces traces.sqlite3` — hide everything else. |
+| `whetkit diff` | Compare two `--summary-json` files: metric deltas + per-task PASS/MISS/FLAKY transitions. |
+| `whetkit export` | Share a plan: `--to markdown` (upstream-PR fix table) or `--to json` (gateway overrides). |
 | `whetkit report` | Rebuild the HTML/JSON report from stored traces. |
 | `whetkit overlay` | Serve the curated view as a stdio MCP server. |
 
+Authenticated servers: any string in `server.json` may reference `${ENV_VARS}`
+(e.g. an `Authorization` header) — credentials never live in the file.
+Summaries flag probable task-spec gaps, errored runs, and failed tool calls
+so a bad score always says why.
+
 ## How scoring works
 
 - **Deterministic tool-match**: each task lists the expected tool calls

{whetkit-0.4.0 → whetkit-0.6.0}/README.md

@@ -132,12 +132,21 @@ back to the original world.
 |---|---|
 | `whetkit doctor` | Lint the tool surface: vague descriptions, cryptic names, near-duplicates, context bloat. `--json`; `--fail-on warn` for CI. |
 | `whetkit inspect` | Tool inventory: names, params, description tokens, schema complexity. |
-| `whetkit generate` | Draft eval tasks from the inventory; validated, review-before-trust YAML. |
-| `whetkit run` | Agentic eval loop with real tool execution; scored results + traces. `--plan` scores a curated view. |
-| `whetkit curate` | Baseline → LLM-proposed overlay plan → curated eval → before/after report. |
+| `whetkit generate` | Draft eval tasks from the inventory (server-context aware, read-only unless `--allow-writes`); review-before-trust YAML. |
+| `whetkit run` | Agentic eval with real tool execution. `--runs N` for mean±range and flaky-task detection, `--concurrency` for independent tasks, `--reset-cmd` for stateful fixtures, `--plan` to score a curated view, `--summary-json` for machine-readable results (with cost estimates). |
+| `whetkit curate` | Baseline → LLM-proposed overlay plan → curated eval → before/after report. `--prune-unused` for the cost play. |
+| `whetkit fix` | Self-correcting curation: propose → eval → feed regressions back → revise, up to `--max-iterations`; keeps the best plan by measured results. |
+| `whetkit plan-init` | Scaffold a view plan: `--keep a,b`, `--from-tasks tasks/`, `--from-traces traces.sqlite3` — hide everything else. |
+| `whetkit diff` | Compare two `--summary-json` files: metric deltas + per-task PASS/MISS/FLAKY transitions. |
+| `whetkit export` | Share a plan: `--to markdown` (upstream-PR fix table) or `--to json` (gateway overrides). |
 | `whetkit report` | Rebuild the HTML/JSON report from stored traces. |
 | `whetkit overlay` | Serve the curated view as a stdio MCP server. |
 
+Authenticated servers: any string in `server.json` may reference `${ENV_VARS}`
+(e.g. an `Authorization` header) — credentials never live in the file.
+Summaries flag probable task-spec gaps, errored runs, and failed tool calls
+so a bad score always says why.
+
 ## How scoring works
 
 - **Deterministic tool-match**: each task lists the expected tool calls

{whetkit-0.4.0 → whetkit-0.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "whetkit"
-version = "0.4.0"
+version = "0.6.0"
 description = "Measure and improve how well LLM agents select and use the tools exposed by an MCP server"
 readme = "README.md"
 license = "Apache-2.0"

{whetkit-0.4.0 → whetkit-0.6.0}/src/whetkit/cli.py

@@ -79,6 +79,8 @@ def _summary_payload(group_name: str, summary, task_runs: list) -> dict:
                 "tool_hit": score.tool_hit,
                 "judge_passed": score.judge.passed if score.judge else None,
                 "spec_gap": score.spec_gap,
+                "run_status": str(score.run_status),
+                "tool_errors": score.tool_errors,
                 "called": score.tool_match.called,
                 "missing": score.tool_match.missing_slots,
                 "extra_calls": score.tool_match.extra_calls,
@@ -271,6 +273,20 @@ def plan_init(
             help="Also keep every tool referenced by these tasks' expected_tools",
         ),
     ] = None,
+    from_traces: Annotated[
+        str | None,
+        typer.Option(
+            "--from-traces",
+            help=(
+                "Also keep every tool actually called in this trace store "
+                "(what real runs used, including tools no spec lists)"
+            ),
+        ),
+    ] = None,
+    traces_group: Annotated[
+        str | None,
+        typer.Option("--traces-group", help="Restrict --from-traces to one run group"),
+    ] = None,
     out: Annotated[
         str, typer.Option("--out", help="Where to write the plan YAML")
     ] = ".whetkit/curation-plan.yaml",
@@ -286,8 +302,18 @@ def plan_init(
         for task in load_tasks(from_tasks):
             for slot in task.expected_tool_slots:
                 keep_set.update(slot)
+    if from_traces:
+        from whetkit.tracing import TraceStore
+
+        if not Path(from_traces).is_file():
+            raise typer.BadParameter(f"no trace store at {from_traces}")
+        with TraceStore(from_traces) as trace_store:
+            for run in trace_store.load_runs(traces_group):
+                keep_set.update(run.called_tool_names)
     if not keep_set:
-        raise typer.BadParameter("nothing to keep — pass --keep and/or --from-tasks")
+        raise typer.BadParameter(
+            "nothing to keep — pass --keep, --from-tasks, and/or --from-traces"
+        )
 
     spec = _resolve_server(server, http_mode)
     inventory = asyncio.run(inspect_server(spec))
@@ -480,6 +506,17 @@ def run(
             help="Write a machine-readable summary (metrics + per-task outcomes) to this path",
         ),
     ] = None,
+    concurrency: Annotated[
+        int,
+        typer.Option(
+            "--concurrency",
+            help=(
+                "Run up to N tasks of a repetition in parallel (each gets its own "
+                "server connection). Only safe when tasks are independent — writes "
+                "to shared state should stay at 1."
+            ),
+        ),
+    ] = 1,
     reset_cmd: Annotated[
         str | None,
         typer.Option(
@@ -500,6 +537,8 @@ def run(
 
     if runs < 1:
         raise typer.BadParameter("--runs must be at least 1")
+    if concurrency < 1:
+        raise typer.BadParameter("--concurrency must be at least 1")
 
     task_list = load_tasks(tasks)
     servers = _resolve_task_servers(task_list, server, http_mode)
@@ -523,12 +562,16 @@ def run(
         name_map = curation_plan.rename_map()
 
     async def _run_once(group_name: str, cache: JudgeCache):
-        task_runs = []
-        for task in task_list:
-            typer.echo(f"running {task.id} ...", err=True)
-            task_runs.append(
-                await run_task(task, servers[task.server], config, client_factory=client_factory)
-            )
+        semaphore = asyncio.Semaphore(concurrency)
+
+        async def _one(task):
+            async with semaphore:
+                typer.echo(f"running {task.id} ...", err=True)
+                return await run_task(
+                    task, servers[task.server], config, client_factory=client_factory
+                )
+
+        task_runs = list(await asyncio.gather(*(_one(task) for task in task_list)))
 
         with TraceStore(store_path) as trace_store:
             trace_store.save_runs(task_runs, run_group=group_name)

{whetkit-0.4.0 → whetkit-0.6.0}/src/whetkit/generate.py

@@ -31,7 +31,11 @@ Rules for every task:
   is either one tool name, or a list of genuinely interchangeable
   alternatives for that step. Use ONLY tool names from the given list.
 - success_criteria is one or two concrete, checkable sentences a grader can
-  verify from the agent's final answer alone.
+  verify from the agent's final answer alone. When the data source is live
+  or changing (news pages, network traffic, timestamps, prices), grade the
+  SHAPE of the answer ("names the current top story", "lists the requests
+  that were observed"), never a specific volatile value — a criterion that
+  is wrong an hour later is a flaky eval, not a strict one.
 {writes_rule}
 - Set "ordered": true only when the steps must happen in sequence.
 - ids are short kebab-case slugs, unique across the set.

{whetkit-0.4.0 → whetkit-0.6.0}/src/whetkit/scoring/aggregate.py

@@ -15,6 +15,7 @@ class TaskScore(BaseModel):
     run_status: RunStatus
     tool_match: ToolMatchResult
     judge: JudgeVerdict | None = None
+    tool_errors: int = 0
 
     @property
     def tool_hit(self) -> bool:
@@ -82,6 +83,14 @@ class EvalSummary(BaseModel):
             return 0.0
         return sum(len(s.tool_match.extra_calls) for s in self.scores) / len(self.scores)
 
+    @property
+    def error_run_count(self) -> int:
+        return sum(s.run_status == RunStatus.ERROR for s in self.scores)
+
+    @property
+    def total_tool_errors(self) -> int:
+        return sum(s.tool_errors for s in self.scores)
+
     def summary_lines(self) -> list[str]:
         lines = [
             f"Tasks: {self.task_count}",
@@ -93,6 +102,17 @@ class EvalSummary(BaseModel):
         ]
         if (rate := self.judge_pass_rate) is not None:
             lines.insert(2, f"Judge pass-rate: {rate:.0%}")
+        if self.error_run_count:
+            lines.append(
+                f"⚠ Errored runs: {self.error_run_count}/{self.task_count} — the agent "
+                "loop failed (connection/provider); scores below reflect failures, "
+                "not tool selection"
+            )
+        if self.total_tool_errors:
+            lines.append(
+                f"⚠ Failed tool calls: {self.total_tool_errors} — agents received "
+                "errors from the server; read the traces before trusting the scores"
+            )
         return lines
 
 
@@ -175,12 +195,14 @@ async def score_runs(
         called = run.called_tool_names
         if name_map:
             called = [name_map.get(name, name) for name in called]
+        tool_errors = sum(call.is_error for turn in run.turns for call in turn.tool_calls)
         scores.append(
             TaskScore(
                 task_id=task.id,
                 run_status=run.status,
                 tool_match=score_tool_match(task, called, mode),
                 judge=verdict,
+                tool_errors=tool_errors,
             )
         )
     return EvalSummary(scores=scores)