whetkit 0.5.0__tar.gz → 0.6.0__tar.gz

{whetkit-0.5.0 → whetkit-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whetkit
-Version: 0.5.0
+Version: 0.6.0
 Summary: Measure and improve how well LLM agents select and use the tools exposed by an MCP server
 Keywords: mcp,model-context-protocol,llm,agents,evals,tool-use,tool-selection,benchmark
 Author: Mohammed Benlamlih
@@ -163,12 +163,21 @@ back to the original world.
 |---|---|
 | `whetkit doctor` | Lint the tool surface: vague descriptions, cryptic names, near-duplicates, context bloat. `--json`; `--fail-on warn` for CI. |
 | `whetkit inspect` | Tool inventory: names, params, description tokens, schema complexity. |
-| `whetkit generate` | Draft eval tasks from the inventory; validated, review-before-trust YAML. |
-| `whetkit run` | Agentic eval loop with real tool execution; scored results + traces. `--plan` scores a curated view. |
-| `whetkit curate` | Baseline → LLM-proposed overlay plan → curated eval → before/after report. |
+| `whetkit generate` | Draft eval tasks from the inventory (server-context aware, read-only unless `--allow-writes`); review-before-trust YAML. |
+| `whetkit run` | Agentic eval with real tool execution. `--runs N` for mean±range and flaky-task detection, `--concurrency` for independent tasks, `--reset-cmd` for stateful fixtures, `--plan` to score a curated view, `--summary-json` for machine-readable results (with cost estimates). |
+| `whetkit curate` | Baseline → LLM-proposed overlay plan → curated eval → before/after report. `--prune-unused` for the cost play. |
+| `whetkit fix` | Self-correcting curation: propose → eval → feed regressions back → revise, up to `--max-iterations`; keeps the best plan by measured results. |
+| `whetkit plan-init` | Scaffold a view plan: `--keep a,b`, `--from-tasks tasks/`, `--from-traces traces.sqlite3` — hide everything else. |
+| `whetkit diff` | Compare two `--summary-json` files: metric deltas + per-task PASS/MISS/FLAKY transitions. |
+| `whetkit export` | Share a plan: `--to markdown` (upstream-PR fix table) or `--to json` (gateway overrides). |
 | `whetkit report` | Rebuild the HTML/JSON report from stored traces. |
 | `whetkit overlay` | Serve the curated view as a stdio MCP server. |
 
+Authenticated servers: any string in `server.json` may reference `${ENV_VARS}`
+(e.g. an `Authorization` header) — credentials never live in the file.
+Summaries flag probable task-spec gaps, errored runs, and failed tool calls
+so a bad score always says why.
+
 ## How scoring works
 
 - **Deterministic tool-match**: each task lists the expected tool calls

{whetkit-0.5.0 → whetkit-0.6.0}/README.md

@@ -132,12 +132,21 @@ back to the original world.
 |---|---|
 | `whetkit doctor` | Lint the tool surface: vague descriptions, cryptic names, near-duplicates, context bloat. `--json`; `--fail-on warn` for CI. |
 | `whetkit inspect` | Tool inventory: names, params, description tokens, schema complexity. |
-| `whetkit generate` | Draft eval tasks from the inventory; validated, review-before-trust YAML. |
-| `whetkit run` | Agentic eval loop with real tool execution; scored results + traces. `--plan` scores a curated view. |
-| `whetkit curate` | Baseline → LLM-proposed overlay plan → curated eval → before/after report. |
+| `whetkit generate` | Draft eval tasks from the inventory (server-context aware, read-only unless `--allow-writes`); review-before-trust YAML. |
+| `whetkit run` | Agentic eval with real tool execution. `--runs N` for mean±range and flaky-task detection, `--concurrency` for independent tasks, `--reset-cmd` for stateful fixtures, `--plan` to score a curated view, `--summary-json` for machine-readable results (with cost estimates). |
+| `whetkit curate` | Baseline → LLM-proposed overlay plan → curated eval → before/after report. `--prune-unused` for the cost play. |
+| `whetkit fix` | Self-correcting curation: propose → eval → feed regressions back → revise, up to `--max-iterations`; keeps the best plan by measured results. |
+| `whetkit plan-init` | Scaffold a view plan: `--keep a,b`, `--from-tasks tasks/`, `--from-traces traces.sqlite3` — hide everything else. |
+| `whetkit diff` | Compare two `--summary-json` files: metric deltas + per-task PASS/MISS/FLAKY transitions. |
+| `whetkit export` | Share a plan: `--to markdown` (upstream-PR fix table) or `--to json` (gateway overrides). |
 | `whetkit report` | Rebuild the HTML/JSON report from stored traces. |
 | `whetkit overlay` | Serve the curated view as a stdio MCP server. |
 
+Authenticated servers: any string in `server.json` may reference `${ENV_VARS}`
+(e.g. an `Authorization` header) — credentials never live in the file.
+Summaries flag probable task-spec gaps, errored runs, and failed tool calls
+so a bad score always says why.
+
 ## How scoring works
 
 - **Deterministic tool-match**: each task lists the expected tool calls

{whetkit-0.5.0 → whetkit-0.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "whetkit"
-version = "0.5.0"
+version = "0.6.0"
 description = "Measure and improve how well LLM agents select and use the tools exposed by an MCP server"
 readme = "README.md"
 license = "Apache-2.0"

{whetkit-0.5.0 → whetkit-0.6.0}/src/whetkit/cli.py

@@ -79,6 +79,8 @@ def _summary_payload(group_name: str, summary, task_runs: list) -> dict:
                 "tool_hit": score.tool_hit,
                 "judge_passed": score.judge.passed if score.judge else None,
                 "spec_gap": score.spec_gap,
+                "run_status": str(score.run_status),
+                "tool_errors": score.tool_errors,
                 "called": score.tool_match.called,
                 "missing": score.tool_match.missing_slots,
                 "extra_calls": score.tool_match.extra_calls,

{whetkit-0.5.0 → whetkit-0.6.0}/src/whetkit/scoring/aggregate.py

@@ -15,6 +15,7 @@ class TaskScore(BaseModel):
     run_status: RunStatus
     tool_match: ToolMatchResult
     judge: JudgeVerdict | None = None
+    tool_errors: int = 0
 
     @property
     def tool_hit(self) -> bool:
@@ -82,6 +83,14 @@ class EvalSummary(BaseModel):
             return 0.0
         return sum(len(s.tool_match.extra_calls) for s in self.scores) / len(self.scores)
 
+    @property
+    def error_run_count(self) -> int:
+        return sum(s.run_status == RunStatus.ERROR for s in self.scores)
+
+    @property
+    def total_tool_errors(self) -> int:
+        return sum(s.tool_errors for s in self.scores)
+
     def summary_lines(self) -> list[str]:
         lines = [
             f"Tasks: {self.task_count}",
@@ -93,6 +102,17 @@ class EvalSummary(BaseModel):
         ]
         if (rate := self.judge_pass_rate) is not None:
             lines.insert(2, f"Judge pass-rate: {rate:.0%}")
+        if self.error_run_count:
+            lines.append(
+                f"⚠ Errored runs: {self.error_run_count}/{self.task_count} — the agent "
+                "loop failed (connection/provider); scores below reflect failures, "
+                "not tool selection"
+            )
+        if self.total_tool_errors:
+            lines.append(
+                f"⚠ Failed tool calls: {self.total_tool_errors} — agents received "
+                "errors from the server; read the traces before trusting the scores"
+            )
         return lines
 
 
@@ -175,12 +195,14 @@ async def score_runs(
         called = run.called_tool_names
         if name_map:
             called = [name_map.get(name, name) for name in called]
+        tool_errors = sum(call.is_error for turn in run.turns for call in turn.tool_calls)
         scores.append(
             TaskScore(
                 task_id=task.id,
                 run_status=run.status,
                 tool_match=score_tool_match(task, called, mode),
                 judge=verdict,
+                tool_errors=tool_errors,
             )
         )
     return EvalSummary(scores=scores)