whetkit 0.4.0__tar.gz → 0.5.0__tar.gz

{whetkit-0.4.0 → whetkit-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whetkit
-Version: 0.4.0
+Version: 0.5.0
 Summary: Measure and improve how well LLM agents select and use the tools exposed by an MCP server
 Keywords: mcp,model-context-protocol,llm,agents,evals,tool-use,tool-selection,benchmark
 Author: Mohammed Benlamlih

{whetkit-0.4.0 → whetkit-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "whetkit"
-version = "0.4.0"
+version = "0.5.0"
 description = "Measure and improve how well LLM agents select and use the tools exposed by an MCP server"
 readme = "README.md"
 license = "Apache-2.0"

{whetkit-0.4.0 → whetkit-0.5.0}/src/whetkit/cli.py

@@ -271,6 +271,20 @@ def plan_init(
             help="Also keep every tool referenced by these tasks' expected_tools",
         ),
     ] = None,
+    from_traces: Annotated[
+        str | None,
+        typer.Option(
+            "--from-traces",
+            help=(
+                "Also keep every tool actually called in this trace store "
+                "(what real runs used, including tools no spec lists)"
+            ),
+        ),
+    ] = None,
+    traces_group: Annotated[
+        str | None,
+        typer.Option("--traces-group", help="Restrict --from-traces to one run group"),
+    ] = None,
     out: Annotated[
         str, typer.Option("--out", help="Where to write the plan YAML")
     ] = ".whetkit/curation-plan.yaml",
@@ -286,8 +300,18 @@ def plan_init(
         for task in load_tasks(from_tasks):
             for slot in task.expected_tool_slots:
                 keep_set.update(slot)
+    if from_traces:
+        from whetkit.tracing import TraceStore
+
+        if not Path(from_traces).is_file():
+            raise typer.BadParameter(f"no trace store at {from_traces}")
+        with TraceStore(from_traces) as trace_store:
+            for run in trace_store.load_runs(traces_group):
+                keep_set.update(run.called_tool_names)
     if not keep_set:
-        raise typer.BadParameter("nothing to keep — pass --keep and/or --from-tasks")
+        raise typer.BadParameter(
+            "nothing to keep — pass --keep, --from-tasks, and/or --from-traces"
+        )
 
     spec = _resolve_server(server, http_mode)
     inventory = asyncio.run(inspect_server(spec))
@@ -480,6 +504,17 @@ def run(
             help="Write a machine-readable summary (metrics + per-task outcomes) to this path",
         ),
     ] = None,
+    concurrency: Annotated[
+        int,
+        typer.Option(
+            "--concurrency",
+            help=(
+                "Run up to N tasks of a repetition in parallel (each gets its own "
+                "server connection). Only safe when tasks are independent — writes "
+                "to shared state should stay at 1."
+            ),
+        ),
+    ] = 1,
     reset_cmd: Annotated[
         str | None,
         typer.Option(
@@ -500,6 +535,8 @@ def run(
 
     if runs < 1:
         raise typer.BadParameter("--runs must be at least 1")
+    if concurrency < 1:
+        raise typer.BadParameter("--concurrency must be at least 1")
 
     task_list = load_tasks(tasks)
     servers = _resolve_task_servers(task_list, server, http_mode)
@@ -523,12 +560,16 @@ def run(
         name_map = curation_plan.rename_map()
 
     async def _run_once(group_name: str, cache: JudgeCache):
-        task_runs = []
-        for task in task_list:
-            typer.echo(f"running {task.id} ...", err=True)
-            task_runs.append(
-                await run_task(task, servers[task.server], config, client_factory=client_factory)
-            )
+        semaphore = asyncio.Semaphore(concurrency)
+
+        async def _one(task):
+            async with semaphore:
+                typer.echo(f"running {task.id} ...", err=True)
+                return await run_task(
+                    task, servers[task.server], config, client_factory=client_factory
+                )
+
+        task_runs = list(await asyncio.gather(*(_one(task) for task in task_list)))
 
         with TraceStore(store_path) as trace_store:
             trace_store.save_runs(task_runs, run_group=group_name)

{whetkit-0.4.0 → whetkit-0.5.0}/src/whetkit/generate.py

@@ -31,7 +31,11 @@ Rules for every task:
   is either one tool name, or a list of genuinely interchangeable
   alternatives for that step. Use ONLY tool names from the given list.
 - success_criteria is one or two concrete, checkable sentences a grader can
-  verify from the agent's final answer alone.
+  verify from the agent's final answer alone. When the data source is live
+  or changing (news pages, network traffic, timestamps, prices), grade the
+  SHAPE of the answer ("names the current top story", "lists the requests
+  that were observed"), never a specific volatile value — a criterion that
+  is wrong an hour later is a flaky eval, not a strict one.
 {writes_rule}
 - Set "ordered": true only when the steps must happen in sequence.
 - ids are short kebab-case slugs, unique across the set.