whetkit 0.1.0__py3-none-any.whl

whetkit/mcp/client.py

@@ -0,0 +1,59 @@
+"""High-level MCP client used by the inspector, runner, and overlay proxy."""
+
+from contextlib import AsyncExitStack
+from types import TracebackType
+from typing import Any, Self
+
+import mcp.types as types
+from mcp import ClientSession
+
+from whetkit.mcp.transport import HttpMode, HttpSpec, ServerSpec, open_session
+
+
+class MCPClient:
+    """Async client over any ServerSpec.
+
+    For stdio and stateful HTTP the client holds one session for its whole
+    lifetime. For stateless HTTP (2026-07-28 spec semantics) every operation
+    runs on a fresh exchange, so no session state is assumed server-side.
+    """
+
+    def __init__(self, spec: ServerSpec):
+        self.spec = spec
+        self._stack: AsyncExitStack | None = None
+        self._session: ClientSession | None = None
+
+    @property
+    def _stateless(self) -> bool:
+        return isinstance(self.spec, HttpSpec) and self.spec.mode == HttpMode.STATELESS
+
+    async def __aenter__(self) -> Self:
+        if not self._stateless:
+            self._stack = AsyncExitStack()
+            self._session = await self._stack.enter_async_context(open_session(self.spec))
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        if self._stack is not None:
+            await self._stack.aclose()
+            self._stack = None
+            self._session = None
+
+    async def list_tools(self) -> list[types.Tool]:
+        if self._session is not None:
+            result = await self._session.list_tools()
+            return list(result.tools)
+        async with open_session(self.spec) as session:
+            result = await session.list_tools()
+            return list(result.tools)
+
+    async def call_tool(self, name: str, arguments: dict[str, Any]) -> types.CallToolResult:
+        if self._session is not None:
+            return await self._session.call_tool(name, arguments)
+        async with open_session(self.spec) as session:
+            return await session.call_tool(name, arguments)

whetkit/mcp/introspect.py

@@ -0,0 +1,106 @@
+"""Tool introspection: inventory a server's tools and summarize their cost."""
+
+from typing import Any
+
+from pydantic import BaseModel
+
+from whetkit.mcp.client import MCPClient
+from whetkit.mcp.transport import ServerSpec
+
+
+def estimate_tokens(text: str) -> int:
+    """Rough token estimate (~4 chars/token for English prose). Good enough
+    for comparing tool-set sizes; not a billing meter."""
+    return max(1, round(len(text) / 4)) if text else 0
+
+
+def schema_complexity(schema: dict[str, Any] | None) -> int:
+    """Score a JSON schema: one point per property/branch, plus nesting depth.
+
+    A flat 2-arg tool scores ~3; deeply nested or union-heavy schemas score
+    much higher — which correlates with how hard the schema is for a model
+    to fill correctly.
+    """
+    if not schema:
+        return 0
+
+    def walk(node: Any, depth: int) -> int:
+        if not isinstance(node, dict):
+            return 0
+        score = 0
+        props = node.get("properties")
+        if isinstance(props, dict):
+            for sub in props.values():
+                score += 1 + walk(sub, depth + 1)
+        for branch_key in ("anyOf", "oneOf", "allOf"):
+            branches = node.get(branch_key)
+            if isinstance(branches, list):
+                for sub in branches:
+                    score += 1 + walk(sub, depth + 1)
+        items = node.get("items")
+        if isinstance(items, dict):
+            score += walk(items, depth + 1)
+        return score + (1 if depth == 0 else 0)
+
+    return walk(schema, 0)
+
+
+class ToolInfo(BaseModel):
+    name: str
+    title: str | None = None
+    description: str = ""
+    input_schema: dict[str, Any] = {}
+
+    @property
+    def description_tokens(self) -> int:
+        return estimate_tokens(self.description)
+
+    @property
+    def complexity(self) -> int:
+        return schema_complexity(self.input_schema)
+
+    @property
+    def param_count(self) -> int:
+        props = self.input_schema.get("properties")
+        return len(props) if isinstance(props, dict) else 0
+
+
+class ServerInventory(BaseModel):
+    server: str
+    tools: list[ToolInfo]
+
+    @property
+    def tool_count(self) -> int:
+        return len(self.tools)
+
+    @property
+    def total_description_tokens(self) -> int:
+        return sum(t.description_tokens for t in self.tools)
+
+    @property
+    def total_complexity(self) -> int:
+        return sum(t.complexity for t in self.tools)
+
+    def summary_lines(self) -> list[str]:
+        avg = self.total_complexity / self.tool_count if self.tools else 0.0
+        return [
+            f"Server: {self.server}",
+            f"Tools: {self.tool_count}",
+            f"Total description tokens (est.): {self.total_description_tokens}",
+            f"Schema complexity: total {self.total_complexity}, avg {avg:.1f}",
+        ]
+
+
+async def inspect_server(spec: ServerSpec) -> ServerInventory:
+    async with MCPClient(spec) as client:
+        tools = await client.list_tools()
+    infos = [
+        ToolInfo(
+            name=t.name,
+            title=t.title,
+            description=t.description or "",
+            input_schema=t.inputSchema or {},
+        )
+        for t in tools
+    ]
+    return ServerInventory(server=spec.label(), tools=infos)

whetkit/mcp/transport.py

@@ -0,0 +1,150 @@
+"""Transport layer: how to reach an MCP server.
+
+All construction of SDK transports is isolated here so the planned move to
+the MCP Python SDK v2 (see MIGRATION.md) stays contained in this module.
+
+Three connection modes are supported, because real-world servers are mixed:
+
+- ``stdio``: a local server subprocess speaking JSON-RPC over stdin/stdout.
+- ``http`` with ``mode="stateful"``: legacy (2025 spec) streamable HTTP —
+  one long-lived session per connection, identified by a server-issued
+  session id.
+- ``http`` with ``mode="stateless"``: the 2026-07-28 stateless spec —
+  no session affinity; the client opens a fresh exchange per operation.
+  On the v1 SDK this is modeled by re-connecting per operation (see
+  ``MCPClient``), which matches the per-POST semantics from the client side.
+"""
+
+import json
+import shutil
+import sys
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from enum import StrEnum
+from pathlib import Path
+from typing import Annotated, Literal
+
+import httpx
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+from mcp.client.streamable_http import streamable_http_client
+from pydantic import BaseModel, Field
+
+
+class HttpMode(StrEnum):
+    STATEFUL = "stateful"
+    STATELESS = "stateless"
+
+
+class StdioSpec(BaseModel):
+    kind: Literal["stdio"] = "stdio"
+    command: str
+    args: list[str] = []
+    env: dict[str, str] | None = None
+    cwd: str | None = None
+
+    def label(self) -> str:
+        return f"stdio: {self.command} {' '.join(self.args)}".strip()
+
+
+class HttpSpec(BaseModel):
+    kind: Literal["http"] = "http"
+    url: str
+    headers: dict[str, str] | None = None
+    mode: HttpMode = HttpMode.STATEFUL
+
+    def label(self) -> str:
+        return f"http ({self.mode}): {self.url}"
+
+
+ServerSpec = Annotated[StdioSpec | HttpSpec, Field(discriminator="kind")]
+
+
+class _SpecAdapter(BaseModel):
+    spec: ServerSpec
+
+
+def spec_from_dict(data: dict) -> ServerSpec:
+    return _SpecAdapter(spec=data).spec
+
+
+def _python_command(command: str) -> str:
+    """Map a bare ``python`` to the running interpreter so stdio servers get
+    the project virtualenv."""
+    if command in ("python", "python3"):
+        return sys.executable
+    return shutil.which(command) or command
+
+
+def resolve_server_spec(value: str, http_mode: HttpMode = HttpMode.STATEFUL) -> ServerSpec:
+    """Turn a CLI-friendly string into a ServerSpec.
+
+    Accepted forms:
+    - ``http(s)://...``            -> streamable-HTTP server
+    - directory                    -> ``server.json`` inside it, else ``server.py`` via stdio
+    - ``*.json`` file              -> full spec document
+    - ``*.py`` file                -> stdio python server
+    """
+    if value.startswith(("http://", "https://")):
+        return HttpSpec(url=value, mode=http_mode)
+
+    path = Path(value)
+    if path.is_dir():
+        config = path / "server.json"
+        if config.is_file():
+            return _spec_from_json(config)
+        script = path / "server.py"
+        if script.is_file():
+            return StdioSpec(command=sys.executable, args=[str(script)])
+        raise ValueError(f"{value}: directory has neither server.json nor server.py")
+    if path.suffix == ".json" and path.is_file():
+        return _spec_from_json(path)
+    if path.suffix == ".py" and path.is_file():
+        return StdioSpec(command=sys.executable, args=[str(path)])
+    raise ValueError(f"{value}: not a URL, directory, .json spec, or .py server")
+
+
+def _spec_from_json(path: Path) -> ServerSpec:
+    data = json.loads(path.read_text())
+    spec = spec_from_dict(data)
+    if isinstance(spec, StdioSpec):
+        spec.command = _python_command(spec.command)
+        # Relative script paths are relative to the spec file, not the CWD.
+        spec.args = [
+            str((path.parent / a).resolve()) if (path.parent / a).is_file() else a
+            for a in spec.args
+        ]
+        if spec.cwd is None:
+            spec.cwd = str(path.parent)
+    return spec
+
+
+@asynccontextmanager
+async def open_session(spec: ServerSpec) -> AsyncIterator[ClientSession]:
+    """Open one initialized MCP session over the spec's transport."""
+    if isinstance(spec, StdioSpec):
+        params = StdioServerParameters(
+            command=spec.command, args=spec.args, env=spec.env, cwd=spec.cwd
+        )
+        async with (
+            stdio_client(params) as (read, write),
+            ClientSession(read, write) as session,
+        ):
+            await session.initialize()
+            yield session
+    else:
+        http_client = (
+            httpx.AsyncClient(headers=spec.headers, follow_redirects=True, timeout=30)
+            if spec.headers
+            else None
+        )
+        try:
+            async with (
+                streamable_http_client(spec.url, http_client=http_client) as (read, write, _sid),
+                ClientSession(read, write) as session,
+            ):
+                await session.initialize()
+                yield session
+        finally:
+            if http_client is not None:
+                await http_client.aclose()

whetkit/report/__init__.py

@@ -0,0 +1,6 @@
+"""Before/after comparison reports (JSON + self-contained HTML)."""
+
+from whetkit.report.builder import ComparisonReport, TaskComparison, build_report
+from whetkit.report.html import render_html
+
+__all__ = ["ComparisonReport", "TaskComparison", "build_report", "render_html"]

whetkit/report/builder.py

@@ -0,0 +1,228 @@
+"""Assemble the before/after comparison from two scored eval batches."""
+
+import json
+from datetime import UTC, datetime
+
+from pydantic import BaseModel, Field
+
+from whetkit.curation.plan import CurationPlan, ToolOverride
+from whetkit.datasets import TaskSpec
+from whetkit.scoring import EvalSummary, TaskScore
+from whetkit.tracing import TaskRun
+
+
+class CallView(BaseModel):
+    """One tool call as shown in the report's trace panel."""
+
+    name: str
+    args: str = ""
+    is_error: bool = False
+
+
+class SideView(BaseModel):
+    """One side (before or after) of a task comparison."""
+
+    hit: bool
+    tool_hit: bool
+    judge_passed: bool | None = None
+    judge_rationale: str | None = None
+    tools_called: list[str] = []
+    missing_slots: list[list[str]] = []
+    calls: list[CallView] = []
+    final_text: str | None = None
+    input_tokens: int = 0
+    output_tokens: int = 0
+    latency_ms: float = 0.0
+
+    @classmethod
+    def from_score(cls, score: TaskScore, run: TaskRun | None) -> "SideView":
+        usage = run.total_usage if run else None
+        calls = []
+        if run:
+            for turn in run.turns:
+                for call in turn.tool_calls:
+                    args = json.dumps(call.arguments, separators=(", ", ": "))
+                    if len(args) > 80:
+                        args = args[:77] + "…"
+                    calls.append(CallView(name=call.name, args=args, is_error=call.is_error))
+        return cls(
+            hit=score.hit,
+            tool_hit=score.tool_hit,
+            judge_passed=score.judge.passed if score.judge else None,
+            judge_rationale=score.judge.rationale if score.judge else None,
+            tools_called=score.tool_match.called,
+            missing_slots=score.tool_match.missing_slots,
+            calls=calls,
+            final_text=run.final_text if run else None,
+            input_tokens=usage.input_tokens if usage else 0,
+            output_tokens=usage.output_tokens if usage else 0,
+            latency_ms=run.total_latency_ms if run else 0.0,
+        )
+
+
+class TaskComparison(BaseModel):
+    task_id: str
+    prompt: str
+    expected_slots: list[list[str]]
+    before: SideView
+    after: SideView
+
+    @property
+    def outcome(self) -> str:
+        if not self.before.hit and self.after.hit:
+            return "improved"
+        if self.before.hit and not self.after.hit:
+            return "regressed"
+        return "unchanged"
+
+
+class ActionImpact(BaseModel):
+    """One curation action and the tasks it plausibly affected.
+
+    Attribution is heuristic: an override is linked to a task when the tool
+    it touches appears in the task's expected slots or in either run's calls.
+    """
+
+    override: ToolOverride
+    action: str
+    touched_tasks: list[str] = []
+    improved_tasks: list[str] = []
+
+
+class Totals(BaseModel):
+    hit_rate: float
+    tool_hit_rate: float
+    judge_pass_rate: float | None
+    avg_precision: float
+    avg_recall: float
+    input_tokens: int
+    output_tokens: int
+    latency_ms: float
+
+    @classmethod
+    def from_summary(cls, summary: EvalSummary, runs: list[TaskRun]) -> "Totals":
+        return cls(
+            hit_rate=summary.hit_rate,
+            tool_hit_rate=summary.tool_hit_rate,
+            judge_pass_rate=summary.judge_pass_rate,
+            avg_precision=summary.avg_precision,
+            avg_recall=summary.avg_recall,
+            input_tokens=sum(r.total_usage.input_tokens for r in runs),
+            output_tokens=sum(r.total_usage.output_tokens for r in runs),
+            latency_ms=sum(r.total_latency_ms for r in runs),
+        )
+
+
+class ComparisonReport(BaseModel):
+    title: str = "whetkit before/after report"
+    server: str = ""
+    model: str = ""
+    generated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    tools_before: int | None = None
+    tools_after: int | None = None
+    before: Totals
+    after: Totals
+    tasks: list[TaskComparison]
+    plan: CurationPlan
+    action_impacts: list[ActionImpact]
+
+    @property
+    def run_id(self) -> str:
+        """Short stable id derived from the report's content."""
+        import hashlib
+
+        payload = (
+            json.dumps([self.server, self.model, [t.task_id for t in self.tasks]], sort_keys=True)
+            + self.generated_at.isoformat()
+        )
+        return hashlib.sha256(payload.encode()).hexdigest()[:6]
+
+    @property
+    def improved(self) -> list[TaskComparison]:
+        return [t for t in self.tasks if t.outcome == "improved"]
+
+    @property
+    def regressed(self) -> list[TaskComparison]:
+        return [t for t in self.tasks if t.outcome == "regressed"]
+
+
+def _override_action(override: ToolOverride) -> str:
+    if override.hidden:
+        return "prune"
+    if override.new_name and override.new_description:
+        return "rename + rewrite"
+    if override.new_name:
+        return "rename"
+    if override.new_description:
+        return "rewrite"
+    return "keep"
+
+
+def _attribute_actions(
+    plan: CurationPlan, comparisons: list[TaskComparison], tasks: list[TaskSpec]
+) -> list[ActionImpact]:
+    tasks_by_id = {t.id: t for t in tasks}
+    impacts: list[ActionImpact] = []
+    for override in plan.overrides:
+        names = {override.original_name, override.presented_name}
+        impact = ActionImpact(override=override, action=_override_action(override))
+        for comparison in comparisons:
+            task = tasks_by_id.get(comparison.task_id)
+            slot_names = {n for slot in (task.expected_tool_slots if task else []) for n in slot}
+            involved = bool(
+                names & slot_names
+                or names & set(comparison.before.tools_called)
+                or names & set(comparison.after.tools_called)
+            )
+            if involved:
+                impact.touched_tasks.append(comparison.task_id)
+                if comparison.outcome == "improved":
+                    impact.improved_tasks.append(comparison.task_id)
+        impacts.append(impact)
+    return impacts
+
+
+def build_report(
+    tasks: list[TaskSpec],
+    baseline_runs: list[TaskRun],
+    baseline_summary: EvalSummary,
+    curated_runs: list[TaskRun],
+    curated_summary: EvalSummary,
+    plan: CurationPlan,
+    model: str = "",
+    server: str = "",
+    tools_before: int | None = None,
+    tools_after: int | None = None,
+) -> ComparisonReport:
+    baseline_runs_by_id = {r.task_id: r for r in baseline_runs}
+    curated_runs_by_id = {r.task_id: r for r in curated_runs}
+    baseline_scores = {s.task_id: s for s in baseline_summary.scores}
+    curated_scores = {s.task_id: s for s in curated_summary.scores}
+
+    comparisons: list[TaskComparison] = []
+    for task in tasks:
+        before_score = baseline_scores.get(task.id)
+        after_score = curated_scores.get(task.id)
+        if before_score is None or after_score is None:
+            continue
+        comparisons.append(
+            TaskComparison(
+                task_id=task.id,
+                prompt=task.prompt.strip(),
+                expected_slots=task.expected_tool_slots,
+                before=SideView.from_score(before_score, baseline_runs_by_id.get(task.id)),
+                after=SideView.from_score(after_score, curated_runs_by_id.get(task.id)),
+            )
+        )
+
+    return ComparisonReport(
+        server=server,
+        model=model,
+        tools_before=tools_before,
+        tools_after=tools_after,
+        before=Totals.from_summary(baseline_summary, baseline_runs),
+        after=Totals.from_summary(curated_summary, curated_runs),
+        tasks=comparisons,
+        plan=plan,
+        action_impacts=_attribute_actions(plan, comparisons, tasks),
+    )