whetkit 0.1.0__py3-none-any.whl

whetkit/curation/overlay.py

@@ -0,0 +1,73 @@
+"""The overlay: presents a curated tool set while delegating every call to
+the untouched origin server.
+
+Two ways to use it:
+
+- :class:`CuratedMCPClient` — in-process overlay used by the eval runner for
+  before/after comparisons.
+- :func:`serve_overlay` — a real stdio MCP server (``whetkit overlay``) so
+  any MCP client (Claude Code, an IDE, another agent) can talk to the
+  curated view. The origin server is never modified; stop the proxy and
+  nothing remains.
+"""
+
+import mcp.types as types
+from mcp.server.lowlevel import Server
+from mcp.server.stdio import stdio_server
+
+from whetkit.curation.plan import CurationPlan
+from whetkit.mcp import MCPClient, ServerSpec
+
+
+class UnknownCuratedTool(Exception):
+    pass
+
+
+class CuratedMCPClient(MCPClient):
+    """MCPClient that shows plan-transformed tools and un-maps names on call."""
+
+    def __init__(self, spec: ServerSpec, plan: CurationPlan):
+        super().__init__(spec)
+        self.plan = plan
+        self._name_map: dict[str, str] | None = None
+
+    async def _mapping(self) -> dict[str, str]:
+        if self._name_map is None:
+            origin_tools = await super().list_tools()
+            self._name_map = self.plan.presented_to_original({t.name for t in origin_tools})
+        return self._name_map
+
+    async def list_tools(self) -> list[types.Tool]:
+        tools = await super().list_tools()
+        self._name_map = self.plan.presented_to_original({t.name for t in tools})
+        return self.plan.transform_tools(tools)
+
+    async def call_tool(self, name: str, arguments: dict) -> types.CallToolResult:
+        mapping = await self._mapping()
+        if name not in mapping:
+            raise UnknownCuratedTool(f"tool {name!r} is not part of the curated tool set")
+        return await super().call_tool(mapping[name], arguments)
+
+
+def build_overlay_server(client: CuratedMCPClient, name: str = "whetkit-overlay") -> Server:
+    server = Server(name)
+
+    @server.list_tools()
+    async def _list_tools() -> list[types.Tool]:
+        return await client.list_tools()
+
+    @server.call_tool()
+    async def _call_tool(tool_name: str, arguments: dict) -> types.CallToolResult:
+        # Pass the origin's result through verbatim (content, structured
+        # content, and error flag) — the overlay transforms metadata only.
+        return await client.call_tool(tool_name, arguments)
+
+    return server
+
+
+async def serve_overlay(origin: ServerSpec, plan: CurationPlan) -> None:
+    """Run the overlay as a stdio MCP server until the client disconnects."""
+    async with CuratedMCPClient(origin, plan) as client:
+        server = build_overlay_server(client)
+        async with stdio_server() as (read, write):
+            await server.run(read, write, server.create_initialization_options())

whetkit/curation/plan.py

@@ -0,0 +1,109 @@
+"""Curation plans: a declarative, reversible description of tool-set changes.
+
+A plan never touches the origin server. It only says how the overlay should
+*present* the origin's tools: hide (prune), rename, or rewrite descriptions.
+Merging duplicate tools is expressed as hiding the redundant copies and
+renaming/redescribing the canonical one — call behavior is always delegated
+1:1 to an existing origin tool, which is what keeps the overlay fully
+reversible (delete the plan and nothing remains).
+
+Plans serialize to YAML so they can be reviewed, versioned, and hand-edited.
+"""
+
+import re
+from pathlib import Path
+
+import mcp.types as types
+import yaml
+from pydantic import BaseModel, Field
+
+_NAME_RE = re.compile(r"^[a-zA-Z0-9_-]{1,128}$")
+
+
+class ToolOverride(BaseModel):
+    """How the overlay presents one origin tool."""
+
+    original_name: str
+    new_name: str | None = None
+    new_description: str | None = None
+    hidden: bool = False
+    reason: str = ""
+
+    @property
+    def presented_name(self) -> str:
+        return self.new_name or self.original_name
+
+
+class CurationPlan(BaseModel):
+    server: str = ""
+    notes: str = ""
+    overrides: list[ToolOverride] = Field(default_factory=list)
+
+    def override_for(self, original_name: str) -> ToolOverride | None:
+        return next((o for o in self.overrides if o.original_name == original_name), None)
+
+    def validate_against(self, origin_tool_names: set[str]) -> list[str]:
+        """Return every problem that makes the plan unsafe to apply."""
+        problems: list[str] = []
+        seen_originals: set[str] = set()
+        for override in self.overrides:
+            if override.original_name not in origin_tool_names:
+                problems.append(f"override targets unknown tool {override.original_name!r}")
+            if override.original_name in seen_originals:
+                problems.append(f"duplicate override for {override.original_name!r}")
+            seen_originals.add(override.original_name)
+            if override.new_name is not None and not _NAME_RE.match(override.new_name):
+                problems.append(f"invalid new name {override.new_name!r}")
+
+        presented = [
+            self.override_for(name).presented_name if self.override_for(name) else name
+            for name in sorted(origin_tool_names)
+            if not (self.override_for(name) and self.override_for(name).hidden)
+        ]
+        duplicates = {name for name in presented if presented.count(name) > 1}
+        problems.extend(f"presented tool name collision: {name!r}" for name in sorted(duplicates))
+        return problems
+
+    def presented_to_original(self, origin_tool_names: set[str]) -> dict[str, str]:
+        """Map every name the agent sees to the origin tool it delegates to."""
+        mapping: dict[str, str] = {}
+        for name in origin_tool_names:
+            override = self.override_for(name)
+            if override and override.hidden:
+                continue
+            mapping[override.presented_name if override else name] = name
+        return mapping
+
+    def transform_tools(self, tools: list[types.Tool]) -> list[types.Tool]:
+        """Present the origin's tool list through this plan."""
+        presented: list[types.Tool] = []
+        for tool in tools:
+            override = self.override_for(tool.name)
+            if override is None:
+                presented.append(tool)
+                continue
+            if override.hidden:
+                continue
+            presented.append(
+                tool.model_copy(
+                    update={
+                        "name": override.presented_name,
+                        "description": (
+                            override.new_description
+                            if override.new_description is not None
+                            else tool.description
+                        ),
+                    }
+                )
+            )
+        return presented
+
+
+def save_plan(plan: CurationPlan, path: str | Path) -> None:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(yaml.safe_dump(plan.model_dump(exclude_defaults=True), sort_keys=False))
+
+
+def load_plan(path: str | Path) -> CurationPlan:
+    return CurationPlan.model_validate(yaml.safe_load(Path(path).read_text()))

whetkit/datasets/__init__.py

@@ -0,0 +1,5 @@
+"""Eval datasets: task schema, loading, and validation."""
+
+from whetkit.datasets.tasks import TaskSpec, load_task_file, load_tasks
+
+__all__ = ["TaskSpec", "load_task_file", "load_tasks"]

whetkit/datasets/tasks.py

@@ -0,0 +1,118 @@
+"""Task schema and YAML loader.
+
+A task file is YAML holding either a single task mapping or a list of task
+mappings. The format is documented in docs/task-format.md; the source of
+truth for validation is :class:`TaskSpec`.
+"""
+
+import re
+from pathlib import Path
+
+import yaml
+from pydantic import BaseModel, Field, field_validator
+
+_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
+
+
+class TaskSpec(BaseModel):
+    """One eval task: a user request plus what a correct agent run looks like."""
+
+    id: str
+    prompt: str = Field(min_length=1)
+    server: str = Field(min_length=1, description="MCP server: URL, directory, or file path")
+    expected_tools: list[str | list[str]] = Field(
+        min_length=1,
+        description=(
+            "Tool calls a correct run makes. Each entry is one expected call; "
+            "an entry may list acceptable alternatives for that call."
+        ),
+    )
+    ordered: bool = Field(
+        default=False,
+        description="If true, expected calls must happen in the listed order.",
+    )
+    success_criteria: str = Field(
+        min_length=1,
+        description="Natural-language rubric the LLM judge grades the final answer against.",
+    )
+    tags: list[str] = []
+
+    @field_validator("id")
+    @classmethod
+    def _valid_id(cls, v: str) -> str:
+        if not _ID_RE.match(v):
+            raise ValueError(
+                f"task id {v!r} must be lowercase alphanumeric with '-' or '_' separators"
+            )
+        return v
+
+    @field_validator("expected_tools")
+    @classmethod
+    def _non_empty_slots(cls, v: list[str | list[str]]) -> list[str | list[str]]:
+        for slot in v:
+            if isinstance(slot, list) and not slot:
+                raise ValueError("an expected_tools alternatives list may not be empty")
+        return v
+
+    @property
+    def expected_tool_slots(self) -> list[list[str]]:
+        """expected_tools normalized so every slot is a list of alternatives."""
+        return [[s] if isinstance(s, str) else list(s) for s in self.expected_tools]
+
+    def resolve_server(self, base_dir: Path | None = None) -> str:
+        """Resolve a relative ``server`` path against the task file's directory."""
+        if self.server.startswith(("http://", "https://")):
+            return self.server
+        path = Path(self.server)
+        if not path.is_absolute() and base_dir is not None:
+            candidate = (base_dir / path).resolve()
+            if candidate.exists():
+                return str(candidate)
+        return self.server
+
+
+def load_task_file(path: Path) -> list[TaskSpec]:
+    """Load one YAML file containing a task mapping or a list of them."""
+    data = yaml.safe_load(path.read_text())
+    if data is None:
+        raise ValueError(f"{path}: file is empty")
+    raw_tasks = data if isinstance(data, list) else [data]
+    tasks = []
+    for i, raw in enumerate(raw_tasks):
+        if not isinstance(raw, dict):
+            raise ValueError(f"{path}: entry {i} is not a mapping")
+        try:
+            task = TaskSpec.model_validate(raw)
+        except Exception as exc:
+            raise ValueError(f"{path}: entry {i} is invalid: {exc}") from exc
+        task = task.model_copy(update={"server": task.resolve_server(path.parent)})
+        tasks.append(task)
+    return tasks
+
+
+def load_tasks(path: str | Path) -> list[TaskSpec]:
+    """Load tasks from a YAML file or every ``*.yaml``/``*.yml`` in a directory.
+
+    Raises ValueError on validation failures or duplicate task ids.
+    """
+    root = Path(path)
+    if root.is_dir():
+        files = sorted(p for p in root.iterdir() if p.suffix in (".yaml", ".yml"))
+        if not files:
+            raise ValueError(f"{root}: no .yaml/.yml task files found")
+    elif root.is_file():
+        files = [root]
+    else:
+        raise ValueError(f"{root}: no such file or directory")
+
+    tasks: list[TaskSpec] = []
+    seen: dict[str, Path] = {}
+    for file in files:
+        for task in load_task_file(file):
+            if task.id in seen:
+                raise ValueError(
+                    f"duplicate task id {task.id!r} in {file} (first seen in {seen[task.id]})"
+                )
+            seen[task.id] = file
+            tasks.append(task)
+    return tasks

whetkit/llm/__init__.py

@@ -0,0 +1,24 @@
+"""Provider-abstracted LLM layer (Anthropic + OpenAI)."""
+
+from whetkit.llm.base import (
+    ChatMessage,
+    LLMProvider,
+    LLMTurn,
+    ToolCall,
+    ToolDef,
+    ToolResult,
+    Usage,
+)
+from whetkit.llm.registry import get_provider, parse_model
+
+__all__ = [
+    "ChatMessage",
+    "LLMProvider",
+    "LLMTurn",
+    "ToolCall",
+    "ToolDef",
+    "ToolResult",
+    "Usage",
+    "get_provider",
+    "parse_model",
+]

whetkit/llm/anthropic_provider.py

@@ -0,0 +1,90 @@
+"""Anthropic provider (Messages API, anthropic SDK).
+
+Reads the API key from ``ANTHROPIC_API_KEY``.
+"""
+
+import json
+from typing import Any
+
+from anthropic import AsyncAnthropic
+
+from whetkit.llm.base import ChatMessage, LLMProvider, LLMTurn, ToolCall, ToolDef, Usage
+
+
+def _to_anthropic_messages(messages: list[ChatMessage]) -> list[dict[str, Any]]:
+    out: list[dict[str, Any]] = []
+    for msg in messages:
+        blocks: list[dict[str, Any]] = []
+        if msg.role == "assistant":
+            if msg.content:
+                blocks.append({"type": "text", "text": msg.content})
+            for call in msg.tool_calls:
+                blocks.append(
+                    {"type": "tool_use", "id": call.id, "name": call.name, "input": call.arguments}
+                )
+        else:
+            for result in msg.tool_results:
+                blocks.append(
+                    {
+                        "type": "tool_result",
+                        "tool_use_id": result.call_id,
+                        "content": result.content,
+                        "is_error": result.is_error,
+                    }
+                )
+            if msg.content:
+                blocks.append({"type": "text", "text": msg.content})
+        out.append({"role": msg.role, "content": blocks})
+    return out
+
+
+class AnthropicProvider(LLMProvider):
+    name = "anthropic"
+
+    def __init__(self, client: AsyncAnthropic | None = None):
+        self._client = client or AsyncAnthropic()
+
+    async def complete(
+        self,
+        *,
+        model: str,
+        system: str | None,
+        messages: list[ChatMessage],
+        tools: list[ToolDef],
+        max_tokens: int = 1024,
+    ) -> LLMTurn:
+        kwargs: dict[str, Any] = {}
+        if system:
+            kwargs["system"] = system
+        if tools:
+            kwargs["tools"] = [
+                {"name": t.name, "description": t.description, "input_schema": t.input_schema}
+                for t in tools
+            ]
+        response = await self._client.messages.create(
+            model=model,
+            max_tokens=max_tokens,
+            messages=_to_anthropic_messages(messages),
+            **kwargs,
+        )
+
+        text_parts: list[str] = []
+        tool_calls: list[ToolCall] = []
+        for block in response.content:
+            if block.type == "text":
+                text_parts.append(block.text)
+            elif block.type == "tool_use":
+                arguments = block.input
+                if isinstance(arguments, str):  # defensive: some models emit JSON strings
+                    arguments = json.loads(arguments or "{}")
+                tool_calls.append(ToolCall(id=block.id, name=block.name, arguments=arguments))
+
+        return LLMTurn(
+            text="\n".join(text_parts) or None,
+            tool_calls=tool_calls,
+            usage=Usage(
+                input_tokens=response.usage.input_tokens,
+                output_tokens=response.usage.output_tokens,
+            ),
+            stop_reason=response.stop_reason,
+        )

whetkit/llm/base.py

@@ -0,0 +1,88 @@
+"""Provider-neutral chat/tool-use types and the provider interface.
+
+The runner speaks only these types; each provider module translates to and
+from its SDK's wire format. Adding a provider means implementing
+:class:`LLMProvider` and registering it in ``registry.py``.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class ToolDef(BaseModel):
+    """A tool offered to the model."""
+
+    name: str
+    description: str = ""
+    input_schema: dict[str, Any] = Field(default_factory=lambda: {"type": "object"})
+
+
+class ToolCall(BaseModel):
+    """A tool invocation requested by the model."""
+
+    id: str
+    name: str
+    arguments: dict[str, Any] = {}
+
+
+class ToolResult(BaseModel):
+    """The outcome of executing a ToolCall, fed back to the model."""
+
+    call_id: str
+    content: str
+    is_error: bool = False
+
+
+class ChatMessage(BaseModel):
+    """One conversation message in provider-neutral form.
+
+    - role="user": ``content`` holds user text and/or ``tool_results`` holds
+      results for the assistant's previous tool calls.
+    - role="assistant": ``content`` holds assistant text and ``tool_calls``
+      holds any tool invocations it requested.
+    """
+
+    role: Literal["user", "assistant"]
+    content: str | None = None
+    tool_calls: list[ToolCall] = []
+    tool_results: list[ToolResult] = []
+
+
+class Usage(BaseModel):
+    input_tokens: int = 0
+    output_tokens: int = 0
+
+    def __add__(self, other: "Usage") -> "Usage":
+        return Usage(
+            input_tokens=self.input_tokens + other.input_tokens,
+            output_tokens=self.output_tokens + other.output_tokens,
+        )
+
+
+class LLMTurn(BaseModel):
+    """One assistant completion."""
+
+    text: str | None = None
+    tool_calls: list[ToolCall] = []
+    usage: Usage = Usage()
+    stop_reason: str | None = None
+
+
+class LLMProvider(ABC):
+    """One chat completion with tool use. Implementations must be stateless
+    across calls: the full conversation is passed in every time."""
+
+    name: str
+
+    @abstractmethod
+    async def complete(
+        self,
+        *,
+        model: str,
+        system: str | None,
+        messages: list[ChatMessage],
+        tools: list[ToolDef],
+        max_tokens: int = 1024,
+    ) -> LLMTurn: ...

whetkit/llm/openai_provider.py

@@ -0,0 +1,100 @@
+"""OpenAI provider (Chat Completions API, openai SDK).
+
+Chat Completions (rather than the Responses API) keeps the tool-use loop
+symmetric with other providers; both live behind :class:`LLMProvider`, so
+swapping APIs is contained here. Reads the API key from ``OPENAI_API_KEY``.
+"""
+
+import json
+from typing import Any
+
+from openai import AsyncOpenAI
+
+from whetkit.llm.base import ChatMessage, LLMProvider, LLMTurn, ToolCall, ToolDef, Usage
+
+
+def _to_openai_messages(system: str | None, messages: list[ChatMessage]) -> list[dict[str, Any]]:
+    out: list[dict[str, Any]] = []
+    if system:
+        out.append({"role": "system", "content": system})
+    for msg in messages:
+        if msg.role == "assistant":
+            entry: dict[str, Any] = {"role": "assistant", "content": msg.content}
+            if msg.tool_calls:
+                entry["tool_calls"] = [
+                    {
+                        "id": call.id,
+                        "type": "function",
+                        "function": {
+                            "name": call.name,
+                            "arguments": json.dumps(call.arguments),
+                        },
+                    }
+                    for call in msg.tool_calls
+                ]
+            out.append(entry)
+        else:
+            for result in msg.tool_results:
+                out.append(
+                    {"role": "tool", "tool_call_id": result.call_id, "content": result.content}
+                )
+            if msg.content:
+                out.append({"role": "user", "content": msg.content})
+    return out
+
+
+class OpenAIProvider(LLMProvider):
+    name = "openai"
+
+    def __init__(self, client: AsyncOpenAI | None = None):
+        self._client = client or AsyncOpenAI()
+
+    async def complete(
+        self,
+        *,
+        model: str,
+        system: str | None,
+        messages: list[ChatMessage],
+        tools: list[ToolDef],
+        max_tokens: int = 1024,
+    ) -> LLMTurn:
+        kwargs: dict[str, Any] = {}
+        if tools:
+            kwargs["tools"] = [
+                {
+                    "type": "function",
+                    "function": {
+                        "name": t.name,
+                        "description": t.description,
+                        "parameters": t.input_schema,
+                    },
+                }
+                for t in tools
+            ]
+        response = await self._client.chat.completions.create(
+            model=model,
+            max_completion_tokens=max_tokens,
+            messages=_to_openai_messages(system, messages),
+            **kwargs,
+        )
+
+        choice = response.choices[0]
+        tool_calls = [
+            ToolCall(
+                id=call.id,
+                name=call.function.name,
+                arguments=json.loads(call.function.arguments or "{}"),
+            )
+            for call in (choice.message.tool_calls or [])
+            if call.type == "function"
+        ]
+        usage = response.usage
+        return LLMTurn(
+            text=choice.message.content or None,
+            tool_calls=tool_calls,
+            usage=Usage(
+                input_tokens=usage.prompt_tokens if usage else 0,
+                output_tokens=usage.completion_tokens if usage else 0,
+            ),
+            stop_reason=choice.finish_reason,
+        )

whetkit/llm/registry.py

@@ -0,0 +1,37 @@
+"""Model-string parsing and provider lookup.
+
+Models are addressed as ``provider:model_id`` (e.g. ``anthropic:claude-sonnet-5``,
+``openai:gpt-5.2``). A bare model id defaults to the Anthropic provider.
+"""
+
+from functools import cache
+
+from whetkit.llm.base import LLMProvider
+
+DEFAULT_PROVIDER = "anthropic"
+
+
+def parse_model(model: str) -> tuple[str, str]:
+    """Split ``provider:model_id`` -> (provider_name, model_id)."""
+    if ":" in model:
+        provider_name, model_id = model.split(":", 1)
+    else:
+        provider_name, model_id = DEFAULT_PROVIDER, model
+    if not model_id:
+        raise ValueError(f"invalid model string {model!r}")
+    return provider_name, model_id
+
+
+@cache
+def get_provider(provider_name: str) -> LLMProvider:
+    """Instantiate (and cache) a provider by name. Imports lazily so one
+    missing SDK/key never blocks the other provider."""
+    if provider_name == "anthropic":
+        from whetkit.llm.anthropic_provider import AnthropicProvider
+
+        return AnthropicProvider()
+    if provider_name == "openai":
+        from whetkit.llm.openai_provider import OpenAIProvider
+
+        return OpenAIProvider()
+    raise ValueError(f"unknown provider {provider_name!r} (expected 'anthropic' or 'openai')")

whetkit/mcp/__init__.py

@@ -0,0 +1,17 @@
+"""MCP connectivity: transports, client, and tool introspection."""
+
+from whetkit.mcp.client import MCPClient
+from whetkit.mcp.introspect import ServerInventory, ToolInfo, inspect_server
+from whetkit.mcp.transport import HttpMode, HttpSpec, ServerSpec, StdioSpec, resolve_server_spec
+
+__all__ = [
+    "HttpMode",
+    "HttpSpec",
+    "MCPClient",
+    "ServerInventory",
+    "ServerSpec",
+    "StdioSpec",
+    "ToolInfo",
+    "inspect_server",
+    "resolve_server_spec",
+]