mcplint-cli 0.1.0__py3-none-any.whl

mcplint/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

mcplint/__init__.py

@@ -0,0 +1,3 @@
+from mcplint.__about__ import __version__
+
+__all__ = ["__version__"]

mcplint/benchmark/dataset.py

@@ -0,0 +1,34 @@
+"""Load and validate a benchmark dataset YAML file."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+from pydantic import ValidationError
+
+from mcplint.models.benchmark import BenchmarkDataset
+
+
+class DatasetError(Exception):
+    """Raised when a benchmark dataset file is missing or fails validation."""
+
+
+def load_dataset(path: Path) -> BenchmarkDataset:
+    if not path.exists():
+        raise DatasetError(f"Benchmark dataset not found: {path}")
+
+    raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    if not isinstance(raw, dict):
+        raise DatasetError(
+            f"{path}: expected a YAML mapping at the top level, got {type(raw).__name__}"
+        )
+
+    try:
+        return BenchmarkDataset.model_validate(raw)
+    except ValidationError as exc:
+        messages = "\n".join(
+            f"  - {'.'.join(str(loc) for loc in error['loc'])}: {error['msg']}"
+            for error in exc.errors()
+        )
+        raise DatasetError(f"{path}: invalid benchmark dataset:\n{messages}") from exc

mcplint/benchmark/providers/anthropic_provider.py

@@ -0,0 +1,92 @@
+"""Anthropic ToolCallingProvider adapter.
+
+Requires the `anthropic` extra: `pip install "mcplint[anthropic]"`. The
+`anthropic` package is imported lazily in `__init__` so importing this
+module (or the rest of mcplint) never requires it to be installed.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import TYPE_CHECKING, Any
+
+from mcplint.benchmark.providers.base import ProviderResult
+from mcplint.models.contracts import ToolContract
+
+if TYPE_CHECKING:
+    import anthropic
+
+# Illustrative only — not authoritative pricing. USD per 1M tokens.
+_PRICING_PER_MILLION_TOKENS: dict[str, tuple[float, float]] = {
+    "claude-sonnet-5": (3.0, 15.0),
+    "claude-opus-5": (15.0, 75.0),
+    "claude-haiku-4-5-20251001": (0.8, 4.0),
+}
+
+
+class AnthropicProvider:
+    def __init__(self, model: str, *, api_key: str | None = None, max_tokens: int = 1024) -> None:
+        try:
+            import anthropic as anthropic_module
+        except ImportError as exc:
+            raise ImportError(
+                "The 'anthropic' package is required for --provider anthropic. "
+                'Install it with: pip install "mcplint[anthropic]"'
+            ) from exc
+
+        self.name = "anthropic"
+        self.model = model
+        self.max_tokens = max_tokens
+        self.client: anthropic.AsyncAnthropic = anthropic_module.AsyncAnthropic(api_key=api_key)
+
+    async def run(self, prompt: str, tools: list[ToolContract]) -> ProviderResult:
+        anthropic_tools: list[dict[str, Any]] = [
+            {
+                "name": tool.name,
+                "description": tool.description or "",
+                "input_schema": tool.input_schema,
+            }
+            for tool in tools
+        ]
+
+        start = time.monotonic()
+        try:
+            message = await self.client.messages.create(
+                model=self.model,
+                max_tokens=self.max_tokens,
+                messages=[{"role": "user", "content": prompt}],
+                tools=anthropic_tools,  # type: ignore[arg-type]
+            )
+        except Exception as exc:  # noqa: BLE001 - surfaced as a scored trial error, not a crash
+            latency_ms = (time.monotonic() - start) * 1000
+            return ProviderResult(tool=None, latency_ms=latency_ms, error=str(exc))
+
+        latency_ms = (time.monotonic() - start) * 1000
+        tool_use = next((block for block in message.content if block.type == "tool_use"), None)
+        input_tokens = message.usage.input_tokens
+        output_tokens = message.usage.output_tokens
+
+        if tool_use is None:
+            return ProviderResult(
+                tool=None,
+                latency_ms=latency_ms,
+                input_tokens=input_tokens,
+                output_tokens=output_tokens,
+                estimated_cost=self._estimate_cost(input_tokens, output_tokens),
+            )
+
+        return ProviderResult(
+            tool=tool_use.name,
+            arguments=dict(tool_use.input),
+            latency_ms=latency_ms,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+            estimated_cost=self._estimate_cost(input_tokens, output_tokens),
+        )
+
+    def _estimate_cost(self, input_tokens: int, output_tokens: int) -> float | None:
+        pricing = _PRICING_PER_MILLION_TOKENS.get(self.model)
+        if pricing is None:
+            return None
+        input_price, output_price = pricing
+        return (input_tokens * input_price + output_tokens * output_price) / 1_000_000

mcplint/benchmark/providers/base.py

@@ -0,0 +1,26 @@
+"""Provider interface every benchmark backend (fake, Anthropic, OpenAI) implements."""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+from pydantic import BaseModel, Field
+
+from mcplint.models.contracts import ToolContract
+
+
+class ProviderResult(BaseModel):
+    tool: str | None
+    arguments: dict[str, object] = Field(default_factory=dict)
+    latency_ms: float = 0.0
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    estimated_cost: float | None = None
+    error: str | None = None
+
+
+class ToolCallingProvider(Protocol):
+    name: str
+    model: str
+
+    async def run(self, prompt: str, tools: list[ToolContract]) -> ProviderResult: ...

mcplint/benchmark/providers/factory.py

@@ -0,0 +1,51 @@
+"""Resolves a --provider/--model CLI selection to a ToolCallingProvider instance.
+
+"fake" and "anthropic" are implemented. "openai" is a typed stub — see
+`openai_provider.py` — that raises NotImplementedError only when actually
+invoked, per the spec's "don't let OpenAI delay Anthropic" guidance.
+"""
+
+from __future__ import annotations
+
+from mcplint.benchmark.providers.base import ProviderResult, ToolCallingProvider
+from mcplint.benchmark.providers.fake import FakeProvider
+from mcplint.models.contracts import ToolContract
+
+SUPPORTED_PROVIDERS = ("fake", "anthropic", "openai")
+
+
+class ProviderNotAvailableError(Exception):
+    """Raised when a requested provider isn't implemented yet or misconfigured."""
+
+
+def _echo_first_tool(prompt: str, tools: list[ToolContract]) -> ProviderResult:
+    if not tools:
+        return ProviderResult(tool=None, error="no tools available")
+    return ProviderResult(tool=tools[0].name, arguments={})
+
+
+def create_provider(provider: str, model: str | None) -> ToolCallingProvider:
+    if provider == "fake":
+        return FakeProvider(model=model or "fake-echo-model", responder=_echo_first_tool)
+
+    if provider == "anthropic":
+        if model is None:
+            raise ProviderNotAvailableError(
+                "--model is required for --provider anthropic (e.g. claude-sonnet-5)."
+            )
+        try:
+            from mcplint.benchmark.providers.anthropic_provider import AnthropicProvider
+        except ImportError as exc:
+            raise ProviderNotAvailableError(str(exc)) from exc
+        return AnthropicProvider(model=model)
+
+    if provider == "openai":
+        from mcplint.benchmark.providers.openai_provider import OpenAIProvider
+
+        if model is None:
+            raise ProviderNotAvailableError("--model is required for --provider openai.")
+        return OpenAIProvider(model=model)
+
+    raise ProviderNotAvailableError(
+        f"Unknown provider '{provider}'. Supported: {', '.join(SUPPORTED_PROVIDERS)}"
+    )

mcplint/benchmark/providers/fake.py

@@ -0,0 +1,26 @@
+"""A deterministic, no-network provider used by scorer tests and offline demos."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from mcplint.benchmark.providers.base import ProviderResult
+from mcplint.models.contracts import ToolContract
+
+
+class FakeProvider:
+    """Wraps a plain Python function so tests never need real API calls."""
+
+    def __init__(
+        self,
+        model: str,
+        responder: Callable[[str, list[ToolContract]], ProviderResult],
+        *,
+        name: str = "fake",
+    ) -> None:
+        self.name = name
+        self.model = model
+        self._responder = responder
+
+    async def run(self, prompt: str, tools: list[ToolContract]) -> ProviderResult:
+        return self._responder(prompt, tools)

mcplint/benchmark/providers/openai_provider.py

@@ -0,0 +1,25 @@
+"""OpenAI ToolCallingProvider adapter.
+
+TODO(Phase 5 follow-up): implement against the OpenAI Responses/Chat
+Completions tool-calling API, mirroring AnthropicProvider. Structured as a
+separate adapter per spec so it does not block the Anthropic implementation.
+Requires the `openai` extra: `pip install "mcplint[openai]"`.
+"""
+
+from __future__ import annotations
+
+from mcplint.benchmark.providers.base import ProviderResult
+from mcplint.models.contracts import ToolContract
+
+
+class OpenAIProvider:
+    def __init__(self, model: str, *, api_key: str | None = None) -> None:
+        self.name = "openai"
+        self.model = model
+        self.api_key = api_key
+
+    async def run(self, prompt: str, tools: list[ToolContract]) -> ProviderResult:
+        raise NotImplementedError(
+            "OpenAI provider is not implemented yet. Use --provider anthropic or "
+            "--provider fake. Tracked in IMPLEMENTATION_STATUS.md."
+        )

mcplint/benchmark/runner.py

@@ -0,0 +1,43 @@
+"""Runs a benchmark dataset against a provider and a fixed set of tools."""
+
+from __future__ import annotations
+
+from mcplint.benchmark.providers.base import ToolCallingProvider
+from mcplint.benchmark.scorer import (
+    BENCHMARK_RESULT_SCHEMA_VERSION,
+    aggregate_metrics,
+    make_trial,
+    to_actual_tool_call,
+)
+from mcplint.models.benchmark import BenchmarkDataset, BenchmarkResult, BenchmarkTrial
+from mcplint.models.common import ArtifactMetadata
+from mcplint.models.contracts import ToolContract
+
+
+async def run_benchmark(
+    dataset: BenchmarkDataset,
+    tools: list[ToolContract],
+    provider: ToolCallingProvider,
+    *,
+    runs: int = 3,
+) -> BenchmarkResult:
+    trials: list[BenchmarkTrial] = []
+    expected_by_case = {case.id: case.expected for case in dataset.cases}
+
+    for case in dataset.cases:
+        for trial_index in range(runs):
+            result = await provider.run(case.prompt, tools)
+            actual = to_actual_tool_call(result, tools)
+            trials.append(make_trial(case.id, trial_index, case.expected, actual))
+
+    metrics = aggregate_metrics(trials, expected_by_case)
+
+    return BenchmarkResult(
+        metadata=ArtifactMetadata.create(schema_version=BENCHMARK_RESULT_SCHEMA_VERSION),
+        dataset_name=dataset.name,
+        provider=provider.name,
+        model=provider.model,
+        runs_per_case=runs,
+        trials=trials,
+        **metrics,
+    )

mcplint/benchmark/scorer.py

@@ -0,0 +1,147 @@
+"""Deterministic scoring of benchmark trials. No LLM judge — every metric here
+is computed from exact tool-name/argument comparison and JSON Schema validation.
+"""
+
+from __future__ import annotations
+
+import jsonschema
+
+from mcplint.benchmark.providers.base import ProviderResult
+from mcplint.models.benchmark import ActualToolCall, BenchmarkTrial, ExpectedToolCall
+from mcplint.models.contracts import ToolContract
+
+BENCHMARK_RESULT_SCHEMA_VERSION = "1.0"
+
+
+def validate_arguments(
+    tool_name: str | None, arguments: dict[str, object], tools: list[ToolContract]
+) -> bool:
+    if tool_name is None:
+        return False
+    tool = next((t for t in tools if t.name == tool_name), None)
+    if tool is None:
+        return False
+    try:
+        jsonschema.validate(instance=arguments, schema=tool.input_schema)
+    except jsonschema.ValidationError:
+        return False
+    return True
+
+
+def to_actual_tool_call(result: ProviderResult, tools: list[ToolContract]) -> ActualToolCall:
+    return ActualToolCall(
+        tool=result.tool,
+        arguments=result.arguments,
+        valid_arguments=validate_arguments(result.tool, result.arguments, tools),
+        error=result.error,
+        latency_ms=result.latency_ms,
+        input_tokens=result.input_tokens,
+        output_tokens=result.output_tokens,
+        estimated_cost=result.estimated_cost,
+    )
+
+
+def score_trial(expected: ExpectedToolCall, actual: ActualToolCall) -> tuple[bool, list[str]]:
+    reasons: list[str] = []
+
+    if actual.tool is None:
+        reasons.append("no tool was selected")
+    elif actual.tool != expected.tool:
+        reasons.append(f"expected tool '{expected.tool}', got '{actual.tool}'")
+
+    if actual.tool is not None and actual.tool in expected.forbidden_tools:
+        reasons.append(f"invoked forbidden tool '{actual.tool}'")
+
+    if not actual.valid_arguments:
+        reasons.append("arguments do not validate against the tool's input schema")
+
+    for key, expected_value in expected.all_expected_arguments().items():
+        actual_value = actual.arguments.get(key)
+        if actual_value != expected_value:
+            reasons.append(f"argument '{key}': expected {expected_value!r}, got {actual_value!r}")
+
+    return (not reasons, reasons)
+
+
+def make_trial(
+    case_id: str, trial_index: int, expected: ExpectedToolCall, actual: ActualToolCall
+) -> BenchmarkTrial:
+    passed, reasons = score_trial(expected, actual)
+    return BenchmarkTrial(
+        case_id=case_id,
+        trial_index=trial_index,
+        actual=actual,
+        passed=passed,
+        failure_reasons=reasons,
+    )
+
+
+def _percentile(values: list[float], percentile: float) -> float:
+    if not values:
+        return 0.0
+    ordered = sorted(values)
+    index = min(len(ordered) - 1, int(round(percentile * (len(ordered) - 1))))
+    return ordered[index]
+
+
+def aggregate_metrics(
+    trials: list[BenchmarkTrial],
+    expected_by_case: dict[str, ExpectedToolCall],
+) -> dict[str, object]:
+    total = len(trials)
+    exact_matches = sum(1 for t in trials if t.actual.tool == expected_by_case[t.case_id].tool)
+    valid_args = sum(1 for t in trials if t.actual.valid_arguments)
+    forbidden = sum(
+        1 for t in trials if t.actual.tool in expected_by_case[t.case_id].forbidden_tools
+    )
+    no_tool = sum(1 for t in trials if t.actual.tool is None)
+
+    required_correct = 0
+    required_total = 0
+    for trial in trials:
+        expected = expected_by_case[trial.case_id]
+        if trial.actual.tool != expected.tool:
+            continue
+        required_total += 1
+        expected_args = expected.all_expected_arguments()
+        if all(trial.actual.arguments.get(k) == v for k, v in expected_args.items()):
+            required_correct += 1
+
+    latencies = [t.actual.latency_ms for t in trials]
+    costs = [t.actual.estimated_cost for t in trials if t.actual.estimated_cost is not None]
+
+    per_case_pass_rate: dict[str, float] = {}
+    stability: dict[str, float] = {}
+    by_case: dict[str, list[BenchmarkTrial]] = {}
+    for trial in trials:
+        by_case.setdefault(trial.case_id, []).append(trial)
+    for case_id, case_trials in by_case.items():
+        passed_count = sum(1 for t in case_trials if t.passed)
+        per_case_pass_rate[case_id] = passed_count / len(case_trials)
+        majority = max(passed_count, len(case_trials) - passed_count)
+        stability[case_id] = majority / len(case_trials)
+
+    return {
+        "exact_tool_selection_accuracy": exact_matches / total if total else 0.0,
+        "valid_argument_rate": valid_args / total if total else 0.0,
+        "required_argument_accuracy": (
+            required_correct / required_total if required_total else 0.0
+        ),
+        "forbidden_tool_invocation_rate": forbidden / total if total else 0.0,
+        "no_tool_rate": no_tool / total if total else 0.0,
+        "mean_latency_ms": sum(latencies) / total if total else 0.0,
+        "p95_latency_ms": _percentile(latencies, 0.95),
+        "total_estimated_cost": sum(costs) if costs else None,
+        "per_case_pass_rate": per_case_pass_rate,
+        "stability": stability,
+    }
+
+
+__all__ = [
+    "BENCHMARK_RESULT_SCHEMA_VERSION",
+    "aggregate_metrics",
+    "make_trial",
+    "score_trial",
+    "to_actual_tool_call",
+    "validate_arguments",
+]

mcplint/cli/commands/benchmark_cmd.py

@@ -0,0 +1,88 @@
+"""`mcplint benchmark` — run a benchmark dataset against a live MCP server."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import anyio
+import typer
+from rich.console import Console
+
+from mcplint.benchmark.dataset import DatasetError, load_dataset
+from mcplint.benchmark.providers.factory import ProviderNotAvailableError, create_provider
+from mcplint.benchmark.runner import run_benchmark
+from mcplint.mcp_client.persistence import load_snapshot
+from mcplint.mcp_client.session import collect_stdio_snapshot
+from mcplint.mcp_client.stdio import parse_command
+from mcplint.models.benchmark import BenchmarkResult
+from mcplint.reporters.benchmark_terminal import render_benchmark_terminal
+
+console = Console()
+error_console = Console(stderr=True)
+
+
+def benchmark_command(
+    dataset_path: Annotated[Path, typer.Argument(help="Path to the benchmark dataset YAML.")],
+    server: Annotated[
+        str | None, typer.Option("--server", help="Command line to launch the MCP server.")
+    ] = None,
+    snapshot: Annotated[
+        Path | None, typer.Option("--snapshot", help="Path to a saved snapshot JSON file.")
+    ] = None,
+    provider: Annotated[
+        str, typer.Option("--provider", help="Benchmark provider: fake, anthropic, or openai.")
+    ] = "fake",
+    model: Annotated[
+        str | None, typer.Option("--model", help="Model name for the provider.")
+    ] = None,
+    runs: Annotated[int, typer.Option("--runs", help="Trials per case.")] = 3,
+    format: Annotated[str, typer.Option("--format", help="Output format.")] = "terminal",
+    output: Annotated[
+        Path | None, typer.Option("--output", help="Path to write the BenchmarkResult JSON to.")
+    ] = None,
+) -> None:
+    if (server is None) == (snapshot is None):
+        error_console.print(
+            "[bold red]Exactly one of --server or --snapshot is required.[/bold red]"
+        )
+        raise typer.Exit(code=2)
+    if format not in ("terminal", "json"):
+        error_console.print(f"[bold red]Unknown format: {format}[/bold red]")
+        raise typer.Exit(code=2)
+
+    try:
+        dataset = load_dataset(dataset_path)
+    except DatasetError as exc:
+        error_console.print(f"[bold red]{exc}[/bold red]")
+        raise typer.Exit(code=2) from exc
+
+    try:
+        chosen_provider = create_provider(provider, model)
+    except ProviderNotAvailableError as exc:
+        error_console.print(f"[bold red]{exc}[/bold red]")
+        raise typer.Exit(code=2) from exc
+
+    try:
+        if snapshot is not None:
+            server_snapshot = load_snapshot(snapshot)
+        else:
+            assert server is not None
+            command, args = parse_command(server)
+            server_snapshot = anyio.run(collect_stdio_snapshot, command, args)
+    except Exception as exc:  # noqa: BLE001 - surfaced as a CI-friendly CLI error
+        error_console.print(f"[bold red]Failed to load server contract:[/bold red] {exc}")
+        raise typer.Exit(code=1) from exc
+
+    async def _run() -> BenchmarkResult:
+        return await run_benchmark(dataset, server_snapshot.tools, chosen_provider, runs=runs)
+
+    result = anyio.run(_run)
+
+    if format == "json":
+        console.print(result.model_dump_json(indent=2))
+    else:
+        console.print(render_benchmark_terminal(result))
+
+    if output is not None:
+        output.write_text(result.model_dump_json(indent=2) + "\n", encoding="utf-8")