mcplint-cli 0.1.0__py3-none-any.whl

mcplint/cli/commands/compare_cmd.py

@@ -0,0 +1,139 @@
+"""`mcplint compare` — diff two snapshots and optionally re-run a benchmark against both."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import anyio
+import typer
+from rich.console import Console
+
+from mcplint.benchmark.dataset import DatasetError, load_dataset
+from mcplint.benchmark.providers.factory import ProviderNotAvailableError, create_provider
+from mcplint.benchmark.runner import run_benchmark
+from mcplint.compare.differ import (
+    diff_ambiguity,
+    diff_benchmarks,
+    diff_findings,
+    diff_tool_contracts,
+    diff_tool_names,
+)
+from mcplint.core.engine import lint_snapshot
+from mcplint.core.registry import RuleRegistry
+from mcplint.mcp_client.persistence import load_snapshot
+from mcplint.models.benchmark import BenchmarkResult
+from mcplint.models.common import ArtifactMetadata
+from mcplint.models.comparison import ComparisonReport
+from mcplint.reporters.comparison_terminal import render_comparison_terminal
+
+console = Console()
+error_console = Console(stderr=True)
+
+COMPARISON_SCHEMA_VERSION = "1.0"
+
+
+def compare_command(
+    baseline: Annotated[Path, typer.Option("--baseline", help="Baseline snapshot JSON path.")],
+    candidate: Annotated[Path, typer.Option("--candidate", help="Candidate snapshot JSON path.")],
+    dataset_path: Annotated[
+        Path | None,
+        typer.Option("--dataset", help="Benchmark dataset YAML to re-run against both."),
+    ] = None,
+    provider: Annotated[str, typer.Option("--provider", help="Benchmark provider.")] = "fake",
+    model: Annotated[
+        str | None, typer.Option("--model", help="Model name for the provider.")
+    ] = None,
+    runs: Annotated[int, typer.Option("--runs", help="Trials per case.")] = 3,
+    min_accuracy_delta: Annotated[
+        float | None,
+        typer.Option("--min-accuracy-delta", help="Fail if accuracy delta falls below this."),
+    ] = None,
+    format: Annotated[str, typer.Option("--format", help="Output format.")] = "terminal",
+    output: Annotated[
+        Path | None, typer.Option("--output", help="Path to write the ComparisonReport JSON to.")
+    ] = None,
+) -> None:
+    if format not in ("terminal", "json"):
+        error_console.print(f"[bold red]Unknown format: {format}[/bold red]")
+        raise typer.Exit(code=2)
+
+    try:
+        baseline_snapshot = load_snapshot(baseline)
+        candidate_snapshot = load_snapshot(candidate)
+    except FileNotFoundError as exc:
+        error_console.print(f"[bold red]{exc}[/bold red]")
+        raise typer.Exit(code=2) from exc
+
+    registry = RuleRegistry.with_builtin_rules()
+    baseline_report = lint_snapshot(baseline_snapshot, registry)
+    candidate_report = lint_snapshot(candidate_snapshot, registry)
+
+    added_tools, removed_tools = diff_tool_names(baseline_snapshot, candidate_snapshot)
+    schema_changes, description_changes = diff_tool_contracts(baseline_snapshot, candidate_snapshot)
+    new_findings, resolved_findings = diff_findings(baseline_report, candidate_report)
+    ambiguity_changes = diff_ambiguity(baseline_snapshot, candidate_snapshot)
+
+    report = ComparisonReport(
+        metadata=ArtifactMetadata.create(schema_version=COMPARISON_SCHEMA_VERSION),
+        baseline_server_name=baseline_snapshot.server_name,
+        candidate_server_name=candidate_snapshot.server_name,
+        added_tools=added_tools,
+        removed_tools=removed_tools,
+        schema_changes=schema_changes,
+        description_changes=description_changes,
+        new_findings=new_findings,
+        resolved_findings=resolved_findings,
+        ambiguity_score_changes=ambiguity_changes,
+        min_accuracy_delta_threshold=min_accuracy_delta,
+    )
+
+    if dataset_path is not None:
+        try:
+            dataset = load_dataset(dataset_path)
+        except DatasetError as exc:
+            error_console.print(f"[bold red]{exc}[/bold red]")
+            raise typer.Exit(code=2) from exc
+        try:
+            chosen_provider = create_provider(provider, model)
+        except ProviderNotAvailableError as exc:
+            error_console.print(f"[bold red]{exc}[/bold red]")
+            raise typer.Exit(code=2) from exc
+
+        async def _run_both() -> tuple[BenchmarkResult, BenchmarkResult]:
+            baseline_result = await run_benchmark(
+                dataset, baseline_snapshot.tools, chosen_provider, runs=runs
+            )
+            candidate_result = await run_benchmark(
+                dataset, candidate_snapshot.tools, chosen_provider, runs=runs
+            )
+            return baseline_result, candidate_result
+
+        baseline_result, candidate_result = anyio.run(_run_both)
+        deltas = diff_benchmarks(baseline_result, candidate_result)
+
+        report = report.model_copy(
+            update={
+                "benchmark_dataset_name": dataset.name,
+                "baseline_accuracy": baseline_result.exact_tool_selection_accuracy,
+                "candidate_accuracy": candidate_result.exact_tool_selection_accuracy,
+                **deltas,
+            }
+        )
+        if min_accuracy_delta is not None:
+            accuracy_delta = deltas["benchmark_accuracy_delta"]
+            assert isinstance(accuracy_delta, float)
+            report = report.model_copy(
+                update={"passes_ci_threshold": accuracy_delta >= min_accuracy_delta}
+            )
+
+    if format == "json":
+        console.print(report.model_dump_json(indent=2))
+    else:
+        console.print(render_comparison_terminal(report))
+
+    if output is not None:
+        output.write_text(report.model_dump_json(indent=2) + "\n", encoding="utf-8")
+
+    if report.passes_ci_threshold is False:
+        raise typer.Exit(code=1)

mcplint/cli/commands/fix_cmd.py

@@ -0,0 +1,60 @@
+"""`mcplint fix` — propose deterministic rewrite suggestions as a Markdown patch report.
+
+Never writes to the MCP server's source files. Only ever writes the patch
+report itself, and only when --output is given.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from mcplint.core.engine import lint_snapshot
+from mcplint.core.registry import RuleRegistry
+from mcplint.fix.suggest import build_suggestions
+from mcplint.mcp_client.persistence import load_snapshot
+from mcplint.reporters.fix_markdown import render_fix_markdown
+
+console = Console()
+error_console = Console(stderr=True)
+
+
+def fix_command(
+    snapshot: Annotated[
+        Path, typer.Option("--snapshot", help="Path to a saved snapshot JSON file.")
+    ],
+    llm_provider: Annotated[
+        str | None,
+        typer.Option(
+            "--llm-provider",
+            help="Enable LLM-assisted rewriting via this provider (not yet available).",
+        ),
+    ] = None,
+    output: Annotated[
+        Path | None, typer.Option("--output", help="Path to write the Markdown patch report to.")
+    ] = None,
+) -> None:
+    if llm_provider is not None:
+        error_console.print(
+            "[bold red]LLM-assisted rewriting is not implemented yet.[/bold red] "
+            "Omit --llm-provider to use deterministic suggestions only."
+        )
+        raise typer.Exit(code=2)
+
+    try:
+        server_snapshot = load_snapshot(snapshot)
+    except FileNotFoundError as exc:
+        error_console.print(f"[bold red]{exc}[/bold red]")
+        raise typer.Exit(code=2) from exc
+
+    report = lint_snapshot(server_snapshot, RuleRegistry.with_builtin_rules())
+    suggestions = build_suggestions(server_snapshot, report)
+    markdown = render_fix_markdown(server_snapshot.server_name, suggestions)
+
+    console.print(markdown)
+
+    if output is not None:
+        output.write_text(markdown, encoding="utf-8")

mcplint/cli/commands/inspect_cmd.py

@@ -0,0 +1,44 @@
+"""`mcplint inspect` — connect to an MCP server and print its tool contracts."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+import anyio
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from mcplint.mcp_client.session import collect_stdio_snapshot
+from mcplint.mcp_client.stdio import parse_command
+
+console = Console()
+error_console = Console(stderr=True)
+
+
+def inspect_command(
+    server: Annotated[str, typer.Option("--server", help="Command line to launch the MCP server.")],
+) -> None:
+    command, args = parse_command(server)
+
+    try:
+        snapshot = anyio.run(collect_stdio_snapshot, command, args)
+    except Exception as exc:  # noqa: BLE001 - surfaced as a CI-friendly CLI error
+        error_console.print(f"[bold red]Failed to inspect server:[/bold red] {exc}")
+        raise typer.Exit(code=1) from exc
+
+    table = Table(title=f"Tools on {snapshot.server_name}")
+    table.add_column("Name")
+    table.add_column("Description")
+    table.add_column("Params", justify="right")
+    table.add_column("Destructive", justify="center")
+
+    for tool in snapshot.tools:
+        table.add_row(
+            tool.name,
+            tool.description or "[dim]—[/dim]",
+            str(len(tool.parameters)),
+            "yes" if tool.annotations.destructive_hint else "",
+        )
+
+    console.print(table)

mcplint/cli/commands/rules_cmd.py

@@ -0,0 +1,30 @@
+"""`mcplint rules` — list every built-in rule."""
+
+from __future__ import annotations
+
+from rich.console import Console
+from rich.table import Table
+
+from mcplint.core.registry import RuleRegistry
+
+console = Console(width=160)
+
+
+def rules_command() -> None:
+    registry = RuleRegistry.with_builtin_rules()
+    table = Table(title="MCPLint rule catalogue")
+    table.add_column("ID")
+    table.add_column("Title")
+    table.add_column("Default severity")
+    table.add_column("Tags")
+
+    for rule in registry.all():
+        metadata = rule.metadata()
+        table.add_row(
+            metadata.id,
+            metadata.title,
+            metadata.default_severity.value,
+            ", ".join(metadata.tags),
+        )
+
+    console.print(table)

mcplint/cli/commands/scan_cmd.py

@@ -0,0 +1,113 @@
+"""`mcplint scan` — lint a live MCP server or a saved snapshot."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import anyio
+import typer
+from rich.console import Console
+
+from mcplint.config.loader import ConfigError, load_config
+from mcplint.core.engine import lint_snapshot
+from mcplint.core.registry import RuleRegistry
+from mcplint.mcp_client.persistence import load_snapshot
+from mcplint.mcp_client.session import collect_stdio_snapshot
+from mcplint.mcp_client.stdio import parse_command
+from mcplint.models.findings import LintReport, Severity
+from mcplint.reporters.html import render_html_report
+from mcplint.reporters.json_reporter import render_json
+from mcplint.reporters.sarif import render_sarif
+from mcplint.reporters.terminal import render_terminal
+
+_FORMATS = ("terminal", "json", "sarif", "html")
+
+console = Console()
+error_console = Console(stderr=True)
+
+_SEVERITY_RANK = {Severity.INFO: 0, Severity.WARNING: 1, Severity.ERROR: 2}
+_FAIL_ON_THRESHOLD = {
+    "error": _SEVERITY_RANK[Severity.ERROR],
+    "warning": _SEVERITY_RANK[Severity.WARNING],
+}
+
+
+def _should_fail(report: LintReport, fail_on: str) -> bool:
+    if fail_on == "never":
+        return False
+    threshold = _FAIL_ON_THRESHOLD[fail_on]
+    return any(_SEVERITY_RANK[finding.severity] >= threshold for finding in report.findings)
+
+
+def scan_command(
+    server: Annotated[
+        str | None, typer.Option("--server", help="Command line to launch the MCP server.")
+    ] = None,
+    snapshot: Annotated[
+        Path | None, typer.Option("--snapshot", help="Path to a saved snapshot JSON file.")
+    ] = None,
+    format: Annotated[str, typer.Option("--format", help="Output format.")] = "terminal",
+    fail_on: Annotated[
+        str, typer.Option("--fail-on", help="Minimum severity that fails the command.")
+    ] = "error",
+    config: Annotated[Path, typer.Option("--config", help="Path to mcplint.yaml.")] = Path(
+        "mcplint.yaml"
+    ),
+    output: Annotated[
+        Path | None, typer.Option("--output", help="Path to also write the report to.")
+    ] = None,
+) -> None:
+    if (server is None) == (snapshot is None):
+        error_console.print(
+            "[bold red]Exactly one of --server or --snapshot is required.[/bold red]"
+        )
+        raise typer.Exit(code=2)
+    if format not in _FORMATS:
+        error_console.print(
+            f"[bold red]Unknown format: {format}. Supported: {', '.join(_FORMATS)}[/bold red]"
+        )
+        raise typer.Exit(code=2)
+    if fail_on not in ("error", "warning", "never"):
+        error_console.print(f"[bold red]Unknown --fail-on: {fail_on}[/bold red]")
+        raise typer.Exit(code=2)
+
+    try:
+        mcplint_config = load_config(config)
+    except ConfigError as exc:
+        error_console.print(f"[bold red]{exc}[/bold red]")
+        raise typer.Exit(code=2) from exc
+
+    try:
+        if snapshot is not None:
+            server_snapshot = load_snapshot(snapshot)
+        else:
+            assert server is not None
+            command, args = parse_command(server)
+            server_snapshot = anyio.run(collect_stdio_snapshot, command, args)
+    except Exception as exc:  # noqa: BLE001 - surfaced as a CI-friendly CLI error
+        error_console.print(f"[bold red]Failed to load server contract:[/bold red] {exc}")
+        raise typer.Exit(code=1) from exc
+
+    report = lint_snapshot(
+        server_snapshot, RuleRegistry.with_builtin_rules(mcplint_config), mcplint_config
+    )
+
+    if format == "json":
+        rendered = render_json(report)
+        console.print(rendered)
+    elif format == "sarif":
+        rendered = render_sarif(report)
+        console.print(rendered)
+    elif format == "html":
+        rendered = render_html_report(server_snapshot, report)
+        console.print(f"[green]HTML report generated ({len(rendered)} bytes).[/green]")
+    else:
+        rendered = render_terminal(report)
+        console.print(rendered)
+
+    if output is not None:
+        output.write_text(rendered, encoding="utf-8")
+
+    if _should_fail(report, fail_on):
+        raise typer.Exit(code=1)

mcplint/cli/commands/snapshot_cmd.py

@@ -0,0 +1,33 @@
+"""`mcplint snapshot` — connect to an MCP server and persist its contract as JSON."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import anyio
+import typer
+from rich.console import Console
+
+from mcplint.mcp_client.persistence import save_snapshot
+from mcplint.mcp_client.session import collect_stdio_snapshot
+from mcplint.mcp_client.stdio import parse_command
+
+console = Console()
+error_console = Console(stderr=True)
+
+
+def snapshot_command(
+    server: Annotated[str, typer.Option("--server", help="Command line to launch the MCP server.")],
+    output: Annotated[Path, typer.Option("--output", help="Path to write the snapshot JSON to.")],
+) -> None:
+    command, args = parse_command(server)
+
+    try:
+        snapshot = anyio.run(collect_stdio_snapshot, command, args)
+    except Exception as exc:  # noqa: BLE001 - surfaced as a CI-friendly CLI error
+        error_console.print(f"[bold red]Failed to snapshot server:[/bold red] {exc}")
+        raise typer.Exit(code=1) from exc
+
+    save_snapshot(snapshot, output)
+    console.print(f"[green]Wrote snapshot for {snapshot.server_name} to {output}[/green]")

mcplint/cli/main.py

@@ -0,0 +1,49 @@
+"""MCPLint CLI entrypoint."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+import typer
+
+from mcplint.__about__ import __version__
+from mcplint.cli.commands.benchmark_cmd import benchmark_command
+from mcplint.cli.commands.compare_cmd import compare_command
+from mcplint.cli.commands.fix_cmd import fix_command
+from mcplint.cli.commands.inspect_cmd import inspect_command
+from mcplint.cli.commands.rules_cmd import rules_command
+from mcplint.cli.commands.scan_cmd import scan_command
+from mcplint.cli.commands.snapshot_cmd import snapshot_command
+
+app = typer.Typer(name="mcplint", help="ESLint for MCP tool contracts.")
+
+
+def _print_version(value: bool) -> None:
+    if value:
+        typer.echo(f"mcplint {__version__}")
+        raise typer.Exit
+
+
+@app.callback()
+def _callback(
+    version: Annotated[
+        bool,
+        typer.Option(
+            "--version", callback=_print_version, is_eager=True, help="Print the version and exit."
+        ),
+    ] = False,
+) -> None:
+    """ESLint for MCP tool contracts."""
+
+
+app.command("inspect")(inspect_command)
+app.command("snapshot")(snapshot_command)
+app.command("scan")(scan_command)
+app.command("rules")(rules_command)
+app.command("benchmark")(benchmark_command)
+app.command("compare")(compare_command)
+app.command("fix")(fix_command)
+
+
+if __name__ == "__main__":
+    app()

mcplint/compare/differ.py

@@ -0,0 +1,148 @@
+"""Pure diff functions between two snapshots, lint reports, and benchmark results."""
+
+from __future__ import annotations
+
+from mcplint.core.rules.ambiguity import compute_ambiguity
+from mcplint.models.benchmark import BenchmarkResult
+from mcplint.models.comparison import AmbiguityScoreChange, DescriptionChange, SchemaChange
+from mcplint.models.findings import Finding, LintReport
+from mcplint.models.snapshot import MCPServerSnapshot
+
+_FLOAT_EPSILON = 1e-9
+
+
+def diff_tool_names(
+    baseline: MCPServerSnapshot, candidate: MCPServerSnapshot
+) -> tuple[list[str], list[str]]:
+    baseline_names = {tool.name for tool in baseline.tools}
+    candidate_names = {tool.name for tool in candidate.tools}
+    added = sorted(candidate_names - baseline_names)
+    removed = sorted(baseline_names - candidate_names)
+    return added, removed
+
+
+def diff_tool_contracts(
+    baseline: MCPServerSnapshot, candidate: MCPServerSnapshot
+) -> tuple[list[SchemaChange], list[DescriptionChange]]:
+    baseline_names = {tool.name for tool in baseline.tools}
+    candidate_names = {tool.name for tool in candidate.tools}
+    common = sorted(baseline_names & candidate_names)
+
+    schema_changes: list[SchemaChange] = []
+    description_changes: list[DescriptionChange] = []
+
+    for name in common:
+        before_tool = baseline.get_tool(name)
+        after_tool = candidate.get_tool(name)
+        assert before_tool is not None and after_tool is not None
+
+        if before_tool.description != after_tool.description:
+            description_changes.append(
+                DescriptionChange(
+                    tool_name=name, before=before_tool.description, after=after_tool.description
+                )
+            )
+        if before_tool.input_schema != after_tool.input_schema:
+            schema_changes.append(
+                SchemaChange(
+                    tool_name=name,
+                    json_path="$.inputSchema",
+                    before=before_tool.input_schema,
+                    after=after_tool.input_schema,
+                )
+            )
+        if before_tool.output_schema != after_tool.output_schema:
+            schema_changes.append(
+                SchemaChange(
+                    tool_name=name,
+                    json_path="$.outputSchema",
+                    before=before_tool.output_schema,
+                    after=after_tool.output_schema,
+                )
+            )
+
+    return schema_changes, description_changes
+
+
+def _finding_key(finding: Finding) -> tuple[str, str, str, str]:
+    return (
+        finding.rule_id,
+        finding.location.tool_name,
+        finding.location.json_path,
+        finding.message,
+    )
+
+
+def diff_findings(
+    baseline_report: LintReport, candidate_report: LintReport
+) -> tuple[list[Finding], list[Finding]]:
+    baseline_keys = {_finding_key(f) for f in baseline_report.findings}
+    candidate_keys = {_finding_key(f) for f in candidate_report.findings}
+
+    new_findings = [f for f in candidate_report.findings if _finding_key(f) not in baseline_keys]
+    resolved_findings = [
+        f for f in baseline_report.findings if _finding_key(f) not in candidate_keys
+    ]
+    return new_findings, resolved_findings
+
+
+def diff_ambiguity(
+    baseline: MCPServerSnapshot, candidate: MCPServerSnapshot
+) -> list[AmbiguityScoreChange]:
+    baseline_names = {tool.name for tool in baseline.tools}
+    candidate_names = {tool.name for tool in candidate.tools}
+    common = sorted(baseline_names & candidate_names)
+
+    changes: list[AmbiguityScoreChange] = []
+    for index, name_a in enumerate(common):
+        for name_b in common[index + 1 :]:
+            baseline_a, baseline_b = baseline.get_tool(name_a), baseline.get_tool(name_b)
+            candidate_a, candidate_b = candidate.get_tool(name_a), candidate.get_tool(name_b)
+            assert baseline_a is not None and baseline_b is not None
+            assert candidate_a is not None and candidate_b is not None
+
+            before_score = compute_ambiguity(baseline_a, baseline_b).score
+            after_score = compute_ambiguity(candidate_a, candidate_b).score
+            if abs(before_score - after_score) > _FLOAT_EPSILON:
+                changes.append(
+                    AmbiguityScoreChange(
+                        tool_a=name_a, tool_b=name_b, before=before_score, after=after_score
+                    )
+                )
+    return changes
+
+
+def diff_benchmarks(
+    baseline_result: BenchmarkResult, candidate_result: BenchmarkResult
+) -> dict[str, object]:
+    baseline_accuracy = baseline_result.exact_tool_selection_accuracy
+    candidate_accuracy = candidate_result.exact_tool_selection_accuracy
+    accuracy_delta = candidate_accuracy - baseline_accuracy
+    argument_validity_delta = (
+        candidate_result.valid_argument_rate - baseline_result.valid_argument_rate
+    )
+    latency_delta_ms = candidate_result.mean_latency_ms - baseline_result.mean_latency_ms
+
+    baseline_cost = baseline_result.total_estimated_cost
+    candidate_cost = candidate_result.total_estimated_cost
+    cost_delta: float | None = None
+    if baseline_cost is not None or candidate_cost is not None:
+        cost_delta = (candidate_cost or 0.0) - (baseline_cost or 0.0)
+
+    regressions_by_case: dict[str, str] = {}
+    baseline_cases = set(baseline_result.per_case_pass_rate)
+    candidate_cases = set(candidate_result.per_case_pass_rate)
+    common_cases = baseline_cases & candidate_cases
+    for case_id in sorted(common_cases):
+        before_rate = baseline_result.per_case_pass_rate[case_id]
+        after_rate = candidate_result.per_case_pass_rate[case_id]
+        if before_rate > after_rate:
+            regressions_by_case[case_id] = f"pass rate {before_rate:.0%} -> {after_rate:.0%}"
+
+    return {
+        "benchmark_accuracy_delta": accuracy_delta,
+        "argument_validity_delta": argument_validity_delta,
+        "latency_delta_ms": latency_delta_ms,
+        "cost_delta": cost_delta,
+        "regressions_by_case": regressions_by_case,
+    }

mcplint/config/loader.py

@@ -0,0 +1,36 @@
+"""Load and validate mcplint.yaml."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+from pydantic import ValidationError
+
+from mcplint.config.schema import MCPLintConfig
+
+
+class ConfigError(Exception):
+    """Raised when mcplint.yaml exists but fails validation."""
+
+
+def load_config(path: Path | None) -> MCPLintConfig:
+    if path is None:
+        return MCPLintConfig()
+    if not path.exists():
+        return MCPLintConfig()
+
+    raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    if not isinstance(raw, dict):
+        raise ConfigError(
+            f"{path}: expected a YAML mapping at the top level, got {type(raw).__name__}"
+        )
+
+    try:
+        return MCPLintConfig.model_validate(raw)
+    except ValidationError as exc:
+        messages = "\n".join(
+            f"  - {'.'.join(str(loc) for loc in error['loc'])}: {error['msg']}"
+            for error in exc.errors()
+        )
+        raise ConfigError(f"{path}: invalid configuration:\n{messages}") from exc