mcplint-cli 0.1.0__py3-none-any.whl

mcplint/core/score.py

@@ -0,0 +1,132 @@
+"""Computes the explainable 0-100 overall score documented in models/score.py.
+
+Weighting (out of 100, each category capped independently, then summed and
+clamped to [0, 100]):
+
+  - critical/error findings (not already counted below): up to 40 points,
+    8 points per finding.
+  - warning/info findings (not already counted below): up to 20 points,
+    2 points per finding.
+  - ambiguity (`ambiguous-tool-overlap`, `missing-tool-distinction`):
+    up to 15 points, 5 points per finding.
+  - schema completeness (`missing-parameter-description`,
+    `undocumented-required-constraint`, `schema-description-type-conflict`):
+    up to 15 points, 3 points per finding.
+  - safety clarity (`tool-name-action-conflict`,
+    `destructive-tool-without-warning`, `state-changing-tool-marked-read-only`):
+    up to 15 points, 5 points per finding.
+  - benchmark accuracy (only when a BenchmarkResult is supplied): up to 15
+    points, proportional to (1 - exact_tool_selection_accuracy).
+
+These weights are a documented, adjustable heuristic — not a scientifically
+derived formula. They exist so a regression in any one category is visible
+without being able to silently zero out the score.
+"""
+
+from __future__ import annotations
+
+from mcplint.models.benchmark import BenchmarkResult
+from mcplint.models.findings import Finding, LintReport, Severity
+from mcplint.models.score import ScoreBreakdown, ScoreDeduction
+
+AMBIGUITY_RULE_IDS = frozenset({"ambiguous-tool-overlap", "missing-tool-distinction"})
+SCHEMA_RULE_IDS = frozenset(
+    {
+        "missing-parameter-description",
+        "undocumented-required-constraint",
+        "schema-description-type-conflict",
+    }
+)
+SAFETY_RULE_IDS = frozenset(
+    {
+        "tool-name-action-conflict",
+        "destructive-tool-without-warning",
+        "state-changing-tool-marked-read-only",
+    }
+)
+
+_ERROR_POINTS_PER_FINDING = 8.0
+_ERROR_CATEGORY_CAP = 40.0
+_WARNING_POINTS_PER_FINDING = 2.0
+_WARNING_CATEGORY_CAP = 20.0
+_AMBIGUITY_POINTS_PER_FINDING = 5.0
+_AMBIGUITY_CATEGORY_CAP = 15.0
+_SCHEMA_POINTS_PER_FINDING = 3.0
+_SCHEMA_CATEGORY_CAP = 15.0
+_SAFETY_POINTS_PER_FINDING = 5.0
+_SAFETY_CATEGORY_CAP = 15.0
+_BENCHMARK_CATEGORY_CAP = 15.0
+
+
+def _category(finding: Finding) -> str:
+    if finding.rule_id in AMBIGUITY_RULE_IDS:
+        return "ambiguity"
+    if finding.rule_id in SCHEMA_RULE_IDS:
+        return "schema_completeness"
+    if finding.rule_id in SAFETY_RULE_IDS:
+        return "safety_clarity"
+    if finding.severity == Severity.ERROR:
+        return "critical_error"
+    return "warning_info"
+
+
+def compute_score(
+    report: LintReport, benchmark_result: BenchmarkResult | None = None
+) -> ScoreBreakdown:
+    counts: dict[str, int] = {}
+    for finding in report.findings:
+        category = _category(finding)
+        counts[category] = counts.get(category, 0) + 1
+
+    deductions: list[ScoreDeduction] = []
+
+    def _add(category: str, label: str, per_finding: float, cap: float) -> None:
+        count = counts.get(category, 0)
+        if count == 0:
+            return
+        points = min(cap, count * per_finding)
+        deductions.append(
+            ScoreDeduction(
+                category=category,
+                points_lost=points,
+                finding_count=count,
+                explanation=(
+                    f"{count} {label} finding(s) x {per_finding:.0f} pts (capped at {cap:.0f})"
+                ),
+            )
+        )
+
+    _add("critical_error", "critical/error", _ERROR_POINTS_PER_FINDING, _ERROR_CATEGORY_CAP)
+    _add("warning_info", "warning/info", _WARNING_POINTS_PER_FINDING, _WARNING_CATEGORY_CAP)
+    _add("ambiguity", "ambiguity", _AMBIGUITY_POINTS_PER_FINDING, _AMBIGUITY_CATEGORY_CAP)
+    _add(
+        "schema_completeness",
+        "schema-completeness",
+        _SCHEMA_POINTS_PER_FINDING,
+        _SCHEMA_CATEGORY_CAP,
+    )
+    _add("safety_clarity", "safety-clarity", _SAFETY_POINTS_PER_FINDING, _SAFETY_CATEGORY_CAP)
+
+    benchmark_accuracy: float | None = None
+    if benchmark_result is not None:
+        benchmark_accuracy = benchmark_result.exact_tool_selection_accuracy
+        benchmark_points = round((1 - benchmark_accuracy) * _BENCHMARK_CATEGORY_CAP, 1)
+        if benchmark_points > 0:
+            deductions.append(
+                ScoreDeduction(
+                    category="benchmark_accuracy",
+                    points_lost=benchmark_points,
+                    finding_count=0,
+                    explanation=(
+                        f"exact tool-selection accuracy {benchmark_accuracy:.0%} "
+                        f"(up to {_BENCHMARK_CATEGORY_CAP:.0f} pts)"
+                    ),
+                )
+            )
+
+    total_deducted = sum(d.points_lost for d in deductions)
+    total_score = max(0, min(100, round(100 - total_deducted)))
+
+    return ScoreBreakdown(
+        total_score=total_score, deductions=deductions, benchmark_accuracy=benchmark_accuracy
+    )

mcplint/fix/suggest.py

@@ -0,0 +1,183 @@
+"""Deterministic rewrite suggestions built directly from JSON Schema and
+annotations — no LLM. Each fixable finding contributes one clause appended
+to the tool's existing description (or, for excessive-description-length,
+a deterministic truncation). Purely semantic issues (vague wording, a
+description that just restates the name, or a missing description) cannot
+be fixed with fabricated prose in deterministic mode: they get a low-
+confidence TODO placeholder instead, honestly flagged as needing manual or
+LLM-assisted rewriting.
+"""
+
+from __future__ import annotations
+
+import re
+
+from mcplint.models.contracts import ToolContract
+from mcplint.models.findings import Finding, LintReport
+from mcplint.models.fixes import RewriteSuggestion
+from mcplint.models.snapshot import MCPServerSnapshot
+
+_SEMANTIC_PLACEHOLDER_RULES = frozenset(
+    {"missing-tool-description", "description-repeats-name", "vague-tool-description"}
+)
+_ACTIONABLE_RULE_IDS = _SEMANTIC_PLACEHOLDER_RULES | frozenset(
+    {
+        "missing-return-semantics",
+        "undocumented-required-constraint",
+        "destructive-tool-without-warning",
+        "missing-tool-distinction",
+        "excessive-description-length",
+    }
+)
+
+_OTHER_TOOL_PATTERN = re.compile(r"'([a-zA-Z0-9_-]+)'")
+
+
+def _describe_output_schema(schema: dict[str, object]) -> str:
+    properties = schema.get("properties")
+    if isinstance(properties, dict) and properties:
+        fields = ", ".join(sorted(properties))
+        return f"Returns an object with fields: {fields}."
+    schema_type = schema.get("type")
+    if isinstance(schema_type, str):
+        return f"Returns a value of type '{schema_type}'."
+    return "Returns a result."
+
+
+def _constraint_clause(param_name: str, schema: dict[str, object]) -> str | None:
+    enum_values = schema.get("enum")
+    if isinstance(enum_values, list):
+        values = ", ".join(str(v) for v in enum_values)
+        return f"Parameter '{param_name}' accepts: {values}."
+    minimum = schema.get("minimum")
+    maximum = schema.get("maximum")
+    if minimum is not None and maximum is not None:
+        return f"Parameter '{param_name}' must be between {minimum} and {maximum}."
+    if minimum is not None:
+        return f"Parameter '{param_name}' must be at least {minimum}."
+    if maximum is not None:
+        return f"Parameter '{param_name}' must be at most {maximum}."
+    return None
+
+
+def _other_tool_name(message: str, this_tool: str) -> str | None:
+    names = [name for name in _OTHER_TOOL_PATTERN.findall(message) if name != this_tool]
+    return names[0] if names else None
+
+
+def _truncate_description(description: str, max_characters: int) -> str:
+    if len(description) <= max_characters:
+        return description
+    truncated = description[:max_characters]
+    last_period = truncated.rfind(". ")
+    if last_period > 0:
+        return truncated[: last_period + 1]
+    return truncated.rstrip() + "…"
+
+
+def suggest_for_tool(tool: ToolContract, findings: list[Finding]) -> RewriteSuggestion | None:
+    actionable = [f for f in findings if f.rule_id in _ACTIONABLE_RULE_IDS]
+    if not actionable:
+        return None
+
+    base = (tool.description or "").strip()
+    clauses: list[str] = []
+    resolved_rule_ids: list[str] = []
+    explanations: list[str] = []
+    confidences: list[float] = []
+
+    for finding in actionable:
+        rule_id = finding.rule_id
+
+        if rule_id == "missing-return-semantics":
+            if tool.output_schema:
+                clauses.append(_describe_output_schema(tool.output_schema))
+                explanations.append("Documented the return shape from outputSchema.")
+                confidences.append(0.9)
+            else:
+                clauses.append("Document what this tool returns (e.g. field names and types).")
+                explanations.append("Flagged the missing return description for manual detail.")
+                confidences.append(0.4)
+            resolved_rule_ids.append(rule_id)
+
+        elif rule_id == "undocumented-required-constraint":
+            parts = finding.location.json_path.split(".")
+            param_name = parts[3] if len(parts) > 3 else None
+            param = next((p for p in tool.parameters if p.name == param_name), None)
+            clause = _constraint_clause(param.name, param.json_schema) if param else None
+            if clause:
+                clauses.append(clause)
+                resolved_rule_ids.append(rule_id)
+                explanations.append(f"Stated the '{param_name}' constraint from its JSON Schema.")
+                confidences.append(0.85)
+
+        elif rule_id == "destructive-tool-without-warning":
+            clauses.append("This action is permanent and cannot be undone.")
+            resolved_rule_ids.append(rule_id)
+            explanations.append("Added a destructive-operation warning.")
+            confidences.append(0.9)
+
+        elif rule_id == "missing-tool-distinction":
+            other = _other_tool_name(finding.message, tool.name)
+            if other:
+                clauses.append(
+                    f"See {other} for a related but different operation — check both "
+                    "descriptions before choosing."
+                )
+                resolved_rule_ids.append(rule_id)
+                explanations.append(
+                    f"Added a placeholder distinction between '{tool.name}' and '{other}'; "
+                    "refine the wording manually or with LLM-assisted rewriting."
+                )
+                confidences.append(0.35)
+
+        elif rule_id in _SEMANTIC_PLACEHOLDER_RULES:
+            placeholder = (
+                "[TODO: describe the specific action, inputs, and output of this tool, "
+                "and when to use it instead of similar tools]"
+            )
+            if placeholder not in clauses:
+                clauses.append(placeholder)
+                resolved_rule_ids.append(rule_id)
+                explanations.append(
+                    "Flagged for manual or LLM-assisted rewriting — deterministic mode "
+                    "cannot fabricate semantic content."
+                )
+                confidences.append(0.2)
+
+    # Truncation takes precedence over appended clauses: adding more text would
+    # defeat the point of shortening an over-long description.
+    length_finding = next(
+        (f for f in actionable if f.rule_id == "excessive-description-length"), None
+    )
+    if length_finding is not None and base and length_finding.rule_id not in resolved_rule_ids:
+        proposed = _truncate_description(base, 800)
+        resolved_rule_ids.append(length_finding.rule_id)
+        explanations.append("Truncated the description to the configured character limit.")
+        confidences.append(0.8)
+    else:
+        proposed = " ".join([base, *clauses]).strip() if base or clauses else ""
+
+    if not resolved_rule_ids:
+        return None
+
+    return RewriteSuggestion(
+        tool_name=tool.name,
+        proposed_description=proposed,
+        resolved_rule_ids=resolved_rule_ids,
+        explanation=" ".join(explanations),
+        confidence=round(min(confidences), 2),
+    )
+
+
+def build_suggestions(snapshot: MCPServerSnapshot, report: LintReport) -> list[RewriteSuggestion]:
+    findings_by_tool: dict[str, list[Finding]] = {}
+    for finding in report.findings:
+        findings_by_tool.setdefault(finding.location.tool_name, []).append(finding)
+
+    suggestions = []
+    for tool in snapshot.tools:
+        suggestion = suggest_for_tool(tool, findings_by_tool.get(tool.name, []))
+        if suggestion is not None:
+            suggestions.append(suggestion)
+    return suggestions

mcplint/mcp_client/canonical.py

@@ -0,0 +1,25 @@
+"""Single source of truth for deterministic IDs and byte-stable snapshot JSON.
+
+MCP servers are untrusted local (or remote) processes; this module only ever
+touches data already parsed into typed models, never raw process output.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from mcplint.models.snapshot import MCPServerSnapshot
+
+
+def stable_tool_id(server_name: str, tool_name: str) -> str:
+    digest = hashlib.sha256(f"{server_name}::{tool_name}".encode()).hexdigest()
+    return digest[:16]
+
+
+def canonical_json(snapshot: MCPServerSnapshot) -> str:
+    payload = snapshot.model_dump(mode="json")
+    payload["metadata"].pop("generated_at", None)
+    return json.dumps(payload, sort_keys=True, separators=(",", ":"))

mcplint/mcp_client/persistence.py

@@ -0,0 +1,17 @@
+"""Read/write MCPServerSnapshot to/from disk as JSON."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from mcplint.models.snapshot import MCPServerSnapshot
+
+
+def save_snapshot(snapshot: MCPServerSnapshot, path: Path) -> None:
+    path.write_text(snapshot.model_dump_json(indent=2) + "\n", encoding="utf-8")
+
+
+def load_snapshot(path: Path) -> MCPServerSnapshot:
+    if not path.exists():
+        raise FileNotFoundError(f"Snapshot file not found: {path}")
+    return MCPServerSnapshot.model_validate_json(path.read_text(encoding="utf-8"))

mcplint/mcp_client/session.py

@@ -0,0 +1,80 @@
+"""Connects to a stdio MCP server and produces an MCPServerSnapshot."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+
+from mcplint.mcp_client.canonical import stable_tool_id
+from mcplint.mcp_client.stdio import parse_command
+from mcplint.models.common import ArtifactMetadata
+from mcplint.models.contracts import ParameterContract, ToolAnnotation, ToolContract
+from mcplint.models.snapshot import MCPServerSnapshot
+
+if TYPE_CHECKING:
+    from mcp.types import Tool as SDKTool
+
+SNAPSHOT_SCHEMA_VERSION = "1.0"
+
+__all__ = ["collect_stdio_snapshot", "tool_from_mcp", "parse_command"]
+
+
+def tool_from_mcp(server_name: str, tool: SDKTool) -> ToolContract:
+    schema = tool.inputSchema or {"type": "object"}
+    properties: dict[str, object] = schema.get("properties", {})
+    required: list[str] = schema.get("required", [])
+
+    parameters = [
+        ParameterContract(
+            name=name,
+            json_schema=prop_schema if isinstance(prop_schema, dict) else {},
+            required=name in required,
+            description=(prop_schema.get("description") if isinstance(prop_schema, dict) else None),
+        )
+        for name, prop_schema in properties.items()
+    ]
+
+    annotations = ToolAnnotation()
+    if tool.annotations is not None:
+        annotations = ToolAnnotation(
+            title=tool.annotations.title,
+            read_only_hint=tool.annotations.readOnlyHint,
+            destructive_hint=tool.annotations.destructiveHint,
+            idempotent_hint=tool.annotations.idempotentHint,
+            open_world_hint=tool.annotations.openWorldHint,
+        )
+
+    return ToolContract(
+        id=stable_tool_id(server_name, tool.name),
+        name=tool.name,
+        description=tool.description,
+        input_schema=schema,
+        output_schema=tool.outputSchema,
+        parameters=parameters,
+        annotations=annotations,
+        raw=tool.model_dump(mode="json"),
+    )
+
+
+async def collect_stdio_snapshot(
+    command: str, args: list[str], *, env: dict[str, str] | None = None
+) -> MCPServerSnapshot:
+    params = StdioServerParameters(command=command, args=args, env=env)
+    async with stdio_client(params) as (read, write), ClientSession(read, write) as session:
+        init_result = await session.initialize()
+        server_name = init_result.serverInfo.name
+        server_version = init_result.serverInfo.version
+        listed = await session.list_tools()
+        tools = [tool_from_mcp(server_name, t) for t in listed.tools]
+
+    command_line = " ".join([command, *args])
+    return MCPServerSnapshot(
+        metadata=ArtifactMetadata.create(schema_version=SNAPSHOT_SCHEMA_VERSION),
+        server_name=server_name,
+        server_version=server_version,
+        transport="stdio",
+        command=command_line,
+        tools=tools,
+    )

mcplint/mcp_client/stdio.py

@@ -0,0 +1,17 @@
+"""Process-spawning helpers for stdio MCP servers.
+
+MCP servers invoked here are treated as untrusted local processes: the
+command/args are passed as a list (never shell=True) and no output beyond
+the MCP protocol stream is trusted or persisted.
+"""
+
+from __future__ import annotations
+
+import shlex
+
+
+def parse_command(command_line: str) -> tuple[str, list[str]]:
+    parts = shlex.split(command_line)
+    if not parts:
+        raise ValueError("Empty server command")
+    return parts[0], parts[1:]

mcplint/models/benchmark.py

@@ -0,0 +1,67 @@
+"""Typed models for the benchmark dataset format, provider results, and scoring."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from mcplint.models.common import ArtifactMetadata
+
+
+class ExpectedToolCall(BaseModel):
+    tool: str
+    arguments: dict[str, object] = Field(default_factory=dict)
+    forbidden_tools: list[str] = Field(default_factory=list)
+    argument_assertions: dict[str, object] = Field(default_factory=dict)
+
+    def all_expected_arguments(self) -> dict[str, object]:
+        return {**self.arguments, **self.argument_assertions}
+
+
+class BenchmarkCase(BaseModel):
+    id: str
+    prompt: str
+    expected: ExpectedToolCall
+
+
+class BenchmarkDataset(BaseModel):
+    name: str
+    version: str
+    cases: list[BenchmarkCase]
+
+
+class ActualToolCall(BaseModel):
+    tool: str | None
+    arguments: dict[str, object] = Field(default_factory=dict)
+    valid_arguments: bool = True
+    error: str | None = None
+    latency_ms: float = 0.0
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    estimated_cost: float | None = None
+
+
+class BenchmarkTrial(BaseModel):
+    case_id: str
+    trial_index: int
+    actual: ActualToolCall
+    passed: bool
+    failure_reasons: list[str] = Field(default_factory=list)
+
+
+class BenchmarkResult(BaseModel):
+    metadata: ArtifactMetadata
+    dataset_name: str
+    provider: str
+    model: str
+    runs_per_case: int
+    trials: list[BenchmarkTrial]
+    exact_tool_selection_accuracy: float
+    valid_argument_rate: float
+    required_argument_accuracy: float
+    forbidden_tool_invocation_rate: float
+    no_tool_rate: float
+    mean_latency_ms: float
+    p95_latency_ms: float
+    total_estimated_cost: float | None
+    per_case_pass_rate: dict[str, float]
+    stability: dict[str, float]

mcplint/models/common.py

@@ -0,0 +1,23 @@
+"""Shared metadata mixin every persisted MCPLint artifact embeds."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from pydantic import BaseModel
+
+from mcplint.__about__ import __version__ as _MCPLINT_VERSION
+
+
+class ArtifactMetadata(BaseModel):
+    schema_version: str
+    generated_at: datetime
+    mcplint_version: str
+
+    @classmethod
+    def create(cls, schema_version: str) -> ArtifactMetadata:
+        return cls(
+            schema_version=schema_version,
+            generated_at=datetime.now(UTC),
+            mcplint_version=_MCPLINT_VERSION,
+        )

mcplint/models/comparison.py

@@ -0,0 +1,53 @@
+"""Typed models for comparing two MCPServerSnapshots (and optionally two benchmark runs)."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from mcplint.models.common import ArtifactMetadata
+from mcplint.models.findings import Finding
+
+
+class SchemaChange(BaseModel):
+    tool_name: str
+    json_path: str
+    before: object
+    after: object
+
+
+class DescriptionChange(BaseModel):
+    tool_name: str
+    before: str | None
+    after: str | None
+
+
+class AmbiguityScoreChange(BaseModel):
+    tool_a: str
+    tool_b: str
+    before: float
+    after: float
+
+
+class ComparisonReport(BaseModel):
+    metadata: ArtifactMetadata
+    baseline_server_name: str
+    candidate_server_name: str
+    added_tools: list[str] = Field(default_factory=list)
+    removed_tools: list[str] = Field(default_factory=list)
+    schema_changes: list[SchemaChange] = Field(default_factory=list)
+    description_changes: list[DescriptionChange] = Field(default_factory=list)
+    new_findings: list[Finding] = Field(default_factory=list)
+    resolved_findings: list[Finding] = Field(default_factory=list)
+    ambiguity_score_changes: list[AmbiguityScoreChange] = Field(default_factory=list)
+
+    benchmark_dataset_name: str | None = None
+    baseline_accuracy: float | None = None
+    candidate_accuracy: float | None = None
+    benchmark_accuracy_delta: float | None = None
+    argument_validity_delta: float | None = None
+    latency_delta_ms: float | None = None
+    cost_delta: float | None = None
+    regressions_by_case: dict[str, str] = Field(default_factory=dict)
+
+    min_accuracy_delta_threshold: float | None = None
+    passes_ci_threshold: bool | None = None

mcplint/models/contracts.py

@@ -0,0 +1,39 @@
+"""Typed representations of an MCP tool contract, independent of the wire format."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class SourceLocation(BaseModel):
+    tool_name: str
+    json_path: str
+
+
+class ParameterContract(BaseModel):
+    name: str
+    json_schema: dict[str, object]
+    required: bool
+    description: str | None = None
+
+
+class ToolAnnotation(BaseModel):
+    title: str | None = None
+    read_only_hint: bool | None = None
+    destructive_hint: bool | None = None
+    idempotent_hint: bool | None = None
+    open_world_hint: bool | None = None
+
+
+class ToolContract(BaseModel):
+    id: str
+    name: str
+    description: str | None
+    input_schema: dict[str, object]
+    output_schema: dict[str, object] | None = None
+    parameters: list[ParameterContract]
+    annotations: ToolAnnotation
+    raw: dict[str, object]
+
+    def parameter_names(self) -> set[str]:
+        return {param.name for param in self.parameters}