mcplint-cli 0.1.0__py3-none-any.whl

mcplint/config/schema.py

@@ -0,0 +1,40 @@
+"""Typed schema for mcplint.yaml."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from mcplint.core.rules.ambiguity import DEFAULT_AMBIGUITY_THRESHOLD
+from mcplint.core.rules.completeness_rules import DEFAULT_MAX_DESCRIPTION_CHARACTERS
+from mcplint.models.findings import Severity
+
+
+class ThresholdsConfig(BaseModel):
+    ambiguity: float = Field(default=DEFAULT_AMBIGUITY_THRESHOLD, ge=0.0, le=1.0)
+    max_description_characters: int = Field(default=DEFAULT_MAX_DESCRIPTION_CHARACTERS, gt=0)
+
+
+class IgnoreEntry(BaseModel):
+    tool: str
+    rules: list[str] = Field(default_factory=list)
+
+
+class BenchmarkConfig(BaseModel):
+    provider: str | None = None
+    model: str | None = None
+    runs: int = Field(default=3, ge=1)
+
+
+class MCPLintConfig(BaseModel):
+    severity: dict[str, Severity] = Field(default_factory=dict)
+    thresholds: ThresholdsConfig = Field(default_factory=ThresholdsConfig)
+    ignore: list[IgnoreEntry] = Field(default_factory=list)
+    benchmark: BenchmarkConfig | None = None
+
+    def ignored_rule_ids_for_tool(self, tool_name: str, all_rule_ids: set[str]) -> set[str]:
+        ignored: set[str] = set()
+        for entry in self.ignore:
+            if entry.tool != tool_name:
+                continue
+            ignored |= set(entry.rules) if entry.rules else all_rule_ids
+        return ignored

mcplint/core/engine.py

@@ -0,0 +1,40 @@
+"""Pure function that runs every registered rule over every tool in a snapshot."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from mcplint.core.registry import RuleRegistry
+from mcplint.core.rules.base import RuleContext
+from mcplint.models.common import ArtifactMetadata
+from mcplint.models.findings import Finding, LintReport
+from mcplint.models.snapshot import MCPServerSnapshot
+
+if TYPE_CHECKING:
+    from mcplint.config.schema import MCPLintConfig
+
+REPORT_SCHEMA_VERSION = "1.0"
+
+
+def lint_snapshot(
+    snapshot: MCPServerSnapshot,
+    registry: RuleRegistry,
+    config: MCPLintConfig | None = None,
+) -> LintReport:
+    context = RuleContext(snapshot=snapshot)
+    all_rule_ids = {rule.id for rule in registry.all()}
+    findings: list[Finding] = []
+    for tool in snapshot.tools:
+        ignored = config.ignored_rule_ids_for_tool(tool.name, all_rule_ids) if config else set()
+        for rule in registry.all():
+            if rule.id in ignored:
+                continue
+            for finding in rule.check(tool, context):
+                if config and rule.id in config.severity:
+                    finding = finding.model_copy(update={"severity": config.severity[rule.id]})
+                findings.append(finding)
+    return LintReport(
+        metadata=ArtifactMetadata.create(schema_version=REPORT_SCHEMA_VERSION),
+        server_name=snapshot.server_name,
+        findings=findings,
+    )

mcplint/core/registry.py

@@ -0,0 +1,52 @@
+"""Collects built-in and plugin rules for the lint engine to run."""
+
+from __future__ import annotations
+
+from importlib.metadata import entry_points
+from typing import TYPE_CHECKING
+
+from mcplint.core.rules.base import Rule
+
+if TYPE_CHECKING:
+    from mcplint.config.schema import MCPLintConfig
+
+
+class RuleRegistry:
+    def __init__(self) -> None:
+        self._rules: dict[str, Rule] = {}
+
+    def register(self, rule: Rule) -> None:
+        if rule.id in self._rules:
+            raise ValueError(f"Rule '{rule.id}' is already registered")
+        self._rules[rule.id] = rule
+
+    def get(self, rule_id: str) -> Rule | None:
+        return self._rules.get(rule_id)
+
+    def all(self) -> list[Rule]:
+        return [self._rules[key] for key in sorted(self._rules)]
+
+    def load_entry_point_plugins(self) -> None:
+        for entry_point in entry_points(group="mcplint.rules"):
+            rule_cls = entry_point.load()
+            self.register(rule_cls())
+
+    @classmethod
+    def with_builtin_rules(cls, config: MCPLintConfig | None = None) -> RuleRegistry:
+        from mcplint.core.rules.ambiguity_rules import (
+            AmbiguousToolOverlapRule,
+            MissingToolDistinctionRule,
+        )
+        from mcplint.core.rules.builtin import BUILTIN_RULES
+        from mcplint.core.rules.completeness_rules import ExcessiveDescriptionLengthRule
+
+        registry = cls()
+        for rule_cls in BUILTIN_RULES:
+            rule = rule_cls()
+            if config is not None:
+                if isinstance(rule, AmbiguousToolOverlapRule | MissingToolDistinctionRule):
+                    rule.threshold = config.thresholds.ambiguity
+                elif isinstance(rule, ExcessiveDescriptionLengthRule):
+                    rule.max_characters = config.thresholds.max_description_characters
+            registry.register(rule)
+        return registry

mcplint/core/rules/ambiguity.py

@@ -0,0 +1,157 @@
+"""Cross-tool semantic ambiguity engine.
+
+Computes a 0-1 ambiguity score between every pair of tools from normalised
+token overlap, name similarity, description similarity, and input-schema
+(parameter name) similarity, plus optional sentence-transformer embeddings
+when the `semantic` extra is installed. Every flagged pair carries structured
+evidence (shared verbs/entities/parameters, absent distinctions) so the
+result is inspectable rather than an opaque score.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from mcplint.core.rules.safety_rules import READ_VERBS, WRITE_VERBS, first_word
+from mcplint.models.contracts import ToolContract
+
+DEFAULT_AMBIGUITY_THRESHOLD = 0.55
+
+_STOPWORDS = frozenset(
+    {
+        "a",
+        "an",
+        "the",
+        "of",
+        "for",
+        "to",
+        "and",
+        "or",
+        "in",
+        "on",
+        "by",
+        "with",
+        "is",
+        "are",
+        "this",
+        "that",
+        "it",
+        "its",
+        "your",
+        "you",
+        "from",
+        "as",
+        "at",
+        "be",
+        "will",
+        "if",
+        "not",
+        "can",
+        "does",
+        "do",
+    }
+)
+_MANY_HINTS = frozenset({"list", "search", "find", "query"})
+_ONE_HINTS = frozenset({"get", "fetch", "show", "view"})
+
+
+def _tokens(text: str, *, drop_stopwords: bool) -> set[str]:
+    cleaned = re.sub(r"[^a-z0-9\s]", " ", text.lower())
+    words = [w for w in cleaned.split() if w]
+    if drop_stopwords:
+        words = [w for w in words if w not in _STOPWORDS]
+    return set(words)
+
+
+def _jaccard(a: set[str], b: set[str]) -> float:
+    if not a and not b:
+        return 0.0
+    union = a | b
+    if not union:
+        return 0.0
+    return len(a & b) / len(union)
+
+
+@dataclass(frozen=True)
+class AmbiguityEvidence:
+    shared_verbs: tuple[str, ...] = ()
+    shared_entities: tuple[str, ...] = ()
+    overlapping_parameters: tuple[str, ...] = ()
+    absent_exact_vs_search_distinction: bool = False
+    absent_one_vs_many_distinction: bool = False
+    absent_read_vs_write_distinction: bool = False
+
+
+@dataclass(frozen=True)
+class AmbiguityPairResult:
+    tool_a: str
+    tool_b: str
+    score: float
+    name_similarity: float
+    description_similarity: float
+    schema_similarity: float
+    evidence: AmbiguityEvidence = field(default_factory=AmbiguityEvidence)
+
+    def has_missing_distinction(self) -> bool:
+        return (
+            self.evidence.absent_exact_vs_search_distinction
+            or self.evidence.absent_one_vs_many_distinction
+            or self.evidence.absent_read_vs_write_distinction
+        )
+
+
+def compute_ambiguity(tool_a: ToolContract, tool_b: ToolContract) -> AmbiguityPairResult:
+    name_tokens_a = _tokens(tool_a.name.replace("_", " ").replace("-", " "), drop_stopwords=False)
+    name_tokens_b = _tokens(tool_b.name.replace("_", " ").replace("-", " "), drop_stopwords=False)
+    name_similarity = _jaccard(name_tokens_a, name_tokens_b)
+
+    desc_tokens_a = _tokens(tool_a.description or "", drop_stopwords=True)
+    desc_tokens_b = _tokens(tool_b.description or "", drop_stopwords=True)
+    description_similarity = _jaccard(desc_tokens_a, desc_tokens_b)
+
+    schema_similarity = _jaccard(tool_a.parameter_names(), tool_b.parameter_names())
+
+    score = 0.25 * name_similarity + 0.45 * description_similarity + 0.30 * schema_similarity
+
+    verb_a, verb_b = first_word(tool_a.name), first_word(tool_b.name)
+    action_verbs = READ_VERBS | WRITE_VERBS
+    shared_verbs = tuple(sorted({verb_a} & {verb_b} & action_verbs))
+    shared_entities = tuple(sorted((name_tokens_a & name_tokens_b) - action_verbs))
+    overlapping_parameters = tuple(sorted(tool_a.parameter_names() & tool_b.parameter_names()))
+
+    combined_description = f"{tool_a.description or ''} {tool_b.description or ''}".lower()
+
+    absent_exact_vs_search = (
+        {verb_a, verb_b} & _ONE_HINTS
+        and {verb_a, verb_b} & _MANY_HINTS
+        and "exact" not in combined_description
+    )
+    one_vs_many_keywords = ("single", "one or more", "list of", "multiple")
+    absent_one_vs_many = (
+        bool({verb_a, verb_b} & _ONE_HINTS)
+        and bool({verb_a, verb_b} & _MANY_HINTS)
+        and not any(kw in combined_description for kw in one_vs_many_keywords)
+    )
+    read_only_a = bool(tool_a.annotations.read_only_hint)
+    read_only_b = bool(tool_b.annotations.read_only_hint)
+    absent_read_vs_write = read_only_a != read_only_b and "read-only" not in combined_description
+
+    evidence = AmbiguityEvidence(
+        shared_verbs=shared_verbs,
+        shared_entities=shared_entities,
+        overlapping_parameters=overlapping_parameters,
+        absent_exact_vs_search_distinction=bool(absent_exact_vs_search),
+        absent_one_vs_many_distinction=bool(absent_one_vs_many),
+        absent_read_vs_write_distinction=absent_read_vs_write,
+    )
+
+    return AmbiguityPairResult(
+        tool_a=tool_a.name,
+        tool_b=tool_b.name,
+        score=round(score, 4),
+        name_similarity=round(name_similarity, 4),
+        description_similarity=round(description_similarity, 4),
+        schema_similarity=round(schema_similarity, 4),
+        evidence=evidence,
+    )

mcplint/core/rules/ambiguity_rules.py

@@ -0,0 +1,109 @@
+"""Rules built on top of the cross-tool ambiguity engine (`ambiguity.py`).
+
+Both rules iterate every *other* tool in the snapshot but only emit a
+finding once per unordered pair (when `tool.name < other.name`), since the
+lint engine calls `check()` once per tool and pairs would otherwise be
+reported twice.
+"""
+
+from __future__ import annotations
+
+from mcplint.core.rules.ambiguity import DEFAULT_AMBIGUITY_THRESHOLD, compute_ambiguity
+from mcplint.core.rules.base import Rule, RuleContext
+from mcplint.models.contracts import SourceLocation, ToolContract
+from mcplint.models.findings import Finding, Severity
+
+
+class AmbiguousToolOverlapRule(Rule):
+    id = "ambiguous-tool-overlap"
+    title = "Ambiguous tool overlap"
+    description = (
+        "Flags pairs of tools whose name, description, and parameters overlap "
+        "enough that an agent could plausibly pick either one for the same task."
+    )
+    default_severity = Severity.WARNING
+    tags = ("ambiguity",)
+    threshold = DEFAULT_AMBIGUITY_THRESHOLD
+
+    def check(self, tool: ToolContract, context: RuleContext) -> list[Finding]:
+        findings = []
+        for other in context.snapshot.tools:
+            if other.name <= tool.name:
+                continue
+            result = compute_ambiguity(tool, other)
+            if result.score < self.threshold:
+                continue
+            evidence_parts = [
+                f"ambiguity score {result.score:.2f} (threshold {self.threshold:.2f})"
+            ]
+            if result.evidence.shared_verbs:
+                evidence_parts.append(f"shared verbs: {', '.join(result.evidence.shared_verbs)}")
+            if result.evidence.shared_entities:
+                evidence_parts.append(
+                    f"shared entities: {', '.join(result.evidence.shared_entities)}"
+                )
+            if result.evidence.overlapping_parameters:
+                evidence_parts.append(
+                    f"overlapping parameters: {', '.join(result.evidence.overlapping_parameters)}"
+                )
+            findings.append(
+                Finding(
+                    rule_id=self.id,
+                    severity=self.default_severity,
+                    message=f"Tools '{tool.name}' and '{other.name}' are semantically ambiguous.",
+                    evidence="; ".join(evidence_parts),
+                    location=SourceLocation(tool_name=tool.name, json_path="$.description"),
+                    remediation=(
+                        f"Clarify how '{tool.name}' differs from '{other.name}': when an "
+                        "agent should pick one over the other."
+                    ),
+                    confidence=min(0.9, 0.4 + result.score / 2),
+                )
+            )
+        return findings
+
+
+class MissingToolDistinctionRule(Rule):
+    id = "missing-tool-distinction"
+    title = "Missing tool distinction"
+    description = (
+        "For ambiguous tool pairs, flags the absence of an explicit "
+        "exact-vs-search, one-vs-many, or read-vs-write distinction."
+    )
+    default_severity = Severity.INFO
+    tags = ("ambiguity",)
+    threshold = DEFAULT_AMBIGUITY_THRESHOLD
+
+    def check(self, tool: ToolContract, context: RuleContext) -> list[Finding]:
+        findings = []
+        for other in context.snapshot.tools:
+            if other.name <= tool.name:
+                continue
+            result = compute_ambiguity(tool, other)
+            if result.score < self.threshold or not result.has_missing_distinction():
+                continue
+            missing = []
+            if result.evidence.absent_exact_vs_search_distinction:
+                missing.append("exact-vs-search")
+            if result.evidence.absent_one_vs_many_distinction:
+                missing.append("one-vs-many")
+            if result.evidence.absent_read_vs_write_distinction:
+                missing.append("read-vs-write")
+            findings.append(
+                Finding(
+                    rule_id=self.id,
+                    severity=self.default_severity,
+                    message=(
+                        f"Tools '{tool.name}' and '{other.name}' don't state a "
+                        f"{'/'.join(missing)} distinction."
+                    ),
+                    evidence=f"missing distinction(s): {', '.join(missing)}",
+                    location=SourceLocation(tool_name=tool.name, json_path="$.description"),
+                    remediation=(
+                        f'Add a sentence like "Use {tool.name} when ... Use {other.name} '
+                        f'when ..." to both descriptions.'
+                    ),
+                    confidence=0.5,
+                )
+            )
+        return findings

mcplint/core/rules/base.py

@@ -0,0 +1,39 @@
+"""The Rule contract every deterministic and plugin rule implements."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import ClassVar
+
+from pydantic import BaseModel, ConfigDict
+
+from mcplint.models.contracts import ToolContract
+from mcplint.models.findings import Finding, RuleMetadata, Severity
+from mcplint.models.snapshot import MCPServerSnapshot
+
+
+class RuleContext(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    snapshot: MCPServerSnapshot
+
+
+class Rule(ABC):
+    id: ClassVar[str]
+    title: ClassVar[str]
+    description: ClassVar[str]
+    default_severity: ClassVar[Severity]
+    tags: ClassVar[tuple[str, ...]] = ()
+
+    @abstractmethod
+    def check(self, tool: ToolContract, context: RuleContext) -> list[Finding]: ...
+
+    @classmethod
+    def metadata(cls) -> RuleMetadata:
+        return RuleMetadata(
+            id=cls.id,
+            title=cls.title,
+            description=cls.description,
+            default_severity=cls.default_severity,
+            tags=list(cls.tags),
+        )

mcplint/core/rules/builtin.py

@@ -0,0 +1,45 @@
+"""Registry of all built-in rule classes."""
+
+from __future__ import annotations
+
+from mcplint.core.rules.ambiguity_rules import AmbiguousToolOverlapRule, MissingToolDistinctionRule
+from mcplint.core.rules.base import Rule
+from mcplint.core.rules.completeness_rules import (
+    ExcessiveDescriptionLengthRule,
+    MissingReturnSemanticsRule,
+    UndefinedDomainTermRule,
+    UndocumentedErrorBehaviourRule,
+    UndocumentedRequiredConstraintRule,
+)
+from mcplint.core.rules.description_rules import (
+    DescriptionRepeatsNameRule,
+    MissingToolDescriptionRule,
+    VagueToolDescriptionRule,
+)
+from mcplint.core.rules.safety_rules import (
+    DestructiveToolWithoutWarningRule,
+    StateChangingToolMarkedReadOnlyRule,
+    ToolNameActionConflictRule,
+)
+from mcplint.core.rules.schema_rules import (
+    MissingParameterDescriptionRule,
+    SchemaDescriptionTypeConflictRule,
+)
+
+BUILTIN_RULES: list[type[Rule]] = [
+    MissingToolDescriptionRule,
+    DescriptionRepeatsNameRule,
+    VagueToolDescriptionRule,
+    MissingParameterDescriptionRule,
+    MissingReturnSemanticsRule,
+    UndocumentedErrorBehaviourRule,
+    UndocumentedRequiredConstraintRule,
+    SchemaDescriptionTypeConflictRule,
+    ToolNameActionConflictRule,
+    DestructiveToolWithoutWarningRule,
+    StateChangingToolMarkedReadOnlyRule,
+    AmbiguousToolOverlapRule,
+    MissingToolDistinctionRule,
+    ExcessiveDescriptionLengthRule,
+    UndefinedDomainTermRule,
+]