mcpprobe-scanner 0.1.0__py3-none-any.whl

mcpprobe/checks/base.py

@@ -0,0 +1,228 @@
+"""Plugin framework for checks.
+
+A check is a class with an ``id``, ``title``, default ``severity``, a
+``standards`` mapping, a ``mode`` (passive or active) and an async
+``run(ctx) -> list[Finding]``. The orchestrator decides whether a check may run
+(active checks require the authorisation gate) and records a
+:class:`~mcpprobe.models.CheckResult` for every check, including skipped ones,
+so reports show coverage honestly.
+"""
+
+from __future__ import annotations
+
+import abc
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from mcpprobe.models import (
+    CheckMode,
+    Confidence,
+    EvidenceRecord,
+    Finding,
+    Fingerprint,
+    Inventory,
+    Severity,
+    Standards,
+)
+from mcpprobe.transport import EvidenceLog, HttpTransport, Transport
+
+if TYPE_CHECKING:
+    from mcpprobe.authz import AuthorisationRecord
+
+#: Signature of the optional LLM-judge hook. It receives the text under review
+#: and a short task description and returns ``None`` (abstain) or a verdict.
+JudgeFn = Callable[[str, str], "JudgeVerdict | None"]
+
+
+@dataclass(slots=True)
+class JudgeVerdict:
+    """Verdict from an optional second-opinion judge (no network call is made by mcpprobe)."""
+
+    suspicious: bool
+    rationale: str = ""
+    confidence: Confidence = Confidence.LOW
+
+
+@dataclass(slots=True)
+class ScanOptions:
+    """User-controlled knobs that checks may consult."""
+
+    mode: CheckMode = CheckMode.PASSIVE
+    baseline_path: str | None = None
+    write_baseline: bool = True
+    judge: JudgeFn | None = None
+    excluded_tools: set[str] = field(default_factory=set)
+    #: AUTH-03 fetches the authorization server's metadata, which may live on a
+    #: third-party host. Set False to keep all traffic on the target host.
+    follow_authorization_server: bool = True
+    #: Environment passed to a stdio server (scanned by META-05 for secrets).
+    stdio_env: dict[str, str] = field(default_factory=dict)
+    stdio_command: list[str] = field(default_factory=list)
+    timeout: float = 15.0
+    #: DNS TXT resolver used for opt-out checks on third-party hosts (injected in tests).
+    txt_lookup: Callable[[str], list[str]] | None = None
+
+
+class ScanContext:
+    """Everything a check needs: the negotiated transport, the inventory and options."""
+
+    def __init__(
+        self,
+        *,
+        target: str,
+        transport: Transport,
+        fingerprint: Fingerprint,
+        inventory: Inventory,
+        evidence: EvidenceLog,
+        options: ScanOptions | None = None,
+        authorisation: AuthorisationRecord | None = None,
+    ) -> None:
+        self.target = target
+        self.transport = transport
+        self.fingerprint = fingerprint
+        self.inventory = inventory
+        self.evidence = evidence
+        self.options = options or ScanOptions()
+        self.authorisation = authorisation
+        self.notes: list[str] = []
+        #: Scratch space for checks to share derived data (e.g. META-03 classes).
+        self.shared: dict[str, Any] = {}
+
+    @property
+    def http(self) -> HttpTransport | None:
+        return self.transport if isinstance(self.transport, HttpTransport) else None
+
+    @property
+    def is_http(self) -> bool:
+        return self.http is not None
+
+    @property
+    def active_allowed(self) -> bool:
+        """Active checks may run only when the authorisation gate produced a verified record."""
+        return (
+            self.options.mode == CheckMode.ACTIVE
+            and self.authorisation is not None
+            and self.authorisation.verified
+        )
+
+    def tool_names(self) -> list[str]:
+        return [str(t.get("name", "")) for t in self.inventory.tools]
+
+
+class Check(abc.ABC):
+    """Base class for all checks. Subclasses set the class attributes and implement ``run``."""
+
+    id: str = "BASE-00"
+    title: str = ""
+    severity: Severity = Severity.MEDIUM
+    standards: Standards = Standards()
+    mode: CheckMode = CheckMode.PASSIVE
+    #: Which transports the check makes sense for.
+    applies_to: frozenset[str] = frozenset({"http", "stdio"})
+    #: One-paragraph description for CHECKS.md and SARIF rule help.
+    description: str = ""
+    #: What the evidence consists of (used in docs).
+    evidence_type: str = ""
+    #: Known false-positive conditions (used in docs and attached to findings).
+    false_positive_notes: str = ""
+    #: Default remediation text.
+    fix: str = ""
+
+    @abc.abstractmethod
+    async def run(self, ctx: ScanContext) -> list[Finding]:
+        """Execute the check and return zero or more findings."""
+
+    def finding(
+        self,
+        *,
+        description: str,
+        severity: Severity | None = None,
+        confidence: Confidence = Confidence.HIGH,
+        evidence: list[EvidenceRecord] | None = None,
+        location: str | None = None,
+        fix: str | None = None,
+        details: dict[str, Any] | None = None,
+        title: str | None = None,
+        excerpt: str | None = None,
+    ) -> Finding:
+        """Build a :class:`Finding` pre-filled from the check's metadata.
+
+        ``excerpt`` overrides the evidence excerpt that is otherwise derived from
+        the first evidence record.
+        """
+        records = [e for e in (evidence or []) if e is not None]
+        return Finding(
+            id=self.id,
+            title=title or self.title,
+            severity=severity or self.severity,
+            confidence=confidence,
+            standards=self.standards,
+            description=description,
+            fix=fix or self.fix,
+            evidence=records,
+            evidence_excerpt=excerpt if excerpt is not None else evidence_excerpt(records),
+            location=location,
+            details=details or {},
+            false_positive_notes=self.false_positive_notes or None,
+        )
+
+    def applies(self, ctx: ScanContext) -> bool:
+        return ctx.transport.name in self.applies_to
+
+
+class ActiveCheck(Check):
+    """Base for checks that may call tools or send probe payloads.
+
+    Subclasses must call :meth:`require_active` first. The orchestrator also
+    refuses to run active checks without authorisation, so this is a second
+    line of defence inside the check itself.
+    """
+
+    mode = CheckMode.ACTIVE
+
+    @staticmethod
+    def require_active(ctx: ScanContext) -> None:
+        if not ctx.active_allowed:
+            raise PermissionError("active check requires --active with a verified authorisation record")
+
+
+EXCERPT_BODY_CHARS = 240
+
+
+def evidence_excerpt(records: list[EvidenceRecord]) -> str:
+    """One-line summary of the first evidence record: request, status and a body snippet."""
+    if not records:
+        return ""
+    rec = records[0]
+    head = rec.method
+    if rec.url and rec.transport == "http":
+        head += f" {rec.url}"
+    if rec.status is not None:
+        head += f" -> HTTP {rec.status}"
+    elif rec.error:
+        head += f" -> {rec.error}"
+    body = " ".join(rec.response_body.split())
+    if len(body) > EXCERPT_BODY_CHARS:
+        body = body[: EXCERPT_BODY_CHARS - 1] + "\u2026"
+    extra = f" (+{len(records) - 1} more)" if len(records) > 1 else ""
+    return f"[{rec.id}] {head}" + (f": {body}" if body else "") + extra
+
+
+def evidence_summary(records: list[EvidenceRecord], summary: str) -> str:
+    """``[ev-0005] tools/list -> HTTP 200: <summary>`` for checks that know what matters."""
+    if not records:
+        return summary
+    rec = records[0]
+    head = rec.method + (f" -> HTTP {rec.status}" if rec.status is not None else "")
+    return f"[{rec.id}] {head}: {summary}"
+
+
+def evidence_of(*responses: Any) -> list[EvidenceRecord]:
+    """Collect evidence records from RpcResponse objects, ignoring ``None``."""
+    out: list[EvidenceRecord] = []
+    for r in responses:
+        rec = getattr(r, "evidence", None)
+        if rec is not None:
+            out.append(rec)
+    return out

mcpprobe/checks/baseline.py

@@ -0,0 +1,219 @@
+"""META-02: canonical hashing of the server's listed surface and rug-pull diffing."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from mcpprobe.checks._util import canonical_json
+from mcpprobe.checks.base import Check, ScanContext
+from mcpprobe.models import Confidence, Finding, Severity, Standards, utcnow
+
+BASELINE_FORMAT_VERSION = 1
+
+
+def sha256_hex(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _key(item: dict[str, Any], fallback: str) -> str:
+    return str(item.get("name") or item.get("uri") or fallback)
+
+
+def build_snapshot(ctx: ScanContext) -> dict[str, Any]:
+    """Canonical snapshot and per-item hashes of tools, prompts, resources and instructions."""
+    items: dict[str, Any] = {"tools": {}, "prompts": {}, "resources": {}, "instructions": None}
+    for kind in ("tools", "prompts", "resources"):
+        for idx, entry in enumerate(getattr(ctx.inventory, kind)):
+            items[kind][_key(entry, f"#{idx}")] = sha256_hex(canonical_json(entry))
+    if ctx.fingerprint.instructions:
+        items["instructions"] = sha256_hex(ctx.fingerprint.instructions)
+    snapshot = {
+        "tools": ctx.inventory.tools,
+        "prompts": ctx.inventory.prompts,
+        "resources": ctx.inventory.resources,
+        "instructions": ctx.fingerprint.instructions,
+    }
+    return {
+        "format": BASELINE_FORMAT_VERSION,
+        "target": ctx.target,
+        "created_at": utcnow().isoformat(),
+        "overall_sha256": sha256_hex(canonical_json(snapshot)),
+        "items": items,
+        "snapshot": snapshot,
+    }
+
+
+def diff_snapshots(old: dict[str, Any], new: dict[str, Any]) -> dict[str, Any]:
+    """Return added/removed/changed names per kind plus changed fields for tools."""
+    out: dict[str, Any] = {}
+    old_items = old.get("items") or {}
+    new_items = new.get("items") or {}
+    for kind in ("tools", "prompts", "resources"):
+        o = old_items.get(kind) or {}
+        n = new_items.get(kind) or {}
+        changed = sorted(k for k in o.keys() & n.keys() if o[k] != n[k])
+        entry: dict[str, Any] = {
+            "added": sorted(n.keys() - o.keys()),
+            "removed": sorted(o.keys() - n.keys()),
+            "changed": changed,
+        }
+        if kind == "tools" and changed:
+            old_tools = {_key(t, ""): t for t in (old.get("snapshot") or {}).get("tools") or []}
+            new_tools = {_key(t, ""): t for t in (new.get("snapshot") or {}).get("tools") or []}
+            fields: dict[str, list[str]] = {}
+            for name in changed:
+                a, b = old_tools.get(name, {}), new_tools.get(name, {})
+                fields[name] = sorted(k for k in set(a) | set(b) if a.get(k) != b.get(k))
+            entry["changed_fields"] = fields
+        out[kind] = entry
+    out["instructions_changed"] = old_items.get("instructions") != new_items.get("instructions")
+    out["any_change"] = out["instructions_changed"] or any(
+        out[k]["added"] or out[k]["removed"] or out[k]["changed"] for k in ("tools", "prompts", "resources")
+    )
+    return out
+
+
+class Meta02RugPullBaseline(Check):
+    id = "META-02"
+    title = "Tool surface changed since the stored baseline (rug-pull indicator)"
+    severity = Severity.MEDIUM
+    standards = Standards(mcp=["MCP03:2025"], asi=["ASI04"])
+    description = (
+        "Computes SHA-256 over the canonical JSON of each tool, prompt and resource definition "
+        "and the server instructions. With --baseline FILE the current snapshot is compared to "
+        "the stored one and any added, removed or changed definition is reported (a changed "
+        "description after approval is the classic 'rug pull'). The snapshot is then written "
+        "back to the file. Also notes when the server advertises tools.listChanged."
+    )
+    evidence_type = "The current tools/list response plus per-item hashes and the diff."
+    false_positive_notes = (
+        "Any legitimate release that edits a description will trigger the diff; the finding "
+        "says what changed so the reviewer can decide. Servers that embed timestamps or random "
+        "data in descriptions will change on every scan."
+    )
+    fix = (
+        "Treat tool definitions as code: version them, review changes, and have clients pin and "
+        "re-approve definitions when the hash changes."
+    )
+
+    async def run(self, ctx: ScanContext) -> list[Finding]:
+        findings: list[Finding] = []
+        current = build_snapshot(ctx)
+        ctx.shared["baseline_current"] = current
+        ctx.notes.append(
+            f"META-02: surface hash {current['overall_sha256'][:16]}... "
+            f"({len(current['items']['tools'])} tools)"
+        )
+        caps = ctx.fingerprint.capabilities or {}
+        tools_caps = caps.get("tools") if isinstance(caps, dict) else None
+        if isinstance(tools_caps, dict) and tools_caps.get("listChanged"):
+            findings.append(
+                self.finding(
+                    title="Server can change its tool list mid-session (listChanged)",
+                    description="The server advertises tools.listChanged, so definitions may change "
+                    "after a client has approved them. Clients should re-review on change.",
+                    severity=Severity.INFO,
+                    confidence=Confidence.HIGH,
+                    location="capabilities.tools.listChanged",
+                )
+            )
+        path = ctx.options.baseline_path
+        if not path:
+            return findings
+        file = Path(path)
+        if file.exists():
+            try:
+                previous = json.loads(file.read_text("utf-8"))
+            except (OSError, ValueError) as exc:
+                ctx.notes.append(f"META-02: could not read baseline {path}: {exc}")
+                previous = None
+            if isinstance(previous, dict):
+                if previous.get("target") != ctx.target:
+                    ctx.notes.append(
+                        f"META-02: baseline target {previous.get('target')!r} differs from {ctx.target!r}"
+                    )
+                diff = diff_snapshots(previous, current)
+                findings.extend(self._findings_from_diff(ctx, diff, previous))
+        if ctx.options.write_baseline:
+            try:
+                file.parent.mkdir(parents=True, exist_ok=True)
+                file.write_text(json.dumps(current, indent=2, sort_keys=True), "utf-8")
+                ctx.notes.append(f"META-02: baseline written to {path}")
+            except OSError as exc:
+                ctx.notes.append(f"META-02: could not write baseline {path}: {exc}")
+        return findings
+
+    def _findings_from_diff(
+        self, ctx: ScanContext, diff: dict[str, Any], previous: dict[str, Any]
+    ) -> list[Finding]:
+        if not diff["any_change"]:
+            ctx.notes.append("META-02: no change since baseline " + str(previous.get("created_at")))
+            return []
+        out: list[Finding] = []
+        ev = [ctx.inventory.tools_evidence] if ctx.inventory.tools_evidence else []
+        tools = diff["tools"]
+        for name in tools["changed"]:
+            fields = tools.get("changed_fields", {}).get(name, [])
+            sev = Severity.MEDIUM if ("description" in fields or "inputSchema" in fields) else Severity.LOW
+            out.append(
+                self.finding(
+                    description=f"Tool '{name}' changed since baseline {previous.get('created_at')}: "
+                    f"fields {', '.join(fields) or 'unknown'}.",
+                    severity=sev,
+                    confidence=Confidence.HIGH,
+                    evidence=ev,
+                    location=f"tools[{name}]",
+                    details={"changed_fields": fields},
+                )
+            )
+        if tools["added"]:
+            out.append(
+                self.finding(
+                    title="New tools appeared since baseline",
+                    description="Added tools: " + ", ".join(tools["added"]) + ".",
+                    severity=Severity.LOW,
+                    confidence=Confidence.HIGH,
+                    evidence=ev,
+                    location="tools",
+                    details={"added": tools["added"]},
+                )
+            )
+        if tools["removed"]:
+            out.append(
+                self.finding(
+                    title="Tools removed since baseline",
+                    description="Removed tools: " + ", ".join(tools["removed"]) + ".",
+                    severity=Severity.INFO,
+                    confidence=Confidence.HIGH,
+                    evidence=ev,
+                    location="tools",
+                    details={"removed": tools["removed"]},
+                )
+            )
+        for kind in ("prompts", "resources"):
+            d = diff[kind]
+            if d["added"] or d["removed"] or d["changed"]:
+                out.append(
+                    self.finding(
+                        title=f"{kind.capitalize()} changed since baseline",
+                        description=f"{kind}: added {d['added']}, removed {d['removed']}, changed {d['changed']}.",
+                        severity=Severity.LOW,
+                        confidence=Confidence.HIGH,
+                        location=kind,
+                        details=d,
+                    )
+                )
+        if diff["instructions_changed"]:
+            out.append(
+                self.finding(
+                    title="Server instructions changed since baseline",
+                    description="The 'instructions' text returned at negotiation differs from the baseline.",
+                    severity=Severity.MEDIUM,
+                    confidence=Confidence.HIGH,
+                    location="initialize.instructions",
+                )
+            )
+        return out

mcpprobe/checks/capability.py

@@ -0,0 +1,171 @@
+"""Heuristic classification of tools into capability classes.
+
+Used by META-03 (annotation cross-check) and to weight the severity of other
+findings (an unauthenticated server exposing ``run_command`` is worse than one
+exposing ``get_weather``). Annotations are *not* used as input, because the
+spec says clients must treat them as untrusted.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+#: Classes considered dangerous when exposed without authentication.
+DANGEROUS_CLASSES: frozenset[str] = frozenset(
+    {"exec", "file_write", "database_write", "network", "email", "credentials", "payments"}
+)
+
+#: Classes that imply a write/side effect (should not be ``readOnlyHint: true``).
+WRITE_CLASSES: frozenset[str] = frozenset(
+    {"exec", "file_write", "database_write", "email", "payments", "delete"}
+)
+
+_RULES: list[tuple[str, re.Pattern[str]]] = [
+    (
+        "exec",
+        re.compile(
+            r"\b(exec(ute)?|shell|bash|sh|cmd|command|run_?(command|script|code)|"
+            r"subprocess|powershell|terminal|eval|spawn|system)\b",
+            re.I,
+        ),
+    ),
+    (
+        "file_write",
+        re.compile(
+            r"\b(write_?file|save_?file|create_?file|edit_?file|move_?file|"
+            r"rename|chmod|mkdir|upload|overwrite|append_?file)\b",
+            re.I,
+        ),
+    ),
+    ("delete", re.compile(r"\b(delete|remove|rm|drop|purge|destroy|truncate|wipe)\b", re.I)),
+    (
+        "file_read",
+        re.compile(
+            r"\b(read_?file|cat|open_?file|list_?dir(ectory)?|ls|glob|"
+            r"file_?(content|path)|filesystem|directory)\b",
+            re.I,
+        ),
+    ),
+    (
+        "database_write",
+        re.compile(
+            r"\b(insert|update|upsert|alter|create_?table|migrate|"
+            r"execute_?sql|run_?sql|sql_?(exec|write))\b",
+            re.I,
+        ),
+    ),
+    (
+        "database_read",
+        re.compile(
+            r"\b(sql|query|select|database|db|postgres|mysql|sqlite|mongo|"
+            r"redis|table|schema)\b",
+            re.I,
+        ),
+    ),
+    (
+        "network",
+        re.compile(
+            r"\b(fetch|http|url|curl|wget|download|request|browse|crawl|scrape|"
+            r"webhook|api_?call|proxy|ssrf)\b",
+            re.I,
+        ),
+    ),
+    (
+        "email",
+        re.compile(
+            r"\b(e-?mail|smtp|send_?(mail|message|sms|slack|notification)|"
+            r"bcc|cc|recipient|inbox|mailbox)\b",
+            re.I,
+        ),
+    ),
+    (
+        "credentials",
+        re.compile(
+            r"\b(secret|token|credential|password|api_?key|ssh|private_?key|"
+            r"keychain|vault|oauth|\.env)\b",
+            re.I,
+        ),
+    ),
+    (
+        "payments",
+        re.compile(
+            r"\b(payment|pay|charge|refund|invoice|stripe|billing|transfer_?funds|"
+            r"wallet|crypto)\b",
+            re.I,
+        ),
+    ),
+    ("search", re.compile(r"\b(search|find|lookup|get|list|read|fetch_?info|describe|show)\b", re.I)),
+]
+
+
+@dataclass(slots=True)
+class Classification:
+    name: str
+    classes: list[str] = field(default_factory=list)
+    matched_terms: dict[str, list[str]] = field(default_factory=dict)
+
+    @property
+    def dangerous(self) -> bool:
+        return any(c in DANGEROUS_CLASSES for c in self.classes)
+
+    @property
+    def writes(self) -> bool:
+        return any(c in WRITE_CLASSES for c in self.classes)
+
+    @property
+    def primary(self) -> str:
+        return self.classes[0] if self.classes else "other"
+
+
+def _text_of(tool: dict[str, Any]) -> tuple[str, str]:
+    """Return (name-ish text, description-ish text)."""
+    name_text = (
+        " ".join(str(tool.get(k, "")) for k in ("name", "title") if tool.get(k))
+        .replace("_", " ")
+        .replace("-", " ")
+        .replace(".", " ")
+    )
+    desc_parts = [str(tool.get("description", "") or "")]
+    schema = tool.get("inputSchema")
+    if isinstance(schema, dict):
+        props = schema.get("properties")
+        if isinstance(props, dict):
+            for pname, pdef in props.items():
+                desc_parts.append(str(pname).replace("_", " "))
+                if isinstance(pdef, dict) and pdef.get("description"):
+                    desc_parts.append(str(pdef["description"]))
+    return name_text, " ".join(desc_parts)
+
+
+def classify_tool(tool: dict[str, Any]) -> Classification:
+    """Classify one tool. Name matches weigh more than description matches."""
+    name_text, desc_text = _text_of(tool)
+    name = str(tool.get("name", ""))
+    scores: dict[str, int] = {}
+    terms: dict[str, list[str]] = {}
+    for cls, pattern in _RULES:
+        name_hits = pattern.findall(name_text + " " + name.replace("_", " "))
+        desc_hits = pattern.findall(desc_text)
+        hits = [h if isinstance(h, str) else h[0] for h in name_hits + desc_hits]
+        hits = [h for h in hits if h]
+        if not hits:
+            continue
+        score = 3 * len([h for h in name_hits if h]) + len([h for h in desc_hits if h])
+        scores[cls] = score
+        terms[cls] = sorted(set(hits))[:5]
+    # "search" is a weak catch-all; drop it when anything stronger matched.
+    if "search" in scores and len(scores) > 1:
+        scores.pop("search")
+        terms.pop("search", None)
+    # database_read is implied by database_write.
+    if "database_write" in scores and "database_read" in scores:
+        scores.pop("database_read")
+        terms.pop("database_read", None)
+    ordered = [c for c, _ in sorted(scores.items(), key=lambda kv: (-kv[1], kv[0]))]
+    return Classification(name=name, classes=ordered, matched_terms=terms)
+
+
+def classify_tools(tools: list[dict[str, Any]]) -> dict[str, Classification]:
+    return {str(t.get("name", "")): classify_tool(t) for t in tools}