mcpprobe-scanner 0.1.0__py3-none-any.whl

mcpprobe/transport/evidence.py

@@ -0,0 +1,135 @@
+"""Evidence collection: every request/response pair the scanner sends is logged."""
+
+from __future__ import annotations
+
+import itertools
+from collections.abc import Iterable, Mapping
+
+from mcpprobe.models import EVIDENCE_BODY_LIMIT, EvidenceRecord, utcnow
+from mcpprobe.redact import redact_secrets, redact_url
+
+#: Response headers worth keeping in evidence (lower-case).
+KEPT_RESPONSE_HEADERS: frozenset[str] = frozenset(
+    {
+        "content-type",
+        "content-length",
+        "www-authenticate",
+        "mcp-session-id",
+        "mcp-protocol-version",
+        "server",
+        "x-powered-by",
+        "strict-transport-security",
+        "x-content-type-options",
+        "x-frame-options",
+        "content-security-policy",
+        "cache-control",
+        "retry-after",
+        "location",
+        "access-control-allow-origin",
+        "access-control-allow-credentials",
+        "allow",
+        "date",
+    }
+)
+
+#: Request headers worth keeping in evidence (lower-case).
+KEPT_REQUEST_HEADERS: frozenset[str] = frozenset(
+    {
+        "accept",
+        "content-type",
+        "origin",
+        "host",
+        "user-agent",
+        "authorization",
+        "mcp-session-id",
+        "mcp-protocol-version",
+        "mcp-method",
+        "mcp-name",
+        "last-event-id",
+    }
+)
+
+REDACTED = "<redacted>"
+
+
+def _subset(headers: Mapping[str, str] | Iterable[tuple[str, str]], keep: frozenset[str]) -> dict[str, str]:
+    items = headers.items() if isinstance(headers, Mapping) else headers
+    out: dict[str, str] = {}
+    for key, value in items:
+        lk = key.lower()
+        if lk not in keep:
+            continue
+        if lk == "authorization":
+            scheme = value.split(" ", 1)[0] if value else ""
+            out[lk] = f"{scheme} {REDACTED}".strip()
+        else:
+            out[lk] = value
+    return out
+
+
+def truncate_body(body: bytes | str | None, limit: int = EVIDENCE_BODY_LIMIT) -> tuple[str, bool]:
+    """Return ``(text, truncated)`` for a body: secrets redacted, at most ``limit`` UTF-8 bytes."""
+    if body is None:
+        return "", False
+    raw = body if isinstance(body, bytes) else body.encode("utf-8", errors="replace")
+    truncated = len(raw) > limit
+    # Redact on a slightly larger window so a secret straddling the cut is still caught.
+    text = redact_secrets(raw[: limit * 2].decode("utf-8", errors="replace"))
+    encoded = text.encode("utf-8")
+    if len(encoded) > limit:
+        truncated = True
+        text = encoded[:limit].decode("utf-8", errors="ignore")
+    return text, truncated
+
+
+class EvidenceLog:
+    """Append-only list of :class:`EvidenceRecord` with sequential IDs."""
+
+    def __init__(self) -> None:
+        self._records: list[EvidenceRecord] = []
+        self._counter = itertools.count(1)
+
+    def record(
+        self,
+        *,
+        transport: str,
+        method: str,
+        url: str | None = None,
+        status: int | None = None,
+        request_headers: Mapping[str, str] | None = None,
+        response_headers: Mapping[str, str] | Iterable[tuple[str, str]] | None = None,
+        request_body: bytes | str | None = None,
+        response_body: bytes | str | None = None,
+        elapsed_ms: float | None = None,
+        note: str | None = None,
+        error: str | None = None,
+    ) -> EvidenceRecord:
+        """Create, store and return a new evidence record."""
+        req_text, req_trunc = truncate_body(request_body)
+        resp_text, resp_trunc = truncate_body(response_body)
+        rec = EvidenceRecord(
+            id=f"ev-{next(self._counter):04d}",
+            timestamp=utcnow(),
+            transport=transport,
+            method=method,
+            url=redact_url(url) if url else url,
+            status=status,
+            request_headers=_subset(request_headers or {}, KEPT_REQUEST_HEADERS),
+            response_headers=_subset(response_headers or {}, KEPT_RESPONSE_HEADERS),
+            request_body=req_text,
+            response_body=resp_text,
+            request_truncated=req_trunc,
+            response_truncated=resp_trunc,
+            elapsed_ms=elapsed_ms,
+            note=note,
+            error=error,
+        )
+        self._records.append(rec)
+        return rec
+
+    @property
+    def records(self) -> list[EvidenceRecord]:
+        return list(self._records)
+
+    def __len__(self) -> int:
+        return len(self._records)

mcpprobe/transport/fingerprint.py

@@ -0,0 +1,154 @@
+"""Protocol-version fingerprinting.
+
+Order of attempts (see the MCP versioning page and the 2026-07-28 changelog):
+
+1. ``server/discover`` with the 2026-07-28 ``_meta`` version (stateless revision).
+2. ``initialize`` with ``2025-11-25``, then ``2025-06-18`` (handshake revisions);
+   on success ``notifications/initialized`` is sent and any ``Mcp-Session-Id``
+   is remembered.
+3. For HTTP targets that answered 400/404/405 to every POST, a single GET is made
+   to detect the deprecated 2024-11-05 HTTP+SSE transport (``endpoint`` event).
+
+A 401 on any attempt marks the target as auth-protected and stops negotiation.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from mcpprobe.models import Fingerprint
+from mcpprobe.transport.base import (
+    CLIENT_INFO,
+    HANDSHAKE_VERSIONS,
+    PROTOCOL_2026_07_28,
+    RpcResponse,
+    Transport,
+)
+from mcpprobe.transport.http import HttpTransport, parse_sse_block
+
+LEGACY_STATUSES = {400, 404, 405}
+
+
+def _attempt(version: str, method: str, resp: RpcResponse) -> dict[str, Any]:
+    err = resp.rpc_error
+    return {
+        "version": version,
+        "method": method,
+        "status": resp.status,
+        "ok": resp.ok,
+        "rpc_error": (err or {}).get("message") if err else None,
+        "transport_error": resp.error_text,
+        "evidence": resp.evidence.id if resp.evidence else None,
+    }
+
+
+def _absorb_server_result(fp: Fingerprint, result: dict[str, Any]) -> None:
+    info = result.get("serverInfo") or result.get("identity") or result.get("server") or {}
+    if isinstance(info, dict):
+        fp.server_info = info
+    caps = result.get("capabilities")
+    if isinstance(caps, dict):
+        fp.capabilities = caps
+    instr = result.get("instructions")
+    if isinstance(instr, str):
+        fp.instructions = instr
+
+
+async def fingerprint(transport: Transport) -> Fingerprint:
+    """Negotiate with the target and return what was learned."""
+    fp = Fingerprint(transport=transport.name)
+    statuses: list[int | None] = []
+
+    # 1. Stateless 2026-07-28
+    resp = await transport.rpc(
+        "server/discover",
+        {},
+        protocol_version=PROTOCOL_2026_07_28,
+        note="fingerprint: 2026-07-28 server/discover",
+    )
+    fp.attempts.append(_attempt(PROTOCOL_2026_07_28, "server/discover", resp))
+    statuses.append(resp.status)
+    if _is_unauthorised(resp):
+        _mark_auth(fp, resp)
+        return fp
+    if resp.ok and resp.result is not None:
+        fp.flavour = "stateless-2026"
+        fp.protocol_version = PROTOCOL_2026_07_28
+        transport.protocol_version = PROTOCOL_2026_07_28
+        _absorb_server_result(fp, resp.result)
+        versions = (
+            resp.result.get("protocolVersions")
+            or resp.result.get("supportedVersions")
+            or resp.result.get("versions")
+        )
+        if versions:
+            fp.server_info = {**fp.server_info, "supportedVersions": versions}
+        return fp
+
+    # 2. Handshake revisions
+    for version in sorted(HANDSHAKE_VERSIONS, reverse=True):
+        resp = await transport.rpc(
+            "initialize",
+            {"protocolVersion": version, "capabilities": {}, "clientInfo": CLIENT_INFO},
+            protocol_version=version,
+            track_session=True,
+            note=f"fingerprint: initialize {version}",
+        )
+        fp.attempts.append(_attempt(version, "initialize", resp))
+        statuses.append(resp.status)
+        if _is_unauthorised(resp):
+            _mark_auth(fp, resp)
+            return fp
+        if resp.ok and resp.result is not None:
+            negotiated = resp.result.get("protocolVersion")
+            fp.protocol_version = str(negotiated) if negotiated else version
+            # Speak the version the server answered with, so later requests match.
+            transport.protocol_version = (
+                fp.protocol_version if fp.protocol_version in HANDSHAKE_VERSIONS else version
+            )
+            fp.flavour = "handshake"
+            _absorb_server_result(fp, resp.result)
+            if isinstance(transport, HttpTransport) and transport.session_id:
+                fp.session_id_seen = True
+            await transport.notify("notifications/initialized")
+            return fp
+
+    # 3. Legacy HTTP+SSE
+    if isinstance(transport, HttpTransport) and all(s in LEGACY_STATUSES for s in statuses if s):
+        resp = await transport.legacy_sse_probe()
+        fp.attempts.append(_attempt("2024-11-05", "GET", resp))
+        if _is_unauthorised(resp):
+            _mark_auth(fp, resp)
+            return fp
+        if resp.status == 200 and "text/event-stream" in resp.headers.get("content-type", ""):
+            first = _first_event_name(resp.body)
+            if first == "endpoint":
+                fp.flavour = "legacy-sse"
+                fp.protocol_version = "2024-11-05"
+                return fp
+
+    fp.error = "no supported protocol revision answered; see attempts"
+    return fp
+
+
+def _first_event_name(body: bytes) -> str | None:
+    lines = body.decode("utf-8", errors="replace").split("\n")
+    block: list[str] = []
+    for line in lines:
+        if line == "":
+            if block:
+                ev = parse_sse_block(block)
+                return ev.event if ev else None
+            continue
+        block.append(line)
+    ev = parse_sse_block(block) if block else None
+    return ev.event if ev else None
+
+
+def _is_unauthorised(resp: RpcResponse) -> bool:
+    return resp.status == 401
+
+
+def _mark_auth(fp: Fingerprint, resp: RpcResponse) -> None:
+    fp.auth_required = True
+    fp.www_authenticate = resp.headers.get("www-authenticate")

mcpprobe/transport/http.py

@@ -0,0 +1,366 @@
+"""Streamable HTTP transport (MCP 2025-03-26 onwards, including 2026-07-28).
+
+One endpoint, JSON-RPC over POST, responses either ``application/json`` or an
+SSE ``text/event-stream`` that is read until the matching response arrives.
+Every exchange is written to the evidence log.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+import httpx
+
+from mcpprobe import USER_AGENT
+from mcpprobe.transport.base import (
+    PROTOCOL_2026_07_28,
+    RpcResponse,
+    Transport,
+    TransportError,
+)
+from mcpprobe.transport.evidence import EvidenceLog
+from mcpprobe.transport.ratelimit import RateLimiter
+
+ACCEPT_BOTH = "application/json, text/event-stream"
+#: Hard cap on how many SSE bytes we buffer for one response.
+SSE_MAX_BYTES = 1_000_000
+
+
+class SseEvent:
+    """A parsed Server-Sent Event."""
+
+    __slots__ = ("data", "event", "id")
+
+    def __init__(self, event: str | None, data: str, event_id: str | None) -> None:
+        self.event = event
+        self.data = data
+        self.id = event_id
+
+
+def parse_sse_block(lines: Iterable[str]) -> SseEvent | None:
+    """Parse one blank-line-delimited SSE block into an event (``None`` if empty)."""
+    event: str | None = None
+    data_parts: list[str] = []
+    event_id: str | None = None
+    for line in lines:
+        if not line or line.startswith(":"):
+            continue
+        field, _, value = line.partition(":")
+        value = value.removeprefix(" ")
+        if field == "event":
+            event = value
+        elif field == "data":
+            data_parts.append(value)
+        elif field == "id":
+            event_id = value
+    if not data_parts and event is None:
+        return None
+    return SseEvent(event, "\n".join(data_parts), event_id)
+
+
+class HttpTransport(Transport):
+    """Permissive streamable-HTTP MCP client used by all HTTP checks."""
+
+    name = "http"
+
+    def __init__(
+        self,
+        url: str,
+        *,
+        extra_headers: Mapping[str, str] | None = None,
+        timeout: float = 15.0,
+        evidence: EvidenceLog | None = None,
+        limiter: RateLimiter | None = None,
+        verify: bool | str = True,
+        user_agent: str = USER_AGENT,
+    ) -> None:
+        super().__init__(evidence=evidence, limiter=limiter)
+        self.url = url
+        self.extra_headers = dict(extra_headers or {})
+        self.timeout = timeout
+        self.user_agent = user_agent
+        self.session_id: str | None = None
+        self.tls_version: str | None = None
+        self.sessions_seen: list[str] = []
+        self._client = httpx.AsyncClient(
+            timeout=httpx.Timeout(timeout),
+            verify=verify,
+            follow_redirects=False,
+            headers={"User-Agent": user_agent},
+        )
+
+    # ------------------------------------------------------------------ headers
+    def default_headers(
+        self,
+        method: str | None,
+        *,
+        protocol_version: str | None = None,
+        include_session: bool = True,
+    ) -> dict[str, str]:
+        """Headers a conformant client would send for ``method``."""
+        version = protocol_version or self.protocol_version
+        headers: dict[str, str] = {
+            "Accept": ACCEPT_BOTH,
+            "Content-Type": "application/json",
+            "User-Agent": self.user_agent,
+        }
+        if version:
+            headers["MCP-Protocol-Version"] = version
+        if version == PROTOCOL_2026_07_28 and method:
+            headers["Mcp-Method"] = method
+        if include_session and self.session_id and version != PROTOCOL_2026_07_28:
+            headers["Mcp-Session-Id"] = self.session_id
+        headers.update(self.extra_headers)
+        return headers
+
+    # ------------------------------------------------------------------ raw I/O
+    async def raw_request(
+        self,
+        http_method: str,
+        *,
+        body: bytes | str | None = None,
+        headers: Mapping[str, str] | None = None,
+        url: str | None = None,
+        label: str = "raw",
+        want_id: int | str | None = None,
+        note: str | None = None,
+        track_session: bool = False,
+        stream_timeout: float | None = None,
+    ) -> RpcResponse:
+        """Send an arbitrary HTTP request to the endpoint and record it.
+
+        If the response is an SSE stream it is read until a JSON-RPC message
+        with id ``want_id`` is seen (or the stream ends / times out).
+        """
+        await self.limiter.acquire()
+        target = url or self.url
+        hdrs = dict(headers or {})
+        data = body.encode("utf-8") if isinstance(body, str) else body
+        started = time.monotonic()
+        status: int | None = None
+        resp_headers: dict[str, str] = {}
+        collected = b""
+        message: dict[str, Any] | None = None
+        stream_messages: list[dict[str, Any]] = []
+        error_text: str | None = None
+        try:
+            async with self._client.stream(http_method, target, content=data, headers=hdrs) as resp:
+                status = resp.status_code
+                resp_headers = {k.lower(): v for k, v in resp.headers.items()}
+                self._capture_tls(resp)
+                ctype = resp_headers.get("content-type", "")
+                if "text/event-stream" in ctype:
+                    collected, stream_messages, message = await self._read_sse(
+                        resp, want_id, stream_timeout or self.timeout
+                    )
+                else:
+                    collected = await resp.aread()
+                    message = _parse_jsonrpc(collected, want_id)
+                    if message is not None:
+                        stream_messages = [message]
+        except httpx.HTTPError as exc:
+            error_text = f"{type(exc).__name__}: {exc}"
+        except TimeoutError:
+            error_text = "stream timeout"
+        elapsed_ms = (time.monotonic() - started) * 1000
+        if track_session:
+            sid = resp_headers.get("mcp-session-id")
+            if sid:
+                self.session_id = sid
+                self.sessions_seen.append(sid)
+        rec = self.evidence.record(
+            transport="http",
+            method=label,
+            url=target,
+            status=status,
+            request_headers=hdrs,
+            response_headers=resp_headers,
+            request_body=data,
+            response_body=collected,
+            elapsed_ms=elapsed_ms,
+            note=note,
+            error=error_text,
+        )
+        return RpcResponse(
+            method=label,
+            status=status,
+            headers=resp_headers,
+            body=collected,
+            message=message,
+            evidence=rec,
+            elapsed_ms=elapsed_ms,
+            error_text=error_text,
+            stream_messages=stream_messages,
+        )
+
+    def _capture_tls(self, resp: httpx.Response) -> None:
+        if self.tls_version is not None:
+            return
+        try:
+            stream = resp.extensions.get("network_stream")
+            ssl_object = stream.get_extra_info("ssl_object") if stream else None
+            if ssl_object is not None:
+                self.tls_version = ssl_object.version()
+        except Exception:  # pragma: no cover - best effort only
+            self.tls_version = None
+
+    async def _read_sse(
+        self,
+        resp: httpx.Response,
+        want_id: int | str | None,
+        timeout: float,
+    ) -> tuple[bytes, list[dict[str, Any]], dict[str, Any] | None]:
+        collected: list[str] = []
+        total = 0
+        block: list[str] = []
+        messages: list[dict[str, Any]] = []
+        matched: dict[str, Any] | None = None
+
+        async def consume() -> None:
+            nonlocal total, matched
+            async for line in resp.aiter_lines():
+                total += len(line) + 1
+                if total <= SSE_MAX_BYTES:
+                    collected.append(line)
+                if line == "":
+                    event = parse_sse_block(block)
+                    block.clear()
+                    if event is None or not event.data:
+                        continue
+                    msg = _parse_jsonrpc(event.data.encode("utf-8"), None)
+                    if msg is not None:
+                        messages.append(msg)
+                        if want_id is not None and msg.get("id") == want_id:
+                            matched = msg
+                            return
+                else:
+                    block.append(line)
+                if total > SSE_MAX_BYTES:
+                    return
+            if block:
+                event = parse_sse_block(block)
+                if event and event.data:
+                    msg = _parse_jsonrpc(event.data.encode("utf-8"), None)
+                    if msg is not None:
+                        messages.append(msg)
+                        if want_id is not None and msg.get("id") == want_id:
+                            matched = msg
+
+        try:
+            await asyncio.wait_for(consume(), timeout)
+        except TimeoutError:
+            pass
+        if matched is None and want_id is None and messages:
+            matched = messages[-1]
+        return "\n".join(collected).encode("utf-8"), messages, matched
+
+    # ------------------------------------------------------------------ JSON-RPC
+    async def rpc(
+        self,
+        method: str,
+        params: dict[str, Any] | None = None,
+        *,
+        note: str | None = None,
+        headers_override: Mapping[str, str] | None = None,
+        drop_headers: Iterable[str] = (),
+        protocol_version: str | None = None,
+        track_session: bool = False,
+        include_session: bool = True,
+        **kwargs: Any,
+    ) -> RpcResponse:
+        """POST a JSON-RPC request. ``headers_override`` lets checks send
+        non-conformant headers; ``drop_headers`` removes defaults by name."""
+        msg = self.build_message(method, params, protocol_version=protocol_version)
+        hdrs = self.default_headers(
+            method, protocol_version=protocol_version, include_session=include_session
+        )
+        for name in drop_headers:
+            hdrs = {k: v for k, v in hdrs.items() if k.lower() != name.lower()}
+        if headers_override:
+            for k, v in headers_override.items():
+                hdrs = {hk: hv for hk, hv in hdrs.items() if hk.lower() != k.lower()}
+                hdrs[k] = v
+        resp = await self.raw_request(
+            "POST",
+            body=json.dumps(msg),
+            headers=hdrs,
+            label=method,
+            want_id=msg["id"],
+            note=note,
+            track_session=track_session,
+        )
+        resp.method = method
+        return resp
+
+    async def notify(self, method: str, params: dict[str, Any] | None = None) -> None:
+        msg = self.build_message(method, params, notification=True)
+        await self.raw_request(
+            "POST",
+            body=json.dumps(msg),
+            headers=self.default_headers(method),
+            label=method,
+            note="notification",
+        )
+
+    async def get(
+        self,
+        url: str,
+        *,
+        headers: Mapping[str, str] | None = None,
+        note: str | None = None,
+        label: str = "GET",
+    ) -> RpcResponse:
+        """Plain GET (well-known documents, opt-out, legacy SSE probe)."""
+        hdrs = {"User-Agent": self.user_agent, "Accept": "application/json, text/plain, */*"}
+        hdrs.update(headers or {})
+        return await self.raw_request(
+            "GET", headers=hdrs, url=url, label=label, note=note, stream_timeout=3.0
+        )
+
+    async def legacy_sse_probe(self) -> RpcResponse:
+        """GET the endpoint as the deprecated 2024-11-05 HTTP+SSE transport would.
+
+        A legacy server answers with an ``endpoint`` event. We read only the
+        first event (3 s cap) and close the stream.
+        """
+        hdrs = {"User-Agent": self.user_agent, "Accept": "text/event-stream"}
+        hdrs.update(self.extra_headers)
+        return await self.raw_request(
+            "GET", headers=hdrs, label="GET(sse)", note="legacy HTTP+SSE probe", stream_timeout=3.0
+        )
+
+    async def close(self) -> None:
+        await self._client.aclose()
+
+
+def _parse_jsonrpc(body: bytes, want_id: int | str | None) -> dict[str, Any] | None:
+    """Parse a JSON-RPC response body (object or batch), picking the one for ``want_id``."""
+    if not body:
+        return None
+    try:
+        data = json.loads(body.decode("utf-8", errors="replace"))
+    except ValueError:
+        return None
+    if isinstance(data, list):
+        for item in data:
+            if isinstance(item, dict) and (want_id is None or item.get("id") == want_id):
+                return item
+        return None
+    if isinstance(data, dict) and "jsonrpc" in data:
+        return data
+    return None
+
+
+def ensure_url(target: str) -> str:
+    """Validate an HTTP(S) target."""
+    try:
+        parsed = httpx.URL(target)
+    except httpx.InvalidURL as exc:
+        raise TransportError(f"invalid URL: {target}") from exc
+    if parsed.scheme not in {"http", "https"} or not parsed.host:
+        raise TransportError(f"target must be an http(s) URL: {target}")
+    return str(parsed)

mcpprobe/transport/ratelimit.py

@@ -0,0 +1,47 @@
+"""A small asyncio token-bucket rate limiter.
+
+The default scan budget is 5 requests per second with a burst of one, so the
+cap also holds within any one-second window. Every outbound request the
+scanner makes (including well-known lookups and opt-out checks) passes through
+one limiter so the cap is global, not per check.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+
+DEFAULT_RATE = 5.0
+
+
+class RateLimiter:
+    """Token bucket: ``rate`` tokens per second, burst of ``burst`` tokens."""
+
+    def __init__(self, rate: float = DEFAULT_RATE, burst: int | None = None) -> None:
+        if rate <= 0:
+            raise ValueError("rate must be positive")
+        self.rate = float(rate)
+        self.capacity = float(burst if burst is not None else 1)
+        self._tokens = self.capacity
+        self._updated = time.monotonic()
+        self._lock = asyncio.Lock()
+        self.total_waits = 0.0
+        self.requests = 0
+
+    def _refill(self) -> None:
+        now = time.monotonic()
+        elapsed = now - self._updated
+        self._updated = now
+        self._tokens = min(self.capacity, self._tokens + elapsed * self.rate)
+
+    async def acquire(self) -> None:
+        """Wait until a token is available, then consume it."""
+        async with self._lock:
+            self._refill()
+            if self._tokens < 1.0:
+                wait = (1.0 - self._tokens) / self.rate
+                self.total_waits += wait
+                await asyncio.sleep(wait)
+                self._refill()
+            self._tokens -= 1.0
+            self.requests += 1