mcp-runtime-check 0.1.1__py3-none-any.whl

mcp_fuzz/__init__.py

@@ -0,0 +1,5 @@
+from mcp_fuzz.engine import CallOutcome, FuzzReport, ToolResult, run_fuzz
+from mcp_fuzz.report import Report, build_report
+
+__all__ = ["CallOutcome", "FuzzReport", "ToolResult", "run_fuzz", "Report", "build_report"]
+__version__ = "0.1.1"

mcp_fuzz/cli.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+
+from mcp_fuzz.engine import DEFAULT_TIMEOUT_SECONDS, run_fuzz
+from mcp_fuzz.report import build_report, render_text, to_dict
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="mcp-fuzz",
+        description=(
+            "Launches an MCP server over stdio and calls each read-only tool with "
+            "schema-derived valid, missing-required, and wrong-type inputs to check "
+            "whether it crashes, hangs, or returns a structured error."
+        ),
+    )
+    parser.add_argument(
+        "command",
+        nargs=argparse.REMAINDER,
+        help="the command (and its arguments) that launches the target MCP server, "
+        "e.g. `mcp-fuzz -- python server.py` or `mcp-fuzz -- npx -y some-mcp-server`",
+    )
+    parser.add_argument(
+        "--include-destructive",
+        action="store_true",
+        help="also test tools not annotated readOnlyHint=true. Off by default — see README's "
+        "Safety section before turning this on against a server with real side effects.",
+    )
+    parser.add_argument(
+        "--timeout", type=float, default=DEFAULT_TIMEOUT_SECONDS,
+        help=f"seconds to wait for a single tool call before treating it as a hang (default {DEFAULT_TIMEOUT_SECONDS})",
+    )
+    parser.add_argument("--json", action="store_true", help="emit machine-readable JSON instead of text")
+    parser.add_argument(
+        "--fail-under", type=float, default=None,
+        help="exit non-zero if the crash-resilience percent is below this threshold",
+    )
+    args = parser.parse_args()
+
+    command_parts = [c for c in args.command if c != "--"]
+    if not command_parts:
+        parser.error("no server command given — e.g. `mcp-fuzz -- python server.py`")
+
+    command, *rest = command_parts
+    raw = asyncio.run(run_fuzz(
+        command=command,
+        args=rest,
+        include_destructive=args.include_destructive,
+        timeout=args.timeout,
+    ))
+    report = build_report(raw)
+
+    if args.json:
+        print(json.dumps(to_dict(report), indent=2))
+    else:
+        print(render_text(report))
+
+    if report.connect_error:
+        sys.exit(2)
+    if args.fail_under is not None and (
+        report.crash_resilience_percent is None or report.crash_resilience_percent < args.fail_under
+    ):
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

mcp_fuzz/engine.py

@@ -0,0 +1,223 @@
+"""Connects to a real, running MCP server over stdio and calls each of its
+tools with schema-derived inputs to see how it actually behaves — distinct
+from static analysis (mcp-doctor), which never runs the code at all.
+
+Safety: a tool that isn't explicitly annotated `readOnlyHint: true` is
+skipped by default. This library has no way to know whether a "write"-shaped
+tool's side effects are safe to trigger against whatever backend the target
+server is actually configured against (a real database, a real inbox, a
+real filesystem) — silently calling it during a fuzz pass would be reckless
+regardless of how careful the input generation is. Pass
+`include_destructive=True` to opt into testing everything, at the caller's
+own risk.
+
+Isolation: any call that raises, times out, or otherwise leaves the
+transport in a bad state triggers a full reconnect (kill + relaunch the
+server subprocess) before the next case runs, so one tool crashing the
+server doesn't invalidate every result after it.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from contextlib import AsyncExitStack
+from dataclasses import dataclass, field
+
+from mcp import ClientSession, StdioServerParameters, types
+from mcp.client.stdio import stdio_client
+
+from mcp_fuzz.generator import (
+    generate_valid_arguments,
+    missing_required_variants,
+    wrong_type_variants,
+)
+
+DEFAULT_TIMEOUT_SECONDS = 15.0
+
+
+@dataclass
+class CallOutcome:
+    case: str  # "valid" | "missing_required" | "wrong_type"
+    property_name: str | None
+    outcome: str  # "ok" | "graceful_error" | "crash" | "timeout"
+    detail: str = ""
+
+
+@dataclass
+class ToolResult:
+    name: str
+    tested: bool
+    skip_reason: str | None = None
+    outcomes: list[CallOutcome] = field(default_factory=list)
+
+
+@dataclass
+class FuzzReport:
+    server_command: str
+    tools: list[ToolResult] = field(default_factory=list)
+    connect_error: str | None = None
+
+
+class _ServerConnection:
+    """One live stdio connection to the target server, reconnectable on
+    demand after a crash/timeout without tearing down the whole fuzz run."""
+
+    def __init__(self, params: StdioServerParameters):
+        self._params = params
+        self._stack: AsyncExitStack | None = None
+        self.session: ClientSession | None = None
+
+    async def connect(self) -> None:
+        await self.close()
+        stack = AsyncExitStack()
+        try:
+            read, write = await stack.enter_async_context(stdio_client(self._params))
+            session = await stack.enter_async_context(ClientSession(read, write))
+            await session.initialize()
+        except BaseException:
+            await stack.aclose()
+            raise
+        self._stack = stack
+        self.session = session
+
+    async def close(self) -> None:
+        if self._stack is not None:
+            try:
+                await self._stack.aclose()
+            except Exception:
+                pass  # best-effort teardown of a possibly-already-dead process
+        self._stack = None
+        self.session = None
+
+
+def _field(model, snake_name: str, camel_name: str):
+    """Reads a pydantic model field whose attribute name differs across
+    `mcp` SDK major versions: mcp<2.0 exposed several `types` fields under
+    their raw camelCase wire name directly (`isError`, `inputSchema`,
+    `readOnlyHint`, ...); mcp>=2.0 renamed them to snake_case
+    (`is_error`, `input_schema`, `read_only_hint`, ...) with the camelCase
+    kept only as a validation alias, not a readable attribute. Verified
+    directly: installing a real target server (`arxiv-mcp-server`, which
+    pins `mcp<2.0`) into the same environment as mcp-fuzz silently
+    downgraded the shared `mcp` package and broke every hardcoded
+    snake_case attribute access with an AttributeError. Since mcp-fuzz's
+    own resolved `mcp` version is independent of whatever the target
+    server uses, and either generation could end up installed here, try
+    the current name first and fall back to the older one rather than
+    assuming either."""
+    if hasattr(model, snake_name):
+        return getattr(model, snake_name)
+    return getattr(model, camel_name)
+
+
+def _is_read_only(tool: types.Tool) -> bool:
+    annotations = tool.annotations
+    if annotations is None:
+        return False
+    return _field(annotations, "read_only_hint", "readOnlyHint") is True
+
+
+async def _call_with_outcome(
+    conn: _ServerConnection,
+    params: StdioServerParameters,
+    tool_name: str,
+    case: str,
+    property_name: str | None,
+    arguments: dict,
+    timeout: float,
+) -> CallOutcome:
+    """Runs one tool call, classifying the result, and reconnects the shared
+    connection afterward if the call left it unusable."""
+    try:
+        assert conn.session is not None
+        result = await asyncio.wait_for(
+            conn.session.call_tool(tool_name, arguments), timeout=timeout
+        )
+    except asyncio.TimeoutError:
+        # `asyncio.wait_for` raises `asyncio.TimeoutError`. Python 3.11
+        # unified that with the builtin `TimeoutError` (same class), but on
+        # 3.10 they're still distinct — `except TimeoutError` alone misses
+        # it there and this falls through to the generic crash handler
+        # below, misclassifying a genuine timeout as a crash. Caught this
+        # via CI running 3.10 (mcp itself requires >=3.10), not locally,
+        # where dev happened to be on 3.11+.
+        await conn.connect()
+        return CallOutcome(case, property_name, "timeout", f"no response within {timeout}s")
+    except Exception as exc:
+        await conn.connect()
+        return CallOutcome(case, property_name, "crash", f"{type(exc).__name__}: {exc}")
+
+    if isinstance(result, types.CallToolResult) and _field(result, "is_error", "isError"):
+        outcome = "graceful_error" if case != "valid" else "ok"
+        # A "valid" call returning is_error is itself worth surfacing, but
+        # it's a content-level finding, not a crash — record it as an error
+        # outcome regardless of which case triggered it so the report shows
+        # the true state rather than papering over a valid-call failure.
+        if case == "valid":
+            outcome = "valid_call_errored"
+        text = "; ".join(
+            c.text for c in result.content if isinstance(c, types.TextContent)
+        )[:300]
+        return CallOutcome(case, property_name, outcome, text)
+
+    return CallOutcome(case, property_name, "ok")
+
+
+async def run_fuzz(
+    command: str,
+    args: list[str] | None = None,
+    env: dict[str, str] | None = None,
+    cwd: str | None = None,
+    include_destructive: bool = False,
+    timeout: float = DEFAULT_TIMEOUT_SECONDS,
+) -> FuzzReport:
+    params = StdioServerParameters(command=command, args=args or [], env=env, cwd=cwd)
+    server_label = " ".join([command, *(args or [])])
+    report = FuzzReport(server_command=server_label)
+
+    conn = _ServerConnection(params)
+    try:
+        await conn.connect()
+    except Exception as exc:
+        report.connect_error = f"{type(exc).__name__}: {exc}"
+        return report
+
+    try:
+        assert conn.session is not None
+        tools_result = await conn.session.list_tools()
+    except Exception as exc:
+        report.connect_error = f"failed to list tools: {type(exc).__name__}: {exc}"
+        await conn.close()
+        return report
+
+    for tool in tools_result.tools:
+        if not include_destructive and not _is_read_only(tool):
+            report.tools.append(ToolResult(
+                name=tool.name,
+                tested=False,
+                skip_reason="not annotated readOnlyHint=true (use include_destructive to test anyway)",
+            ))
+            continue
+
+        result = ToolResult(name=tool.name, tested=True)
+        schema = _field(tool, "input_schema", "inputSchema")
+
+        valid_args = generate_valid_arguments(schema)
+        result.outcomes.append(
+            await _call_with_outcome(conn, params, tool.name, "valid", None, valid_args, timeout)
+        )
+
+        for prop_name, args in missing_required_variants(schema):
+            result.outcomes.append(
+                await _call_with_outcome(conn, params, tool.name, "missing_required", prop_name, args, timeout)
+            )
+
+        for prop_name, args in wrong_type_variants(schema):
+            result.outcomes.append(
+                await _call_with_outcome(conn, params, tool.name, "wrong_type", prop_name, args, timeout)
+            )
+
+        report.tools.append(result)
+
+    await conn.close()
+    return report

mcp_fuzz/generator.py

@@ -0,0 +1,180 @@
+"""Generates test-call arguments from a tool's JSON input schema.
+
+Three kinds of test cases, all derived purely from the schema (no LLM, no
+network calls of its own):
+
+- a "valid" call: one plausible value per property, respecting `type`,
+  `enum`, `minimum`/`maximum`, `minLength`/`maxLength`, and `format` where
+  present, so a well-behaved tool should accept it without complaint.
+- a "missing required" call per required property: the valid call with that
+  one property removed, to check the server returns a structured error
+  instead of crashing or silently proceeding with an absent argument.
+- a "wrong type" call per property with an unambiguous `type`: the valid
+  call with that one property swapped for a value of a different JSON type,
+  same reasoning.
+
+Anything the schema doesn't pin down (no `type`, an `anyOf`/`oneOf` with
+genuinely different shapes, a `$ref` this module doesn't resolve) is left
+out of the wrong-type set rather than guessed at — a value that might
+legitimately be valid isn't a useful "wrong type" test case.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+_STRING_FORMAT_SAMPLES = {
+    "date": "2026-01-01",
+    "date-time": "2026-01-01T00:00:00Z",
+    "email": "test@example.com",
+    "uri": "https://example.com",
+    "uuid": "00000000-0000-0000-0000-000000000000",
+    "hostname": "example.com",
+    "ipv4": "127.0.0.1",
+    "ipv6": "::1",
+}
+
+
+def _sample_string(schema: dict[str, Any]) -> str:
+    if "enum" in schema and schema["enum"]:
+        return schema["enum"][0]
+    fmt = schema.get("format")
+    if fmt in _STRING_FORMAT_SAMPLES:
+        return _STRING_FORMAT_SAMPLES[fmt]
+    base = "test"
+    min_len = schema.get("minLength")
+    if isinstance(min_len, int) and min_len > len(base):
+        base = base + "x" * (min_len - len(base))
+    max_len = schema.get("maxLength")
+    if isinstance(max_len, int) and max_len < len(base):
+        base = base[:max_len] if max_len > 0 else ""
+    return base
+
+
+def _sample_number(schema: dict[str, Any], integer: bool) -> int | float:
+    minimum = schema.get("minimum", schema.get("exclusiveMinimum"))
+    maximum = schema.get("maximum", schema.get("exclusiveMaximum"))
+    if isinstance(minimum, (int, float)):
+        value = minimum + (1 if "exclusiveMinimum" in schema else 0)
+    elif isinstance(maximum, (int, float)):
+        value = maximum - (1 if "exclusiveMaximum" in schema else 0)
+    else:
+        value = 1
+    return int(value) if integer else float(value)
+
+
+def generate_valid_value(schema: dict[str, Any]) -> Any:
+    """One plausible value for a single property's schema fragment."""
+    if "const" in schema:
+        return schema["const"]
+    if "default" in schema:
+        return schema["default"]
+    if "enum" in schema and schema["enum"]:
+        return schema["enum"][0]
+    for combinator in ("anyOf", "oneOf"):
+        if combinator in schema and schema[combinator]:
+            return generate_valid_value(schema[combinator][0])
+    schema_type = schema.get("type")
+    if isinstance(schema_type, list):
+        schema_type = next((t for t in schema_type if t != "null"), schema_type[0])
+    if schema_type == "string":
+        return _sample_string(schema)
+    if schema_type == "integer":
+        return _sample_number(schema, integer=True)
+    if schema_type == "number":
+        return _sample_number(schema, integer=False)
+    if schema_type == "boolean":
+        return True
+    if schema_type == "array":
+        items_schema = schema.get("items", {})
+        min_items = schema.get("minItems", 1) or 1
+        sample = generate_valid_value(items_schema) if isinstance(items_schema, dict) else "test"
+        return [sample for _ in range(max(min_items, 1))]
+    if schema_type == "object":
+        return generate_valid_object(schema)
+    if schema_type == "null":
+        return None
+    # No usable type information — a generic placeholder is better than
+    # omitting the property outright, which would itself look like a
+    # missing-required-field test rather than a valid call.
+    return "test"
+
+
+def generate_valid_object(schema: dict[str, Any]) -> dict[str, Any]:
+    """A plausible object for an `{"type": "object", "properties": {...}}`
+    schema — every `required` property filled in, plus any optional
+    property that itself has an `enum`/`const`/`default` (cheap to include,
+    makes the "valid" call more representative)."""
+    properties = schema.get("properties", {})
+    required = set(schema.get("required", []))
+    result: dict[str, Any] = {}
+    for name, prop_schema in properties.items():
+        if not isinstance(prop_schema, dict):
+            continue
+        if name in required or any(k in prop_schema for k in ("enum", "const", "default")):
+            result[name] = generate_valid_value(prop_schema)
+    return result
+
+
+def generate_valid_arguments(input_schema: dict[str, Any] | None) -> dict[str, Any]:
+    """The full argument dict for a tool call that should succeed."""
+    if not input_schema:
+        return {}
+    return generate_valid_object(input_schema)
+
+
+def missing_required_variants(input_schema: dict[str, Any] | None) -> list[tuple[str, dict[str, Any]]]:
+    """(property_name, arguments) for each required property, omitted one
+    at a time from an otherwise-valid call."""
+    if not input_schema:
+        return []
+    required = input_schema.get("required", [])
+    if not required:
+        return []
+    base = generate_valid_arguments(input_schema)
+    variants = []
+    for name in required:
+        args = dict(base)
+        args.pop(name, None)
+        variants.append((name, args))
+    return variants
+
+
+_WRONG_TYPE_SAMPLES: dict[str, Any] = {
+    "string": 12345,
+    "integer": "not-a-number",
+    "number": "not-a-number",
+    "boolean": "not-a-boolean",
+    "array": "not-an-array",
+    "object": "not-an-object",
+}
+
+
+def _wrong_type_value(schema: dict[str, Any]) -> Any | None:
+    schema_type = schema.get("type")
+    if isinstance(schema_type, list) or schema_type is None:
+        return None
+    return _WRONG_TYPE_SAMPLES.get(schema_type)
+
+
+def wrong_type_variants(input_schema: dict[str, Any] | None) -> list[tuple[str, dict[str, Any]]]:
+    """(property_name, arguments) for each property whose schema has an
+    unambiguous single `type`, with that one property swapped to a value of
+    a different JSON type in an otherwise-valid call."""
+    if not input_schema:
+        return []
+    properties = input_schema.get("properties", {})
+    if not properties:
+        return []
+    base = generate_valid_arguments(input_schema)
+    variants = []
+    for name, prop_schema in properties.items():
+        if not isinstance(prop_schema, dict):
+            continue
+        wrong = _wrong_type_value(prop_schema)
+        if wrong is None:
+            continue
+        args = dict(base)
+        args[name] = wrong
+        variants.append((name, args))
+    return variants

mcp_fuzz/report.py

@@ -0,0 +1,194 @@
+"""Turns a raw FuzzReport into a scored, human- or JSON-readable report.
+
+The score is deliberately narrow: it's a *crash-resilience* score — the
+fraction of deliberately-bad-input calls (a missing required field, a
+wrong-typed field) that the server handled with a structured error instead
+of crashing or hanging. It does NOT grade whether a tool's "valid" call
+produced a *correct* result, since a synthetic, schema-only-derived value
+(e.g. a placeholder string for a field that's really supposed to be a real
+arXiv ID or a reachable URL) commonly isn't realistic enough for that to be
+a fair judgment — see `mcp_fuzz.engine`'s "valid_call_errored" outcome,
+which is reported separately as "worth investigating", not folded into the
+score, precisely because it can be a false positive from unrealistic
+synthetic data rather than a real tool bug.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from mcp_fuzz.engine import CallOutcome, FuzzReport, ToolResult
+
+BAD_INPUT_CASES = {"missing_required", "wrong_type"}
+
+
+@dataclass
+class ToolReport:
+    name: str
+    tested: bool
+    skip_reason: str | None
+    crashes: list[CallOutcome] = field(default_factory=list)
+    timeouts: list[CallOutcome] = field(default_factory=list)
+    valid_call_issue: CallOutcome | None = None
+    bad_input_case_count: int = 0
+
+
+@dataclass
+class Report:
+    server_command: str
+    connect_error: str | None
+    tools: list[ToolReport]
+    tested_count: int
+    skipped_count: int
+    total_bad_input_cases: int
+    crash_count: int
+    timeout_count: int
+    crash_resilience_percent: float | None
+    grade: str | None
+
+
+def _grade_for_percent(pct: float) -> str:
+    if pct >= 97:
+        return "A"
+    if pct >= 90:
+        return "B"
+    if pct >= 75:
+        return "C"
+    if pct >= 50:
+        return "D"
+    return "F"
+
+
+def build_report(raw: FuzzReport) -> Report:
+    tool_reports: list[ToolReport] = []
+    total_bad_input = 0
+    total_crashes = 0
+    total_timeouts = 0
+    tested_count = 0
+    skipped_count = 0
+
+    for tool in raw.tools:
+        if not tool.tested:
+            skipped_count += 1
+            tool_reports.append(ToolReport(
+                name=tool.name, tested=False, skip_reason=tool.skip_reason,
+            ))
+            continue
+
+        tested_count += 1
+        tr = ToolReport(name=tool.name, tested=True, skip_reason=None)
+        for outcome in tool.outcomes:
+            if outcome.case == "valid":
+                if outcome.outcome in ("crash", "timeout", "valid_call_errored"):
+                    tr.valid_call_issue = outcome
+                continue
+            if outcome.case not in BAD_INPUT_CASES:
+                continue
+            tr.bad_input_case_count += 1
+            total_bad_input += 1
+            if outcome.outcome == "crash":
+                tr.crashes.append(outcome)
+                total_crashes += 1
+            elif outcome.outcome == "timeout":
+                tr.timeouts.append(outcome)
+                total_timeouts += 1
+        tool_reports.append(tr)
+
+    if total_bad_input > 0:
+        percent = 100.0 * (1 - (total_crashes + total_timeouts) / total_bad_input)
+        grade = _grade_for_percent(percent)
+    else:
+        percent = None
+        grade = None
+
+    return Report(
+        server_command=raw.server_command,
+        connect_error=raw.connect_error,
+        tools=tool_reports,
+        tested_count=tested_count,
+        skipped_count=skipped_count,
+        total_bad_input_cases=total_bad_input,
+        crash_count=total_crashes,
+        timeout_count=total_timeouts,
+        crash_resilience_percent=percent,
+        grade=grade,
+    )
+
+
+def render_text(report: Report) -> str:
+    lines: list[str] = []
+    if report.connect_error:
+        lines.append(f"Failed to connect: {report.connect_error}")
+        return "\n".join(lines)
+
+    lines.append(f"mcp-fuzz: {report.server_command}")
+    lines.append("")
+    if report.crash_resilience_percent is not None:
+        lines.append(
+            f"Crash resilience: {report.crash_resilience_percent:.0f}% ({report.grade}) "
+            f"— {report.crash_count} crash(es), {report.timeout_count} timeout(s) "
+            f"across {report.total_bad_input_cases} bad-input calls"
+        )
+    else:
+        lines.append("Crash resilience: n/a (no testable tools had any parameters to fuzz)")
+    lines.append(f"Tested {report.tested_count} tool(s), skipped {report.skipped_count} (not read-only)")
+    lines.append("")
+
+    for tool in report.tools:
+        if not tool.tested:
+            lines.append(f"  [skip] {tool.name} — {tool.skip_reason}")
+            continue
+        flags = []
+        if tool.crashes:
+            flags.append(f"{len(tool.crashes)} crash(es)")
+        if tool.timeouts:
+            flags.append(f"{len(tool.timeouts)} timeout(s)")
+        if tool.valid_call_issue:
+            flags.append(f"valid call: {tool.valid_call_issue.outcome}")
+        marker = "FAIL" if (tool.crashes or tool.timeouts) else ("WARN" if tool.valid_call_issue else "ok")
+        summary = f" — {'; '.join(flags)}" if flags else ""
+        lines.append(f"  [{marker}] {tool.name}{summary}")
+        for outcome in tool.crashes + tool.timeouts:
+            lines.append(f"      {outcome.case} ({outcome.property_name}): {outcome.detail}")
+        if tool.valid_call_issue:
+            lines.append(
+                f"      valid call — {tool.valid_call_issue.outcome}: {tool.valid_call_issue.detail} "
+                "(may be a synthetic-input false positive, not a confirmed bug — see README)"
+            )
+
+    return "\n".join(lines)
+
+
+def to_dict(report: Report) -> dict:
+    return {
+        "server_command": report.server_command,
+        "connect_error": report.connect_error,
+        "tested_count": report.tested_count,
+        "skipped_count": report.skipped_count,
+        "total_bad_input_cases": report.total_bad_input_cases,
+        "crash_count": report.crash_count,
+        "timeout_count": report.timeout_count,
+        "crash_resilience_percent": report.crash_resilience_percent,
+        "grade": report.grade,
+        "tools": [
+            {
+                "name": t.name,
+                "tested": t.tested,
+                "skip_reason": t.skip_reason,
+                "crashes": [_outcome_dict(o) for o in t.crashes],
+                "timeouts": [_outcome_dict(o) for o in t.timeouts],
+                "valid_call_issue": _outcome_dict(t.valid_call_issue) if t.valid_call_issue else None,
+                "bad_input_case_count": t.bad_input_case_count,
+            }
+            for t in report.tools
+        ],
+    }
+
+
+def _outcome_dict(outcome: CallOutcome) -> dict:
+    return {
+        "case": outcome.case,
+        "property": outcome.property_name,
+        "outcome": outcome.outcome,
+        "detail": outcome.detail,
+    }

mcp_runtime_check-0.1.1.dist-info/METADATA

@@ -0,0 +1,75 @@
+Metadata-Version: 2.5
+Name: mcp-runtime-check
+Version: 0.1.1
+Summary: Runtime behavioral testing for MCP servers: calls every tool with schema-derived inputs and checks whether it actually behaves the way its description and schema claim.
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: mcp>=1.0
+Description-Content-Type: text/markdown
+
+# mcp-fuzz
+
+Runtime behavioral testing for [MCP](https://modelcontextprotocol.io) servers.
+
+[`mcp-doctor`](https://github.com/vishalhabib99/mcp-doctor) reads an MCP server's *source code* and checks whether its tools are well-documented. `mcp-fuzz` does the opposite: it actually **launches the server and calls its tools**, with inputs derived from each tool's own declared JSON schema, and checks whether the server behaves the way that schema and its description claim — does a missing required field get a structured error back, or does the server crash? Does a wrong-typed field get rejected cleanly, or does it hang?
+
+Static analysis can't see any of that. Only running the code can.
+
+## Install
+
+```bash
+pip install mcp-runtime-check
+```
+
+(The PyPI *distribution* name is `mcp-runtime-check` — `mcp-fuzz` and close variants were blocked by PyPI's anti-typosquat check as too similar to existing packages, same naming friction mcp-doctor hit. The installed CLI command is still `mcp-fuzz`, and the repo/import package are unchanged.)
+
+## Use
+
+```bash
+mcp-fuzz -- python server.py
+mcp-fuzz -- npx -y some-mcp-server
+```
+
+`mcp-fuzz` launches the command you give it as an MCP server over stdio, lists its tools, and for each one runs three kinds of calls built purely from that tool's own `inputSchema` — no LLM, no network calls of its own:
+
+- **valid** — one plausible value per property (respecting `type`, `enum`, `minimum`/`maximum`, `format`, ...) — should succeed.
+- **missing required** — the valid call, with one required property removed at a time — should come back as a structured MCP error, not a crash.
+- **wrong type** — the valid call, with one property swapped to a value of a different JSON type — same expectation.
+
+Any call that crashes the server, hangs past `--timeout` (default 15s), or leaves the connection unusable triggers a full reconnect before the next case runs, so one bad tool doesn't invalidate the rest of the report.
+
+## Safety — read this before pointing it at anything real
+
+`mcp-fuzz` actually **executes** tool calls. Unlike mcp-doctor, it has real side effects if a tool does. By default, **only tools annotated `readOnlyHint: true` are tested** — everything else is skipped and listed as such in the report. Pass `--include-destructive` to test everything, but only against a server you're confident is safe to call blindly (a local sandbox, a test/staging backend) — never a server wired to production data, a real inbox, a real payment system, etc. Many real-world servers don't set `readOnlyHint` accurately or at all, in which case those tools are conservatively skipped rather than assumed safe.
+
+## What the score means
+
+The reported "crash resilience" percentage covers only the **missing-required** and **wrong-type** cases — the fraction that came back as a structured error instead of a crash or hang. It does **not** grade whether the tool's "valid" call produced a *correct* result: a synthetic, schema-only-derived value (a placeholder string where the field really expects a real arXiv ID, or a URL that has to actually resolve) often isn't realistic enough for a failure there to be a fair judgment. A failed "valid" call is reported separately, flagged explicitly as **"may be a synthetic-input false positive, not a confirmed bug"** — worth a manual look, not proof of a bug.
+
+## JSON output / CI
+
+```bash
+mcp-fuzz --json -- python server.py
+mcp-fuzz --fail-under 90 -- python server.py   # non-zero exit if crash resilience < 90%
+```
+
+## Real-world spot check
+
+| Repo | Stars | Lang | What mcp-fuzz found |
+|---|---|---|---|
+| [`modelcontextprotocol/server-everything`](https://github.com/modelcontextprotocol/servers/tree/main/src/everything) | — | TS | Official reference server, run cross-language via `npx`. Clean pass — 9/9 read-only tools handled every bad-input case cleanly (100%/A). `trigger-long-running-operation`'s valid call correctly timed out — it's deliberately a long-running operation, exactly the kind of result the report's own "may be a false positive" framing exists for. |
+| [`blazickjp/arxiv-mcp-server`](https://github.com/blazickjp/arxiv-mcp-server) | 3.1k | Python | Found a real bug in mcp-fuzz itself, not the target: installing this repo (which pins `mcp<2.0`) into the same environment downgraded the shared `mcp` package from 2.1.1 to 1.29.1. mcp<2.0 exposes several fields under their raw camelCase wire name (`isError`, `inputSchema`, `readOnlyHint`); mcp>=2.0 renamed them to snake_case. Every hardcoded snake_case attribute access broke with an `AttributeError` the moment an older `mcp` happened to be installed. Fixed with a small compatibility helper that tries the current name first, falls back to the old one. 11 tools tested cleanly afterward (100%/A) — the several "valid call errored" flags are exactly the documented synthetic-input false-positive case (a placeholder `"paper_id": "test"` isn't a real arXiv ID). |
+| [`punitarani/fli`](https://github.com/punitarani/fli) | — | Python | Clean pass — all 4 read-only tools (Google Flights MCP) handled every bad-input case cleanly, and even the synthetic "valid" inputs succeeded without error (100%/A, no "worth investigating" flags at all). |
+| [`modelcontextprotocol/server-sequential-thinking`](https://github.com/modelcontextprotocol/servers/tree/main/src/sequentialthinking) | — | TS | Official reference server. Clean pass, tested with `--include-destructive` (its one tool isn't read-only-annotated but has no real side effects) — 100%/A. |
+| [`modelcontextprotocol/server-filesystem`](https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem) | — | TS | Official reference server, run against a throwaway sandbox directory. Clean pass — 6 of 10 read-only tools "valid call errored" on `ENOENT: no such file`, exactly the documented synthetic-input false positive: a generic placeholder string isn't a real path that exists in the sandbox. The 4 write/edit/move/create tools were correctly skipped as not read-only. |
+
+## Known limitations
+
+- Input generation is schema-only. A property with no `type` (or a genuinely ambiguous `anyOf`) is skipped from the wrong-type test set rather than guessed at.
+- No semantic check of *what* a successful response actually contains — that's a deliberately separate, opt-in, LLM-backed capability planned for a later release, not v1.
+- stdio transport only for now; no HTTP/SSE servers yet.
+
+## License
+
+MIT

mcp_runtime_check-0.1.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+mcp_fuzz/__init__.py,sha256=xkQ8E7UFbpxmNjtF-jAxUzM7i8x-7ubdUKPsa2OYTWc,238
+mcp_fuzz/cli.py,sha256=2kh7JjuYWonu_4NNqC7YD8kj24vHR5fEcpMAdrg_Hdk,2424
+mcp_fuzz/engine.py,sha256=7f0TIFJzDo10ACEElOmE5uY_mUEdeniU8ruMiABP9ks,8504
+mcp_fuzz/generator.py,sha256=rpT-KpFg6uIOg6bCEIkgUUQLfUfyw0ZZllHtCFTpJ48,6904
+mcp_fuzz/report.py,sha256=rBpG6L7OmRUeYhOBDN8lpjXZ_djGctEy-n-U1zNrhbI,6840
+mcp_runtime_check-0.1.1.dist-info/METADATA,sha256=hUpJSL66KJAYDwVJcSrhdI3DexXyckVKW1Bjk-5E_CI,6852
+mcp_runtime_check-0.1.1.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+mcp_runtime_check-0.1.1.dist-info/entry_points.txt,sha256=VLWVntGQU2GrM2dxpt-pN3VtSzBR8EA8-qfBwKeoQ-s,47
+mcp_runtime_check-0.1.1.dist-info/licenses/LICENSE,sha256=onEcx2Q9IzcIF62m2mPbmrqPzFbz80DHBUsPoCIoLe4,1069
+mcp_runtime_check-0.1.1.dist-info/RECORD,,

mcp_runtime_check-0.1.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

mcp_runtime_check-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+mcp-fuzz = mcp_fuzz.cli:main

mcp_runtime_check-0.1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Vishal Habib
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.