mcp-lighthouse 0.1.0__tar.gz

mcp_lighthouse-0.1.0/.gitignore

@@ -0,0 +1,6 @@
+__pycache__/
+*.pyc
+.venv/
+dist/
+*.egg-info/
+GEMINI.md

mcp_lighthouse-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Makito Chiba
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mcp_lighthouse-0.1.0/PKG-INFO

@@ -0,0 +1,146 @@
+Metadata-Version: 2.4
+Name: mcp-lighthouse
+Version: 0.1.0
+Summary: Audit tool for MCP servers — test protocol compliance, schema quality, and robustness
+Author: Makito Chiba
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai-agent,audit,mcp,model-context-protocol,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: rich>=13.0
+Description-Content-Type: text/markdown
+
+# MCP Lighthouse
+
+Audit tool for [MCP](https://modelcontextprotocol.io) servers. Run 21 automated checks across 5 dimensions and get a compliance score — like Lighthouse, but for your MCP server.
+
+```
+MCP Lighthouse — my-server v1.0
+
+  Protocol    ████████████████████  100
+  Schema      ██████████████░░░░░░   70
+  Robustness  ████████████████░░░░   75
+  Practices   ██████████░░░░░░░░░░   50
+  Performance ████████████████████  100
+
+  Overall Score: 83/100
+
+  21 checks: 17 passed, 2 warnings, 2 failed
+```
+
+## Why
+
+You built an MCP server. It works in Claude Code. But does it:
+- Return proper JSON-RPC 2.0 responses?
+- Include `inputSchema` on every tool?
+- Handle invalid tool names without crashing?
+- Respond to `initialize` within a reasonable time?
+
+MCP Lighthouse tests all of this automatically.
+
+## Install
+
+```bash
+pip install mcp-lighthouse
+```
+
+Or from source:
+
+```bash
+git clone https://github.com/MakiDevelop/mcp-lighthouse.git
+cd mcp-lighthouse
+pip install -e .
+```
+
+## Quick Start
+
+```bash
+# Audit an MCP server via stdio
+mcp-lighthouse scan --stdio "python my_server.py"
+mcp-lighthouse scan --stdio "npx @modelcontextprotocol/server-filesystem /"
+
+# Only run specific category
+mcp-lighthouse scan --stdio "python my_server.py" --category protocol
+
+# Export markdown report
+mcp-lighthouse scan --stdio "python my_server.py" --report audit.md
+
+# List all checks
+mcp-lighthouse list
+```
+
+## Checks (21 total)
+
+### Protocol Compliance (5 checks, 40% weight) — critical
+
+| Check | What it tests |
+|-------|---------------|
+| `proto-init` | Server responds to `initialize` with valid protocolVersion + capabilities + serverInfo |
+| `proto-init-version` | protocolVersion is a known version (2024-11-05, 2025-03-26, 2025-06-18, 2025-11-25) |
+| `proto-jsonrpc-version` | All responses include `"jsonrpc": "2.0"` |
+| `proto-id-match` | Response `id` matches request `id` |
+| `proto-error-format` | Error responses have `code` (int) + `message` (string) |
+
+### Schema Quality (6 checks, 25% weight) — warning
+
+| Check | What it tests |
+|-------|---------------|
+| `schema-tools-list` | `tools/list` returns non-empty tools array |
+| `schema-tool-name` | Every tool has a non-empty `name` |
+| `schema-tool-description` | Every tool has a description (>10 chars) |
+| `schema-tool-input-schema` | Every tool has `inputSchema` with `type: "object"` |
+| `schema-required-fields` | `inputSchema` with properties has a `required` array |
+| `schema-no-duplicate-tools` | No duplicate tool names |
+
+### Robustness (4 checks, 20% weight) — warning
+
+| Check | What it tests |
+|-------|---------------|
+| `robust-unknown-method` | Server returns `-32601` for unknown method |
+| `robust-invalid-tool` | `tools/call` with non-existent tool returns error (not crash) |
+| `robust-missing-args` | `tools/call` with missing required args returns error |
+| `robust-malformed-json` | Server handles malformed JSON without crashing |
+
+### Best Practices (4 checks, 10% weight) — info
+
+| Check | What it tests |
+|-------|---------------|
+| `bp-tool-name-format` | Tool names use snake_case or kebab-case |
+| `bp-description-length` | Tool descriptions are 20-500 chars |
+| `bp-server-info` | `serverInfo` includes both `name` and `version` |
+| `bp-capabilities-declared` | Server declares at least one capability |
+
+### Performance (2 checks, 5% weight) — info
+
+| Check | What it tests |
+|-------|---------------|
+| `perf-init-time` | `initialize` completes in < 5 seconds |
+| `perf-tools-list-time` | `tools/list` responds in < 3 seconds |
+
+## Scoring
+
+- **Overall**: Weighted average of category scores (protocol 40%, schema 25%, robustness 20%, practices 10%, performance 5%)
+- **Per category**: (passed checks / total checks) * 100
+- A single critical failure in Protocol drops that category to 0
+
+## CLI Reference
+
+```
+mcp-lighthouse scan [OPTIONS]
+  --stdio COMMAND       Server command to spawn (required for now)
+  --category CATEGORY   Only run checks in this category
+  --timeout SECONDS     Per-check timeout (default: 10)
+  --verbose             Show detailed output
+  --report PATH         Write markdown report
+
+mcp-lighthouse list
+  Lists all available checks
+```
+
+## License
+
+MIT

mcp_lighthouse-0.1.0/README.md

@@ -0,0 +1,130 @@
+# MCP Lighthouse
+
+Audit tool for [MCP](https://modelcontextprotocol.io) servers. Run 21 automated checks across 5 dimensions and get a compliance score — like Lighthouse, but for your MCP server.
+
+```
+MCP Lighthouse — my-server v1.0
+
+  Protocol    ████████████████████  100
+  Schema      ██████████████░░░░░░   70
+  Robustness  ████████████████░░░░   75
+  Practices   ██████████░░░░░░░░░░   50
+  Performance ████████████████████  100
+
+  Overall Score: 83/100
+
+  21 checks: 17 passed, 2 warnings, 2 failed
+```
+
+## Why
+
+You built an MCP server. It works in Claude Code. But does it:
+- Return proper JSON-RPC 2.0 responses?
+- Include `inputSchema` on every tool?
+- Handle invalid tool names without crashing?
+- Respond to `initialize` within a reasonable time?
+
+MCP Lighthouse tests all of this automatically.
+
+## Install
+
+```bash
+pip install mcp-lighthouse
+```
+
+Or from source:
+
+```bash
+git clone https://github.com/MakiDevelop/mcp-lighthouse.git
+cd mcp-lighthouse
+pip install -e .
+```
+
+## Quick Start
+
+```bash
+# Audit an MCP server via stdio
+mcp-lighthouse scan --stdio "python my_server.py"
+mcp-lighthouse scan --stdio "npx @modelcontextprotocol/server-filesystem /"
+
+# Only run specific category
+mcp-lighthouse scan --stdio "python my_server.py" --category protocol
+
+# Export markdown report
+mcp-lighthouse scan --stdio "python my_server.py" --report audit.md
+
+# List all checks
+mcp-lighthouse list
+```
+
+## Checks (21 total)
+
+### Protocol Compliance (5 checks, 40% weight) — critical
+
+| Check | What it tests |
+|-------|---------------|
+| `proto-init` | Server responds to `initialize` with valid protocolVersion + capabilities + serverInfo |
+| `proto-init-version` | protocolVersion is a known version (2024-11-05, 2025-03-26, 2025-06-18, 2025-11-25) |
+| `proto-jsonrpc-version` | All responses include `"jsonrpc": "2.0"` |
+| `proto-id-match` | Response `id` matches request `id` |
+| `proto-error-format` | Error responses have `code` (int) + `message` (string) |
+
+### Schema Quality (6 checks, 25% weight) — warning
+
+| Check | What it tests |
+|-------|---------------|
+| `schema-tools-list` | `tools/list` returns non-empty tools array |
+| `schema-tool-name` | Every tool has a non-empty `name` |
+| `schema-tool-description` | Every tool has a description (>10 chars) |
+| `schema-tool-input-schema` | Every tool has `inputSchema` with `type: "object"` |
+| `schema-required-fields` | `inputSchema` with properties has a `required` array |
+| `schema-no-duplicate-tools` | No duplicate tool names |
+
+### Robustness (4 checks, 20% weight) — warning
+
+| Check | What it tests |
+|-------|---------------|
+| `robust-unknown-method` | Server returns `-32601` for unknown method |
+| `robust-invalid-tool` | `tools/call` with non-existent tool returns error (not crash) |
+| `robust-missing-args` | `tools/call` with missing required args returns error |
+| `robust-malformed-json` | Server handles malformed JSON without crashing |
+
+### Best Practices (4 checks, 10% weight) — info
+
+| Check | What it tests |
+|-------|---------------|
+| `bp-tool-name-format` | Tool names use snake_case or kebab-case |
+| `bp-description-length` | Tool descriptions are 20-500 chars |
+| `bp-server-info` | `serverInfo` includes both `name` and `version` |
+| `bp-capabilities-declared` | Server declares at least one capability |
+
+### Performance (2 checks, 5% weight) — info
+
+| Check | What it tests |
+|-------|---------------|
+| `perf-init-time` | `initialize` completes in < 5 seconds |
+| `perf-tools-list-time` | `tools/list` responds in < 3 seconds |
+
+## Scoring
+
+- **Overall**: Weighted average of category scores (protocol 40%, schema 25%, robustness 20%, practices 10%, performance 5%)
+- **Per category**: (passed checks / total checks) * 100
+- A single critical failure in Protocol drops that category to 0
+
+## CLI Reference
+
+```
+mcp-lighthouse scan [OPTIONS]
+  --stdio COMMAND       Server command to spawn (required for now)
+  --category CATEGORY   Only run checks in this category
+  --timeout SECONDS     Per-check timeout (default: 10)
+  --verbose             Show detailed output
+  --report PATH         Write markdown report
+
+mcp-lighthouse list
+  Lists all available checks
+```
+
+## License
+
+MIT

mcp_lighthouse-0.1.0/mcp_lighthouse/__init__.py

@@ -0,0 +1,3 @@
+"""MCP Lighthouse — audit tool for MCP servers."""
+
+__version__ = "0.1.0"

mcp_lighthouse-0.1.0/mcp_lighthouse/checks.py

@@ -0,0 +1,294 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any, Awaitable, Callable
+
+from .transport import JsonRpcError, TransportError
+
+
+KNOWN_PROTOCOL_VERSIONS = {"2024-11-05", "2025-03-26", "2025-06-18", "2025-11-25"}
+
+
+@dataclass
+class CheckResult:
+    check_id: str
+    name: str
+    category: str
+    severity: str
+    passed: bool
+    message: str
+    details: str = ""
+    elapsed_ms: float = 0
+
+
+@dataclass
+class CheckInfo:
+    check_id: str
+    name: str
+    category: str
+    severity: str
+    func: Callable[[Any], Awaitable[CheckResult]]
+
+
+_checks: list[CheckInfo] = []
+
+
+def check(id: str, name: str, category: str, severity: str):
+    def decorator(func: Callable[[Any], Awaitable[CheckResult]]):
+        _checks.append(CheckInfo(id, name, category, severity, func))
+        return func
+
+    return decorator
+
+
+async def run_all_checks(transport: Any, categories: list[str] | None = None) -> list[CheckResult]:
+    selected = set(categories or [])
+    results: list[CheckResult] = []
+    for info in _checks:
+        if selected and info.category not in selected:
+            continue
+        if not transport.is_running():
+            results.append(CheckResult(
+                info.check_id, info.name, info.category, info.severity,
+                False, "Skipped: server process died", details="subprocess_dead",
+            ))
+            continue
+        start = _now_ms()
+        try:
+            result = await info.func(transport)
+            if not result.elapsed_ms:
+                result.elapsed_ms = _now_ms() - start
+        except Exception as exc:
+            result = CheckResult(
+                info.check_id,
+                info.name,
+                info.category,
+                info.severity,
+                False,
+                f"Check failed: {exc}",
+                details=type(exc).__name__,
+                elapsed_ms=_now_ms() - start,
+            )
+        results.append(result)
+    return results
+
+
+def all_checks() -> list[CheckInfo]:
+    return list(_checks)
+
+
+def _now_ms() -> float:
+    import time
+
+    return time.perf_counter() * 1000
+
+
+def _result(info_id: str, passed: bool, message: str, details: str = "", elapsed_ms: float = 0) -> CheckResult:
+    info = next(item for item in _checks if item.check_id == info_id)
+    return CheckResult(info.check_id, info.name, info.category, info.severity, passed, message, details, elapsed_ms)
+
+
+async def _tools(transport: Any) -> list[dict[str, Any]]:
+    if not hasattr(transport, "_tools_cache"):
+        result = await transport.send_request("tools/list")
+        tools = result.get("tools")
+        if not isinstance(tools, list):
+            raise TransportError("tools/list result.tools is not an array")
+        transport._tools_cache = [tool for tool in tools if isinstance(tool, dict)]
+    return transport._tools_cache
+
+
+@check("proto-init", "Initialize response", "protocol", "critical")
+async def proto_init(transport: Any) -> CheckResult:
+    result = await transport.initialize()
+    ok = (
+        isinstance(result.get("protocolVersion"), str)
+        and isinstance(result.get("capabilities"), dict)
+        and isinstance(result.get("serverInfo"), dict)
+    )
+    return _result("proto-init", ok, "Server initializes correctly" if ok else "Initialize response is missing required fields")
+
+
+@check("proto-init-version", "Protocol version", "protocol", "critical")
+async def proto_init_version(transport: Any) -> CheckResult:
+    result = await transport.initialize()
+    version = result.get("protocolVersion")
+    ok = version in KNOWN_PROTOCOL_VERSIONS
+    return _result("proto-init-version", ok, f"Protocol version {version} is known" if ok else f"Unknown protocol version: {version}")
+
+
+@check("proto-jsonrpc-version", "JSON-RPC version", "protocol", "critical")
+async def proto_jsonrpc_version(transport: Any) -> CheckResult:
+    await transport.send_request("tools/list")
+    bad = [resp for resp in transport.response_history if resp.get("jsonrpc") != "2.0"]
+    return _result("proto-jsonrpc-version", not bad, "All responses use JSON-RPC 2.0" if not bad else f"{len(bad)} responses missing jsonrpc 2.0")
+
+
+@check("proto-id-match", "Response ID matching", "protocol", "critical")
+async def proto_id_match(transport: Any) -> CheckResult:
+    await transport.send_request("tools/list")
+    response = transport.last_response or {}
+    ok = response.get("id") == transport.last_request_id
+    return _result("proto-id-match", ok, "Response id matches request id" if ok else "Response id does not match request id")
+
+
+@check("proto-error-format", "Error format", "protocol", "critical")
+async def proto_error_format(transport: Any) -> CheckResult:
+    try:
+        await transport.send_request("lighthouse/unknown")
+    except JsonRpcError as exc:
+        error = exc.error
+        ok = isinstance(error.get("code"), int) and isinstance(error.get("message"), str)
+        return _result("proto-error-format", ok, "Error response has code and message" if ok else "Error response has invalid shape")
+    return _result("proto-error-format", False, "Unknown method did not return a JSON-RPC error")
+
+
+@check("schema-tools-list", "Tools list", "schema", "warning")
+async def schema_tools_list(transport: Any) -> CheckResult:
+    tools = await _tools(transport)
+    return _result("schema-tools-list", bool(tools), f"tools/list returned {len(tools)} tools" if tools else "tools/list returned no tools")
+
+
+@check("schema-tool-name", "Tool names present", "schema", "warning")
+async def schema_tool_name(transport: Any) -> CheckResult:
+    tools = await _tools(transport)
+    bad = [tool for tool in tools if not isinstance(tool.get("name"), str) or not tool["name"].strip()]
+    return _result("schema-tool-name", not bad, "Every tool has a non-empty name" if not bad else f"{len(bad)} tools have missing names")
+
+
+@check("schema-tool-description", "Tool descriptions present", "schema", "warning")
+async def schema_tool_description(transport: Any) -> CheckResult:
+    tools = await _tools(transport)
+    bad = [tool for tool in tools if not isinstance(tool.get("description"), str) or len(tool["description"].strip()) <= 10]
+    return _result("schema-tool-description", not bad, "Every tool has a useful description" if not bad else f"{len(bad)} tools have short or missing descriptions")
+
+
+@check("schema-tool-input-schema", "Tool input schemas", "schema", "warning")
+async def schema_tool_input_schema(transport: Any) -> CheckResult:
+    tools = await _tools(transport)
+    bad = [tool for tool in tools if not isinstance(tool.get("inputSchema"), dict) or tool["inputSchema"].get("type") != "object"]
+    return _result("schema-tool-input-schema", not bad, "Every tool inputSchema is an object" if not bad else f"{len(bad)} tools have invalid inputSchema")
+
+
+@check("schema-required-fields", "Required fields", "schema", "warning")
+async def schema_required_fields(transport: Any) -> CheckResult:
+    tools = await _tools(transport)
+    bad = []
+    for tool in tools:
+        schema = tool.get("inputSchema")
+        if isinstance(schema, dict) and schema.get("properties") and "required" not in schema:
+            bad.append(tool.get("name", "<unnamed>"))
+    return _result("schema-required-fields", not bad, "Schemas with properties declare required fields" if not bad else f"{len(bad)} schemas with properties omit required")
+
+
+@check("schema-no-duplicate-tools", "No duplicate tools", "schema", "warning")
+async def schema_no_duplicate_tools(transport: Any) -> CheckResult:
+    tools = await _tools(transport)
+    names: list[str] = [tool["name"] for tool in tools if isinstance(tool.get("name"), str)]
+    duplicates = sorted(n for n in set(names) if names.count(n) > 1)
+    return _result("schema-no-duplicate-tools", not duplicates, "No duplicate tool names" if not duplicates else f"Duplicate tool names: {', '.join(duplicates)}")
+
+
+@check("robust-unknown-method", "Unknown method handling", "robustness", "warning")
+async def robust_unknown_method(transport: Any) -> CheckResult:
+    try:
+        await transport.send_request("foo/bar")
+    except JsonRpcError as exc:
+        ok = exc.error.get("code") == -32601
+        return _result("robust-unknown-method", ok, "Unknown method returns -32601" if ok else f"Unknown method returned {exc.error.get('code')}")
+    return _result("robust-unknown-method", False, "Unknown method did not return an error")
+
+
+@check("robust-invalid-tool", "Invalid tool handling", "robustness", "warning")
+async def robust_invalid_tool(transport: Any) -> CheckResult:
+    try:
+        await transport.send_request("tools/call", {"name": "__mcp_lighthouse_missing_tool__", "arguments": {}})
+    except JsonRpcError:
+        return _result("robust-invalid-tool", transport.is_running(), "Invalid tool returns an error without crashing")
+    except TransportError as exc:
+        return _result("robust-invalid-tool", False, f"Invalid tool broke transport: {exc}")
+    return _result("robust-invalid-tool", False, "Invalid tool unexpectedly succeeded")
+
+
+@check("robust-missing-args", "Missing arguments handling", "robustness", "warning")
+async def robust_missing_args(transport: Any) -> CheckResult:
+    tools = await _tools(transport)
+    candidate = None
+    for tool in tools:
+        schema = tool.get("inputSchema")
+        if isinstance(schema, dict) and schema.get("required"):
+            candidate = tool
+            break
+    if candidate is None:
+        return _result("robust-missing-args", True, "No tool with required arguments found")
+    try:
+        await transport.send_request("tools/call", {"name": candidate.get("name"), "arguments": {}})
+    except JsonRpcError:
+        return _result("robust-missing-args", transport.is_running(), "Missing required arguments return an error")
+    return _result("robust-missing-args", False, f"{candidate.get('name')} accepted missing required arguments")
+
+
+@check("robust-malformed-json", "Malformed JSON handling", "robustness", "warning")
+async def robust_malformed_json(transport: Any) -> CheckResult:
+    await transport.send_raw_line('{"jsonrpc":"2.0","id":999,"method":')
+    try:
+        response = await transport.read_raw_response(timeout=1)
+    except TransportError:
+        ok = transport.is_running()
+        return _result("robust-malformed-json", ok, "Malformed JSON did not crash server" if ok else "Server stopped after malformed JSON")
+    error = response.get("error") if response else None
+    ok = isinstance(error, dict) and error.get("code") == -32700
+    return _result("robust-malformed-json", ok, "Malformed JSON returns parse error" if ok else "Malformed JSON response is not a parse error")
+
+
+@check("bp-tool-name-format", "Tool name format", "best_practices", "info")
+async def bp_tool_name_format(transport: Any) -> CheckResult:
+    tools = await _tools(transport)
+    pattern = re.compile(r"^[a-z0-9]+([_-][a-z0-9]+)*$")
+    bad = [tool.get("name", "") for tool in tools if not pattern.match(str(tool.get("name", "")))]
+    return _result("bp-tool-name-format", not bad, "Tool names use snake_case or kebab-case" if not bad else f"Non-standard tool names: {', '.join(bad)}")
+
+
+@check("bp-description-length", "Description length", "best_practices", "info")
+async def bp_description_length(transport: Any) -> CheckResult:
+    tools = await _tools(transport)
+    bad = []
+    for tool in tools:
+        description = tool.get("description")
+        if not isinstance(description, str) or not 20 <= len(description.strip()) <= 500:
+            bad.append(str(tool.get("name", "<unnamed>")))
+    return _result("bp-description-length", not bad, "Tool descriptions are 20-500 chars" if not bad else f"{len(bad)} tool descriptions are outside 20-500 chars")
+
+
+@check("bp-server-info", "Server info", "best_practices", "info")
+async def bp_server_info(transport: Any) -> CheckResult:
+    result = await transport.initialize()
+    server_info = result.get("serverInfo")
+    ok = isinstance(server_info, dict) and bool(server_info.get("name")) and bool(server_info.get("version"))
+    return _result("bp-server-info", ok, "serverInfo includes name and version" if ok else "serverInfo should include name and version")
+
+
+@check("bp-capabilities-declared", "Capabilities declared", "best_practices", "info")
+async def bp_capabilities_declared(transport: Any) -> CheckResult:
+    result = await transport.initialize()
+    capabilities = result.get("capabilities")
+    ok = isinstance(capabilities, dict) and bool(capabilities)
+    return _result("bp-capabilities-declared", ok, "Server declares at least one capability" if ok else "Server declares no capabilities")
+
+
+@check("perf-init-time", "Initialize time", "performance", "info")
+async def perf_init_time(transport: Any) -> CheckResult:
+    await transport.initialize()
+    elapsed = transport.initialize_elapsed_ms
+    ok = elapsed < 5000
+    return _result("perf-init-time", ok, f"Initialize completed in {elapsed:.0f} ms", elapsed_ms=elapsed)
+
+
+@check("perf-tools-list-time", "Tools list time", "performance", "info")
+async def perf_tools_list_time(transport: Any) -> CheckResult:
+    start = _now_ms()
+    await transport.send_request("tools/list")
+    elapsed = _now_ms() - start
+    ok = elapsed < 3000
+    return _result("perf-tools-list-time", ok, f"tools/list completed in {elapsed:.0f} ms", elapsed_ms=elapsed)

mcp_lighthouse-0.1.0/mcp_lighthouse/cli.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+from pathlib import Path
+
+from .checks import all_checks, run_all_checks
+from .reporter import render_markdown, render_terminal
+from .transport import StdioTransport
+
+
+VALID_CATEGORIES = ["protocol", "schema", "robustness", "best_practices", "performance"]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(prog="mcp-lighthouse")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    scan_parser = subparsers.add_parser("scan", help="scan an MCP server")
+    scan_parser.add_argument("--stdio", required=True, help="stdio command to launch the MCP server")
+    scan_parser.add_argument("--report", help="write a Markdown report to this path")
+    scan_parser.add_argument("--category", choices=VALID_CATEGORIES, action="append", help="run only this check category")
+    scan_parser.add_argument("--timeout", type=float, default=10, help="transport timeout in seconds")
+
+    subparsers.add_parser("list", help="list registered checks")
+
+    args = parser.parse_args()
+    if args.command == "list":
+        _list_checks()
+        return
+    if args.command == "scan":
+        asyncio.run(_scan(args))
+
+
+def _list_checks() -> None:
+    for info in all_checks():
+        print(f"{info.check_id:<28} {info.category:<15} {info.severity:<8} {info.name}")
+
+
+async def _scan(args: argparse.Namespace) -> None:
+    transport = StdioTransport(args.stdio, timeout=args.timeout)
+    try:
+        await transport.start()
+        await transport.initialize()
+        results = await run_all_checks(transport, categories=args.category)
+        render_terminal(results, transport.server_info)
+        if args.report:
+            Path(args.report).write_text(render_markdown(results, transport.server_info), encoding="utf-8")
+    finally:
+        await transport.close()
+
+
+if __name__ == "__main__":
+    main()

mcp_lighthouse-0.1.0/mcp_lighthouse/reporter.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any
+
+from .checks import CheckResult
+
+
+CATEGORY_WEIGHTS = {
+    "protocol": 40,
+    "schema": 25,
+    "robustness": 20,
+    "best_practices": 10,
+    "performance": 5,
+}
+
+CATEGORY_LABELS = {
+    "protocol": "Protocol",
+    "schema": "Schema",
+    "robustness": "Robustness",
+    "best_practices": "Practices",
+    "performance": "Performance",
+}
+
+
+def category_scores(results: list[CheckResult]) -> dict[str, float]:
+    grouped: dict[str, list[CheckResult]] = defaultdict(list)
+    for result in results:
+        grouped[result.category].append(result)
+    scores: dict[str, float] = {}
+    for category in CATEGORY_WEIGHTS:
+        items = grouped.get(category, [])
+        scores[category] = 100.0 if not items else sum(1 for item in items if item.passed) / len(items) * 100
+    return scores
+
+
+def overall_score(results: list[CheckResult]) -> int:
+    scores = category_scores(results)
+    executed = {r.category for r in results}
+    if not executed:
+        return 0
+    total = sum(CATEGORY_WEIGHTS[cat] for cat in executed if cat in CATEGORY_WEIGHTS)
+    if total == 0:
+        return 0
+    weighted = sum(scores[cat] * CATEGORY_WEIGHTS.get(cat, 0) for cat in executed) / total
+    return round(weighted)
+
+
+def render_terminal(results: list[CheckResult], server_info: dict[str, Any] | None = None) -> None:
+    try:
+        from rich.console import Console
+        from rich.text import Text
+    except ImportError:
+        print(render_plain(results, server_info))
+        return
+
+    console = Console()
+    name = (server_info or {}).get("name", "unknown-server")
+    version = (server_info or {}).get("version", "unknown")
+    console.print(f"[bold]MCP Lighthouse[/bold] — {name} v{version}\n")
+
+    scores = category_scores(results)
+    for category, score in scores.items():
+        filled = round(score / 5)
+        bar = "█" * filled + "░" * (20 - filled)
+        console.print(f"  {CATEGORY_LABELS[category]:<11} [cyan]{bar}[/cyan]  {score:>3.0f}")
+
+    console.print(f"\n  [bold]Overall Score:[/bold] {overall_score(results)}/100\n")
+
+    for result in results:
+        icon = "✅" if result.passed else ("❌" if result.severity == "critical" else "⚠️")
+        style = "green" if result.passed else ("red" if result.severity == "critical" else "yellow")
+        line = Text(f"  {icon} {result.check_id:<26} {result.message}", style=style)
+        console.print(line)
+
+    passed = sum(1 for result in results if result.passed)
+    warnings = sum(1 for result in results if not result.passed and result.severity != "critical")
+    failed = sum(1 for result in results if not result.passed and result.severity == "critical")
+    console.print(f"\n  {len(results)} checks: {passed} passed, {warnings} warnings, {failed} failed")
+
+
+def render_plain(results: list[CheckResult], server_info: dict[str, Any] | None = None) -> str:
+    name = (server_info or {}).get("name", "unknown-server")
+    version = (server_info or {}).get("version", "unknown")
+    lines = [f"MCP Lighthouse — {name} v{version}", ""]
+    for category, score in category_scores(results).items():
+        filled = round(score / 5)
+        lines.append(f"  {CATEGORY_LABELS[category]:<11} {'#' * filled}{'.' * (20 - filled)}  {score:.0f}")
+    lines.append("")
+    lines.append(f"  Overall Score: {overall_score(results)}/100")
+    lines.append("")
+    for result in results:
+        icon = "PASS" if result.passed else ("FAIL" if result.severity == "critical" else "WARN")
+        lines.append(f"  {icon:<4} {result.check_id:<26} {result.message}")
+    return "\n".join(lines)
+
+
+def render_markdown(results: list[CheckResult], server_info: dict[str, Any] | None = None) -> str:
+    name = (server_info or {}).get("name", "unknown-server")
+    version = (server_info or {}).get("version", "unknown")
+    lines = [
+        f"# MCP Lighthouse Report — {name} v{version}",
+        "",
+        f"Overall Score: **{overall_score(results)}/100**",
+        "",
+        "## Category Scores",
+        "",
+        "| Category | Score |",
+        "|---|---:|",
+    ]
+    for category, score in category_scores(results).items():
+        lines.append(f"| {CATEGORY_LABELS[category]} | {score:.0f} |")
+
+    lines.extend(
+        [
+            "",
+            "## Checks",
+            "",
+            "| Status | Check | Category | Severity | Message | Elapsed |",
+            "|---|---|---|---|---|---:|",
+        ]
+    )
+    for result in results:
+        status = "Passed" if result.passed else "Failed"
+        lines.append(
+            f"| {status} | `{result.check_id}` | {result.category} | {result.severity} | "
+            f"{_escape(result.message)} | {result.elapsed_ms:.0f} ms |"
+        )
+
+    failures = [result for result in results if not result.passed]
+    if failures:
+        lines.extend(["", "## Recommendations", ""])
+        for result in failures:
+            lines.append(f"- `{result.check_id}`: {_recommendation(result)}")
+
+    return "\n".join(lines) + "\n"
+
+
+def _escape(value: str) -> str:
+    return value.replace("|", "\\|").replace("\n", " ")
+
+
+def _recommendation(result: CheckResult) -> str:
+    recommendations = {
+        "protocol": "Fix JSON-RPC protocol handling before relying on this server in automated clients.",
+        "schema": "Tighten tool metadata and JSON Schema so clients can validate calls correctly.",
+        "robustness": "Return JSON-RPC errors for invalid input while keeping the server process alive.",
+        "best_practices": "Improve metadata quality for better discoverability and client compatibility.",
+        "performance": "Profile startup or request handling and remove avoidable blocking work.",
+    }
+    return recommendations.get(result.category, "Review the failing behavior and add a regression test.")

mcp_lighthouse-0.1.0/mcp_lighthouse/transport.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import shlex
+from typing import Any
+
+
+class TransportError(RuntimeError):
+    pass
+
+
+class JsonRpcError(TransportError):
+    def __init__(self, error: dict[str, Any], response: dict[str, Any] | None = None) -> None:
+        self.error = error
+        self.response = response or {}
+        message = error.get("message", "JSON-RPC error")
+        code = error.get("code", "unknown")
+        super().__init__(f"{code}: {message}")
+
+
+class StdioTransport:
+    def __init__(self, command: str, timeout: float = 10) -> None:
+        self.command = command
+        self.timeout = timeout
+        self.process: asyncio.subprocess.Process | None = None
+        self._next_id = 1
+        self.last_request_id: int | None = None
+        self.last_response: dict[str, Any] | None = None
+        self.response_history: list[dict[str, Any]] = []
+        self.server_info: dict[str, Any] = {}
+        self.capabilities: dict[str, Any] = {}
+        self.initialize_result: dict[str, Any] | None = None
+        self.initialize_elapsed_ms: float = 0
+
+    async def start(self) -> None:
+        if self.process is not None:
+            return
+        args = shlex.split(self.command)
+        if not args:
+            raise TransportError("Empty stdio command")
+        self.process = await asyncio.create_subprocess_exec(
+            *args,
+            stdin=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+
+    async def send_request(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
+        request_id = self._next_id
+        self._next_id += 1
+        self.last_request_id = request_id
+        message: dict[str, Any] = {"jsonrpc": "2.0", "id": request_id, "method": method}
+        if params is not None:
+            message["params"] = params
+
+        await self._write_message(message)
+        response = await self._read_message()
+        self.last_response = response
+        self.response_history.append(response)
+
+        if response.get("id") != request_id:
+            raise TransportError(f"Response id mismatch: expected {request_id}, got {response.get('id')}")
+        if "error" in response:
+            error = response["error"]
+            if isinstance(error, dict):
+                raise JsonRpcError(error, response)
+            raise TransportError("JSON-RPC error field is not an object")
+        if "result" not in response:
+            raise TransportError("JSON-RPC response missing result")
+        result = response["result"]
+        if not isinstance(result, dict):
+            raise TransportError("JSON-RPC result is not an object")
+        return result
+
+    async def send_notification(self, method: str, params: dict[str, Any] | None = None) -> None:
+        message: dict[str, Any] = {"jsonrpc": "2.0", "method": method}
+        if params is not None:
+            message["params"] = params
+        await self._write_message(message)
+
+    async def initialize(self) -> dict[str, Any]:
+        if self.initialize_result is not None:
+            return self.initialize_result
+
+        start = asyncio.get_running_loop().time()
+        result = await self.send_request(
+            "initialize",
+            {
+                "protocolVersion": "2025-06-18",
+                "capabilities": {},
+                "clientInfo": {"name": "mcp-lighthouse", "version": "0.1.0"},
+            },
+        )
+        self.initialize_elapsed_ms = (asyncio.get_running_loop().time() - start) * 1000
+        self.initialize_result = result
+        self.capabilities = result.get("capabilities") if isinstance(result.get("capabilities"), dict) else {}
+        self.server_info = result.get("serverInfo") if isinstance(result.get("serverInfo"), dict) else {}
+        await self.send_notification("notifications/initialized")
+        return result
+
+    async def send_raw_line(self, line: str) -> None:
+        if not line.endswith("\n"):
+            line += "\n"
+        await self._ensure_started()
+        assert self.process is not None
+        if self.process.stdin is None:
+            raise TransportError("Subprocess stdin is unavailable")
+        self.process.stdin.write(line.encode("utf-8"))
+        await self.process.stdin.drain()
+
+    async def read_raw_response(self, timeout: float | None = None) -> dict[str, Any] | None:
+        response = await self._read_message(timeout=timeout)
+        self.last_response = response
+        self.response_history.append(response)
+        return response
+
+    def is_running(self) -> bool:
+        return self.process is not None and self.process.returncode is None
+
+    async def close(self) -> None:
+        if self.process is None:
+            return
+        if self.process.returncode is None:
+            self.process.terminate()
+            try:
+                await asyncio.wait_for(self.process.wait(), timeout=2)
+            except asyncio.TimeoutError:
+                self.process.kill()
+                await self.process.wait()
+        self.process = None
+
+    async def _ensure_started(self) -> None:
+        if self.process is None or self.process.returncode is not None:
+            self.process = None
+            await self.start()
+
+    async def _write_message(self, message: dict[str, Any]) -> None:
+        await self._ensure_started()
+        assert self.process is not None
+        if self.process.stdin is None:
+            raise TransportError("Subprocess stdin is unavailable")
+        data = json.dumps(message, separators=(",", ":")) + "\n"
+        self.process.stdin.write(data.encode("utf-8"))
+        await self.process.stdin.drain()
+
+    async def _read_message(self, timeout: float | None = None) -> dict[str, Any]:
+        await self._ensure_started()
+        assert self.process is not None
+        if self.process.stdout is None:
+            raise TransportError("Subprocess stdout is unavailable")
+        try:
+            line = await asyncio.wait_for(self.process.stdout.readline(), timeout=timeout or self.timeout)
+        except asyncio.TimeoutError as exc:
+            raise TransportError("Timed out waiting for JSON-RPC response") from exc
+        if not line:
+            raise TransportError("Subprocess closed stdout")
+        try:
+            response = json.loads(line.decode("utf-8"))
+        except json.JSONDecodeError as exc:
+            raise TransportError(f"Invalid JSON response: {line.decode('utf-8', errors='replace').strip()}") from exc
+        if not isinstance(response, dict):
+            raise TransportError("JSON-RPC response is not an object")
+        return response

mcp_lighthouse-0.1.0/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "mcp-lighthouse"
+version = "0.1.0"
+description = "Audit tool for MCP servers — test protocol compliance, schema quality, and robustness"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [{ name = "Makito Chiba" }]
+keywords = ["mcp", "model-context-protocol", "testing", "audit", "ai-agent"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Topic :: Software Development :: Testing",
+]
+dependencies = ["rich>=13.0", "httpx>=0.27"]
+
+[project.scripts]
+mcp-lighthouse = "mcp_lighthouse.cli:main"