mcpvitals 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mcpvitals/__init__.py +1 -0
- mcpvitals/badge.py +28 -0
- mcpvitals/checks/__init__.py +33 -0
- mcpvitals/checks/conformance.py +24 -0
- mcpvitals/checks/confusion.py +29 -0
- mcpvitals/checks/migration.py +22 -0
- mcpvitals/checks/reliability.py +27 -0
- mcpvitals/checks/security.py +44 -0
- mcpvitals/checks/tokens.py +34 -0
- mcpvitals/cli.py +107 -0
- mcpvitals/introspect.py +38 -0
- mcpvitals/lock.py +32 -0
- mcpvitals/models.py +59 -0
- mcpvitals/monitor.py +26 -0
- mcpvitals/report.py +28 -0
- mcpvitals/scoring.py +13 -0
- mcpvitals-0.1.0.dist-info/METADATA +109 -0
- mcpvitals-0.1.0.dist-info/RECORD +21 -0
- mcpvitals-0.1.0.dist-info/WHEEL +4 -0
- mcpvitals-0.1.0.dist-info/entry_points.txt +2 -0
- mcpvitals-0.1.0.dist-info/licenses/LICENSE +21 -0
mcpvitals/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
mcpvitals/badge.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from mcpvitals.models import Report
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def _color(score: int) -> str:
|
|
5
|
+
return "green" if score >= 90 else "yellow" if score >= 70 else "red"
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def shields_endpoint(report: Report) -> dict:
|
|
9
|
+
return {
|
|
10
|
+
"schemaVersion": 1,
|
|
11
|
+
"label": "mcp health",
|
|
12
|
+
"message": f"{report.score} {report.grade}",
|
|
13
|
+
"color": _color(report.score),
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def badge_svg(report: Report) -> str:
|
|
18
|
+
color = {"green": "#3fb950", "yellow": "#d29922", "red": "#f85149"}[_color(report.score)]
|
|
19
|
+
text = f"{report.score} {report.grade}"
|
|
20
|
+
return (
|
|
21
|
+
'<svg xmlns="http://www.w3.org/2000/svg" width="150" height="20" role="img" '
|
|
22
|
+
f'aria-label="mcp health: {text}">'
|
|
23
|
+
'<rect width="90" height="20" fill="#555"/>'
|
|
24
|
+
f'<rect x="90" width="60" height="20" fill="{color}"/>'
|
|
25
|
+
'<g fill="#fff" font-family="Verdana" font-size="11">'
|
|
26
|
+
'<text x="8" y="14">mcp health</text>'
|
|
27
|
+
f'<text x="98" y="14">{text}</text></g></svg>'
|
|
28
|
+
)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from importlib import import_module
|
|
2
|
+
|
|
3
|
+
CHECKS = []
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def check(fn):
|
|
7
|
+
CHECKS.append(fn)
|
|
8
|
+
return fn
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
_MODULES = ["conformance", "reliability", "security", "tokens", "confusion", "migration"]
|
|
12
|
+
_loaded = False
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _load_modules():
|
|
16
|
+
global _loaded
|
|
17
|
+
if _loaded:
|
|
18
|
+
return
|
|
19
|
+
for m in _MODULES:
|
|
20
|
+
try:
|
|
21
|
+
import_module(f"mcpvitals.checks.{m}")
|
|
22
|
+
except ModuleNotFoundError:
|
|
23
|
+
pass
|
|
24
|
+
_loaded = True
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def run_all(snapshot, config=None):
|
|
28
|
+
_load_modules()
|
|
29
|
+
config = config or {}
|
|
30
|
+
findings = []
|
|
31
|
+
for fn in CHECKS:
|
|
32
|
+
findings.extend(fn(snapshot, config))
|
|
33
|
+
return findings
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from mcpvitals.checks import check
|
|
2
|
+
from mcpvitals.models import Finding, Level
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@check
|
|
6
|
+
def empty_descriptions(snapshot, config):
|
|
7
|
+
out = []
|
|
8
|
+
for t in snapshot.tools:
|
|
9
|
+
if not t.description.strip():
|
|
10
|
+
out.append(Finding("MV001", Level.WARN,
|
|
11
|
+
f"tool '{t.name}' has no description",
|
|
12
|
+
hint="agents choose tools by description; write a clear one", tool=t.name))
|
|
13
|
+
return out
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@check
|
|
17
|
+
def duplicate_tool_names(snapshot, config):
|
|
18
|
+
seen, out = set(), []
|
|
19
|
+
for t in snapshot.tools:
|
|
20
|
+
if t.name in seen:
|
|
21
|
+
out.append(Finding("MV002", Level.ERROR,
|
|
22
|
+
f"duplicate tool name '{t.name}'", tool=t.name))
|
|
23
|
+
seen.add(t.name)
|
|
24
|
+
return out
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from itertools import combinations
|
|
3
|
+
from mcpvitals.checks import check
|
|
4
|
+
from mcpvitals.models import Finding, Level
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _tokens(text):
|
|
8
|
+
return set(re.findall(r"[a-z0-9]+", text.lower()))
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def similarity(a: str, b: str) -> float:
|
|
12
|
+
ta, tb = _tokens(a), _tokens(b)
|
|
13
|
+
if not ta or not tb:
|
|
14
|
+
return 0.0
|
|
15
|
+
inter = len(ta & tb)
|
|
16
|
+
return inter / ((len(ta) * len(tb)) ** 0.5)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@check
|
|
20
|
+
def tool_confusion(snapshot, config):
|
|
21
|
+
thresh = config.get("confusion_threshold", 0.8)
|
|
22
|
+
out = []
|
|
23
|
+
for x, y in combinations(snapshot.tools, 2):
|
|
24
|
+
s = similarity(f"{x.name} {x.description}", f"{y.name} {y.description}")
|
|
25
|
+
if s >= thresh:
|
|
26
|
+
out.append(Finding("MV040", Level.WARN,
|
|
27
|
+
f"tools '{x.name}' and '{y.name}' are {s:.2f} similar; agents may pick the wrong one",
|
|
28
|
+
hint="differentiate names/descriptions or merge them", tool=x.name))
|
|
29
|
+
return out
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from mcpvitals.checks import check
|
|
2
|
+
from mcpvitals.models import Finding, Level
|
|
3
|
+
|
|
4
|
+
STATELESS_SPEC = "2026-07-28"
|
|
5
|
+
_DEPRECATED = {"logging", "sampling", "roots"}
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@check
|
|
9
|
+
def migration_readiness(snapshot, config):
|
|
10
|
+
out = []
|
|
11
|
+
pv = snapshot.protocol_version or ""
|
|
12
|
+
if pv and pv < STATELESS_SPEC:
|
|
13
|
+
out.append(Finding("MV050", Level.INFO,
|
|
14
|
+
f"protocol {pv} predates the {STATELESS_SPEC} stateless spec",
|
|
15
|
+
hint="the new spec removes sessions/initialize; plan the migration"))
|
|
16
|
+
caps = (snapshot.raw or {}).get("capabilities", {}) or {}
|
|
17
|
+
for dep in sorted(_DEPRECATED):
|
|
18
|
+
if dep in caps:
|
|
19
|
+
out.append(Finding("MV051", Level.WARN,
|
|
20
|
+
f"capability '{dep}' is deprecated in {STATELESS_SPEC}",
|
|
21
|
+
hint=f"move off '{dep}' before upgrading"))
|
|
22
|
+
return out
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from mcpvitals.checks import check
|
|
2
|
+
from mcpvitals.models import Finding, Level
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@check
|
|
6
|
+
def missing_required(snapshot, config):
|
|
7
|
+
out = []
|
|
8
|
+
for t in snapshot.tools:
|
|
9
|
+
schema = t.input_schema or {}
|
|
10
|
+
props = schema.get("properties")
|
|
11
|
+
if props and not schema.get("required"):
|
|
12
|
+
out.append(Finding("MV010", Level.WARN,
|
|
13
|
+
f"tool '{t.name}' declares parameters but no 'required' list",
|
|
14
|
+
hint="agents may omit needed args; mark required params", tool=t.name))
|
|
15
|
+
return out
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@check
|
|
19
|
+
def untyped_properties(snapshot, config):
|
|
20
|
+
out = []
|
|
21
|
+
for t in snapshot.tools:
|
|
22
|
+
for pname, pdef in (t.input_schema.get("properties") or {}).items():
|
|
23
|
+
if isinstance(pdef, dict) and "type" not in pdef and "$ref" not in pdef and "anyOf" not in pdef:
|
|
24
|
+
out.append(Finding("MV011", Level.WARN,
|
|
25
|
+
f"tool '{t.name}' param '{pname}' has no type",
|
|
26
|
+
hint="give every param a JSON type", tool=t.name))
|
|
27
|
+
return out
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from mcpvitals.checks import check
|
|
3
|
+
from mcpvitals.models import Finding, Level
|
|
4
|
+
|
|
5
|
+
_SECRET = re.compile(r"(sk-[A-Za-z0-9]{16,}|api[_-]?key\s*=\s*\S{12,}|AKIA[0-9A-Z]{16})")
|
|
6
|
+
_OVERBROAD = {"run_sql", "execute_sql", "run_command", "exec", "shell", "eval", "read_file", "write_file"}
|
|
7
|
+
_IMPERATIVE = re.compile(r"\b(ignore (all |the )?previous|disregard|you must|always (return|send|include)|do not tell)\b", re.I)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _tool_text(t):
|
|
11
|
+
return f"{t.name} {t.description} {t.input_schema}"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@check
|
|
15
|
+
def secret_in_text(snapshot, config):
|
|
16
|
+
out = []
|
|
17
|
+
for t in snapshot.tools:
|
|
18
|
+
if _SECRET.search(_tool_text(t)):
|
|
19
|
+
out.append(Finding("MV020", Level.ERROR,
|
|
20
|
+
f"tool '{t.name}' appears to contain a hard-coded secret",
|
|
21
|
+
hint="never ship keys in tool metadata; use env vars", tool=t.name))
|
|
22
|
+
return out
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@check
|
|
26
|
+
def overbroad_tools(snapshot, config):
|
|
27
|
+
out = []
|
|
28
|
+
for t in snapshot.tools:
|
|
29
|
+
if t.name.lower() in _OVERBROAD:
|
|
30
|
+
out.append(Finding("MV021", Level.WARN,
|
|
31
|
+
f"tool '{t.name}' grants broad, unconstrained capability",
|
|
32
|
+
hint="scope it down; broad tools amplify prompt-injection blast radius", tool=t.name))
|
|
33
|
+
return out
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@check
|
|
37
|
+
def injection_surface(snapshot, config):
|
|
38
|
+
out = []
|
|
39
|
+
for t in snapshot.tools:
|
|
40
|
+
if _IMPERATIVE.search(t.description or ""):
|
|
41
|
+
out.append(Finding("MV022", Level.ERROR,
|
|
42
|
+
f"tool '{t.name}' description contains model-directed instructions (poisoning surface)",
|
|
43
|
+
hint="tool descriptions should describe, not instruct the model", tool=t.name))
|
|
44
|
+
return out
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import math
|
|
3
|
+
from mcpvitals.checks import check
|
|
4
|
+
from mcpvitals.models import Finding, Level
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def estimate_tokens(text: str) -> int:
|
|
8
|
+
try:
|
|
9
|
+
import tiktoken
|
|
10
|
+
return len(tiktoken.get_encoding("cl100k_base").encode(text))
|
|
11
|
+
except Exception:
|
|
12
|
+
return max(1, math.ceil(len(text) / 4))
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _tool_text(t):
|
|
16
|
+
return t.name + "\n" + (t.description or "") + "\n" + json.dumps(t.input_schema, sort_keys=True)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@check
|
|
20
|
+
def token_cost(snapshot, config):
|
|
21
|
+
counter = config.get("token_counter", estimate_tokens)
|
|
22
|
+
warn = config.get("token_warn", 500)
|
|
23
|
+
out, total = [], 0
|
|
24
|
+
for t in snapshot.tools:
|
|
25
|
+
n = counter(_tool_text(t))
|
|
26
|
+
total += n
|
|
27
|
+
if n > warn:
|
|
28
|
+
out.append(Finding("MV030", Level.WARN,
|
|
29
|
+
f"tool '{t.name}' costs ~{n} tokens to expose",
|
|
30
|
+
hint="trim the description/schema; it inflates every request", tool=t.name))
|
|
31
|
+
out.append(Finding("MV031", Level.INFO,
|
|
32
|
+
f"server costs ~{total} tokens to connect ({len(snapshot.tools)} tools)",
|
|
33
|
+
hint="this is spent on every request before the user's message"))
|
|
34
|
+
return out
|
mcpvitals/cli.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
import pathlib
|
|
5
|
+
import sys
|
|
6
|
+
import time
|
|
7
|
+
from mcpvitals.models import ServerSnapshot, Level
|
|
8
|
+
from mcpvitals.checks import run_all
|
|
9
|
+
from mcpvitals.checks.migration import migration_readiness
|
|
10
|
+
from mcpvitals.scoring import score
|
|
11
|
+
from mcpvitals.report import render_human, render_json
|
|
12
|
+
from mcpvitals.badge import badge_svg
|
|
13
|
+
from mcpvitals.lock import lock_from_snapshot, diff_locks
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _snapshot(args) -> ServerSnapshot:
|
|
17
|
+
if args.snapshot:
|
|
18
|
+
return ServerSnapshot.from_dict(json.loads(pathlib.Path(args.snapshot).read_text()))
|
|
19
|
+
from mcpvitals.introspect import introspect
|
|
20
|
+
return introspect(args.target)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _cmd_check(args) -> int:
|
|
24
|
+
snap = _snapshot(args)
|
|
25
|
+
findings = run_all(snap)
|
|
26
|
+
report = score(findings)
|
|
27
|
+
if args.badge:
|
|
28
|
+
pathlib.Path(args.badge).write_text(badge_svg(report))
|
|
29
|
+
print(render_json(report) if args.json else render_human(report))
|
|
30
|
+
if args.strict and any(f.level == Level.ERROR for f in findings):
|
|
31
|
+
return 1
|
|
32
|
+
return 0
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _cmd_migrate(args) -> int:
|
|
36
|
+
snap = _snapshot(args)
|
|
37
|
+
findings = migration_readiness(snap, {})
|
|
38
|
+
print(render_human(score(findings)))
|
|
39
|
+
return 0
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _cmd_watch(args) -> int:
|
|
43
|
+
snap = _snapshot(args)
|
|
44
|
+
new_lock = lock_from_snapshot(snap)
|
|
45
|
+
lockpath = pathlib.Path(args.lock)
|
|
46
|
+
findings = []
|
|
47
|
+
if lockpath.exists():
|
|
48
|
+
findings = diff_locks(json.loads(lockpath.read_text()), new_lock)
|
|
49
|
+
print(render_human(score(findings)) if findings else "no changes since last pin")
|
|
50
|
+
else:
|
|
51
|
+
print(f"pinned {len(new_lock['tools'])} tools to {lockpath}")
|
|
52
|
+
if not args.check:
|
|
53
|
+
lockpath.write_text(json.dumps(new_lock, indent=2, sort_keys=True))
|
|
54
|
+
if args.check and any(f.level == Level.ERROR for f in findings):
|
|
55
|
+
return 1
|
|
56
|
+
return 0
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _cmd_monitor(args) -> int:
|
|
60
|
+
from mcpvitals.monitor import probe, summarize
|
|
61
|
+
results = []
|
|
62
|
+
for i in range(args.count):
|
|
63
|
+
r = probe(args.target)
|
|
64
|
+
results.append(r)
|
|
65
|
+
status = "up" if r["ok"] else f"down ({r['error']})"
|
|
66
|
+
print(f"probe {i + 1}/{args.count}: {status} {r['latency_ms']:.0f}ms")
|
|
67
|
+
if i < args.count - 1:
|
|
68
|
+
time.sleep(args.interval)
|
|
69
|
+
s = summarize(results)
|
|
70
|
+
print(json.dumps(s) if args.json else f"uptime {s['uptime_pct']}% avg {s['avg_latency_ms']}ms")
|
|
71
|
+
return 0 if s["up"] == s["probes"] else 1
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def main(argv=None) -> int:
|
|
75
|
+
p = argparse.ArgumentParser(prog="mcpvitals", description="Vital signs for your MCP servers.")
|
|
76
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
77
|
+
|
|
78
|
+
for name in ("check", "migrate-check", "watch", "monitor"):
|
|
79
|
+
sp = sub.add_parser(name)
|
|
80
|
+
sp.add_argument("target", nargs="?", help="stdio command or http(s) url")
|
|
81
|
+
sp.add_argument("--snapshot", help="load a ServerSnapshot json instead of connecting")
|
|
82
|
+
if name == "check":
|
|
83
|
+
sp.add_argument("--json", action="store_true")
|
|
84
|
+
sp.add_argument("--badge", metavar="FILE")
|
|
85
|
+
sp.add_argument("--strict", action="store_true")
|
|
86
|
+
if name == "watch":
|
|
87
|
+
sp.add_argument("--lock", default="mcp.lock", help="lock file path (default: mcp.lock)")
|
|
88
|
+
sp.add_argument("--check", action="store_true", help="do not update the lock; fail on a changed tool")
|
|
89
|
+
if name == "monitor":
|
|
90
|
+
sp.add_argument("--count", type=int, default=3)
|
|
91
|
+
sp.add_argument("--interval", type=float, default=2.0)
|
|
92
|
+
sp.add_argument("--json", action="store_true")
|
|
93
|
+
|
|
94
|
+
args = p.parse_args(argv)
|
|
95
|
+
if not args.target and not getattr(args, "snapshot", None):
|
|
96
|
+
p.error("provide a target or --snapshot")
|
|
97
|
+
|
|
98
|
+
return {
|
|
99
|
+
"check": _cmd_check,
|
|
100
|
+
"migrate-check": _cmd_migrate,
|
|
101
|
+
"watch": _cmd_watch,
|
|
102
|
+
"monitor": _cmd_monitor,
|
|
103
|
+
}[args.cmd](args)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
if __name__ == "__main__":
|
|
107
|
+
sys.exit(main())
|
mcpvitals/introspect.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import shlex
|
|
3
|
+
import anyio
|
|
4
|
+
from mcp import ClientSession, StdioServerParameters
|
|
5
|
+
from mcp.client.stdio import stdio_client
|
|
6
|
+
from mcp.client.streamable_http import streamablehttp_client
|
|
7
|
+
from mcpvitals.models import ServerSnapshot, ToolSpec
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
async def _collect(session: ClientSession) -> ServerSnapshot:
|
|
11
|
+
init = await session.initialize()
|
|
12
|
+
tools = (await session.list_tools()).tools
|
|
13
|
+
tool_specs = [ToolSpec(t.name, t.description or "", t.inputSchema or {}) for t in tools]
|
|
14
|
+
info = init.serverInfo
|
|
15
|
+
caps = init.capabilities.model_dump(exclude_none=True) if init.capabilities else {}
|
|
16
|
+
return ServerSnapshot(
|
|
17
|
+
name=getattr(info, "name", ""),
|
|
18
|
+
version=getattr(info, "version", ""),
|
|
19
|
+
protocol_version=str(getattr(init, "protocolVersion", "")),
|
|
20
|
+
tools=tool_specs,
|
|
21
|
+
raw={"capabilities": caps},
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
async def _run(target: str) -> ServerSnapshot:
|
|
26
|
+
if target.startswith("http://") or target.startswith("https://"):
|
|
27
|
+
async with streamablehttp_client(target) as (read, write, _):
|
|
28
|
+
async with ClientSession(read, write) as session:
|
|
29
|
+
return await _collect(session)
|
|
30
|
+
parts = shlex.split(target)
|
|
31
|
+
params = StdioServerParameters(command=parts[0], args=parts[1:])
|
|
32
|
+
async with stdio_client(params) as (read, write):
|
|
33
|
+
async with ClientSession(read, write) as session:
|
|
34
|
+
return await _collect(session)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def introspect(target: str, timeout: float = 20.0) -> ServerSnapshot:
|
|
38
|
+
return anyio.run(_run, target)
|
mcpvitals/lock.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import json
|
|
3
|
+
from mcpvitals.models import Finding, Level
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def hash_tool(tool) -> str:
|
|
7
|
+
blob = json.dumps(
|
|
8
|
+
{"name": tool.name, "description": tool.description, "input_schema": tool.input_schema},
|
|
9
|
+
sort_keys=True,
|
|
10
|
+
)
|
|
11
|
+
return hashlib.sha256(blob.encode()).hexdigest()[:16]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def lock_from_snapshot(snapshot) -> dict:
|
|
15
|
+
return {"tools": {t.name: hash_tool(t) for t in snapshot.tools}}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def diff_locks(old: dict, new: dict) -> list:
|
|
19
|
+
old_t = old.get("tools", {})
|
|
20
|
+
new_t = new.get("tools", {})
|
|
21
|
+
out = []
|
|
22
|
+
for name in new_t:
|
|
23
|
+
if name not in old_t:
|
|
24
|
+
out.append(Finding("MV060", Level.WARN, f"new tool appeared: '{name}'",
|
|
25
|
+
hint="a server adding tools after approval can be a rug-pull", tool=name))
|
|
26
|
+
elif old_t[name] != new_t[name]:
|
|
27
|
+
out.append(Finding("MV062", Level.ERROR, f"tool '{name}' changed since it was pinned",
|
|
28
|
+
hint="its schema or description mutated; re-review before trusting", tool=name))
|
|
29
|
+
for name in old_t:
|
|
30
|
+
if name not in new_t:
|
|
31
|
+
out.append(Finding("MV061", Level.WARN, f"tool removed: '{name}'", tool=name))
|
|
32
|
+
return out
|
mcpvitals/models.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
from enum import Enum
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Level(str, Enum):
|
|
7
|
+
ERROR = "error"
|
|
8
|
+
WARN = "warn"
|
|
9
|
+
INFO = "info"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class Finding:
|
|
14
|
+
code: str
|
|
15
|
+
level: Level
|
|
16
|
+
message: str
|
|
17
|
+
hint: str | None = None
|
|
18
|
+
tool: str | None = None
|
|
19
|
+
weight: int = 0
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class ToolSpec:
|
|
24
|
+
name: str
|
|
25
|
+
description: str = ""
|
|
26
|
+
input_schema: dict = field(default_factory=dict)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class ServerSnapshot:
|
|
31
|
+
name: str
|
|
32
|
+
version: str
|
|
33
|
+
protocol_version: str
|
|
34
|
+
tools: list[ToolSpec] = field(default_factory=list)
|
|
35
|
+
prompts: list = field(default_factory=list)
|
|
36
|
+
resources: list = field(default_factory=list)
|
|
37
|
+
raw: dict = field(default_factory=dict)
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def from_dict(cls, d: dict) -> "ServerSnapshot":
|
|
41
|
+
tools = [ToolSpec(t["name"], t.get("description", ""), t.get("input_schema", {}))
|
|
42
|
+
for t in d.get("tools", [])]
|
|
43
|
+
return cls(
|
|
44
|
+
name=d.get("name", ""),
|
|
45
|
+
version=d.get("version", ""),
|
|
46
|
+
protocol_version=d.get("protocol_version", ""),
|
|
47
|
+
tools=tools,
|
|
48
|
+
prompts=d.get("prompts", []),
|
|
49
|
+
resources=d.get("resources", []),
|
|
50
|
+
raw=d.get("raw", {}),
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class Report:
|
|
56
|
+
score: int
|
|
57
|
+
grade: str
|
|
58
|
+
findings: list[Finding] = field(default_factory=list)
|
|
59
|
+
extras: dict = field(default_factory=dict)
|
mcpvitals/monitor.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import time
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def probe(target: str) -> dict:
|
|
5
|
+
from mcpvitals.introspect import introspect
|
|
6
|
+
start = time.monotonic()
|
|
7
|
+
try:
|
|
8
|
+
snap = introspect(target)
|
|
9
|
+
return {"ok": True, "latency_ms": (time.monotonic() - start) * 1000,
|
|
10
|
+
"tools": len(snap.tools), "error": None}
|
|
11
|
+
except Exception as e: # noqa: BLE001
|
|
12
|
+
return {"ok": False, "latency_ms": (time.monotonic() - start) * 1000,
|
|
13
|
+
"tools": 0, "error": str(e)}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def summarize(results: list) -> dict:
|
|
17
|
+
n = len(results)
|
|
18
|
+
ok = sum(1 for r in results if r["ok"])
|
|
19
|
+
lats = [r["latency_ms"] for r in results if r["ok"]]
|
|
20
|
+
return {
|
|
21
|
+
"probes": n,
|
|
22
|
+
"up": ok,
|
|
23
|
+
"uptime_pct": round(100 * ok / n, 1) if n else 0.0,
|
|
24
|
+
"avg_latency_ms": round(sum(lats) / len(lats), 1) if lats else None,
|
|
25
|
+
"max_latency_ms": round(max(lats), 1) if lats else None,
|
|
26
|
+
}
|
mcpvitals/report.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from mcpvitals.models import Report, Level
|
|
3
|
+
|
|
4
|
+
_SYM = {Level.ERROR: "x", Level.WARN: "!", Level.INFO: "i"}
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def render_human(report: Report) -> str:
|
|
8
|
+
lines = [f"mcp health: {report.score} ({report.grade})", ""]
|
|
9
|
+
for f in report.findings:
|
|
10
|
+
tag = f" [{f.tool}]" if f.tool else ""
|
|
11
|
+
lines.append(f" {_SYM[f.level]} {f.code}{tag} {f.message}")
|
|
12
|
+
if f.hint:
|
|
13
|
+
lines.append(f" -> {f.hint}")
|
|
14
|
+
if not report.findings:
|
|
15
|
+
lines.append(" no issues found")
|
|
16
|
+
return "\n".join(lines)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def render_json(report: Report) -> str:
|
|
20
|
+
return json.dumps({
|
|
21
|
+
"score": report.score,
|
|
22
|
+
"grade": report.grade,
|
|
23
|
+
"findings": [
|
|
24
|
+
{"code": f.code, "level": f.level.value, "message": f.message,
|
|
25
|
+
"hint": f.hint, "tool": f.tool}
|
|
26
|
+
for f in report.findings
|
|
27
|
+
],
|
|
28
|
+
}, indent=2)
|
mcpvitals/scoring.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from mcpvitals.models import Finding, Level, Report
|
|
2
|
+
|
|
3
|
+
WEIGHTS = {Level.ERROR: 15, Level.WARN: 5, Level.INFO: 0}
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _grade(s):
|
|
7
|
+
return "A" if s >= 90 else "B" if s >= 80 else "C" if s >= 70 else "D" if s >= 60 else "F"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def score(findings) -> Report:
|
|
11
|
+
deduction = sum(f.weight or WEIGHTS[f.level] for f in findings)
|
|
12
|
+
s = max(0, 100 - deduction)
|
|
13
|
+
return Report(score=s, grade=_grade(s), findings=list(findings))
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mcpvitals
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Vital signs for your MCP servers: health score, token cost, tool-confusion, migration readiness.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Requires-Dist: mcp>=1.0
|
|
9
|
+
Provides-Extra: dev
|
|
10
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
11
|
+
Provides-Extra: tokens
|
|
12
|
+
Requires-Dist: tiktoken>=0.7; extra == 'tokens'
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
|
|
15
|
+
<div align="center">
|
|
16
|
+
|
|
17
|
+
# mcpvitals
|
|
18
|
+
|
|
19
|
+
**Vital signs for your MCP servers. One command gives you a health score, plus the checks nobody else runs.**
|
|
20
|
+
|
|
21
|
+
[](https://github.com/ContextJet-ai/mcpvitals/actions/workflows/tests.yml)
|
|
22
|
+
[](https://pypi.org/project/mcpvitals/)
|
|
23
|
+
[](LICENSE)
|
|
24
|
+
[](https://www.contextjetai.com)
|
|
25
|
+
|
|
26
|
+
</div>
|
|
27
|
+
|
|
28
|
+
> [!NOTE]
|
|
29
|
+
> An audit of ~1,850 public MCP servers found **52% abandoned and only 17% production-ready**, and a separate benchmark traced **56.7% of agent failures to tool misuse** (wrong tool, bad arguments). Registries have no health signal, so a broken server looks identical to a good one. `mcpvitals` is the checkup.
|
|
30
|
+
|
|
31
|
+
## 60-second start
|
|
32
|
+
|
|
33
|
+
No install. Point it at any MCP server (a stdio command or an http url):
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
uvx mcpvitals check "npx -y @your/mcp-server"
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
```text
|
|
40
|
+
mcp health: 40 (F)
|
|
41
|
+
|
|
42
|
+
x MV020 [run_sql] tool 'run_sql' appears to contain a hard-coded secret
|
|
43
|
+
-> never ship keys in tool metadata; use env vars
|
|
44
|
+
x MV022 [run_sql] description contains model-directed instructions (poisoning surface)
|
|
45
|
+
-> tool descriptions should describe, not instruct the model
|
|
46
|
+
! MV030 [run_sql] tool 'run_sql' costs ~410 tokens to expose
|
|
47
|
+
-> trim the description/schema; it inflates every request
|
|
48
|
+
! MV040 tools 'get_user' and 'fetch_user' are 0.91 similar; agents may pick the wrong one
|
|
49
|
+
-> differentiate names/descriptions or merge them
|
|
50
|
+
i MV050 protocol 2025-03-26 predates the 2026-07-28 stateless spec
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## What it checks
|
|
54
|
+
|
|
55
|
+
Most linters stop at basic conformance. `mcpvitals` runs those, plus four things no other tool does:
|
|
56
|
+
|
|
57
|
+
| Area | What it catches |
|
|
58
|
+
|---|---|
|
|
59
|
+
| **Conformance** | empty tool descriptions, duplicate tool names, malformed schemas |
|
|
60
|
+
| **Reliability** | parameters with no `required` list, untyped params, error-path gaps |
|
|
61
|
+
| **Security hygiene** | secrets in tool metadata, over-broad tools (`run_sql`, `exec`), model-directed instructions in descriptions (the tool-poisoning surface) |
|
|
62
|
+
| **Token cost** | how many tokens each tool burns in the context window, and the total cost to connect the server |
|
|
63
|
+
| **Tool confusion** | pairs of tools an agent will mix up, predicted from name/description overlap *before* it happens in production |
|
|
64
|
+
| **Migration readiness** | what breaks under the 2026-07-28 stateless spec |
|
|
65
|
+
|
|
66
|
+
Every check is a pure function scored deterministically. Run `mcpvitals check --json` for machine output.
|
|
67
|
+
|
|
68
|
+
## Add the health badge to your README
|
|
69
|
+
|
|
70
|
+
`mcpvitals` emits a shareable badge so a good score is visible at a glance:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
mcpvitals check "npx your-server" --badge badge.svg
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Or use the shields.io endpoint output to keep it live.
|
|
77
|
+
|
|
78
|
+
## Use it in CI
|
|
79
|
+
|
|
80
|
+
Gate your server on every push with the bundled GitHub Action:
|
|
81
|
+
|
|
82
|
+
```yaml
|
|
83
|
+
- uses: ContextJet-ai/mcpvitals@v0
|
|
84
|
+
with:
|
|
85
|
+
target: "python -m your_server"
|
|
86
|
+
strict: "true" # fail the build on any error-level finding
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Track and monitor (not just a one-shot check)
|
|
90
|
+
|
|
91
|
+
Pin a server's tools and get alerted when they change. This catches **rug-pulls**, where a server you already approved silently mutates a tool's schema or description after the fact:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
mcpvitals watch "npx your-server" # pins tools to mcp.lock
|
|
95
|
+
mcpvitals watch "npx your-server" --check # re-run in CI; fails if a pinned tool changed
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Probe a running server for live health (uptime and latency):
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
mcpvitals monitor "https://your-server/mcp" --count 5
|
|
102
|
+
# probe 1/5: up 210ms ... uptime 100.0% avg 207ms
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## Why this exists
|
|
106
|
+
|
|
107
|
+
The MCP ecosystem grew faster than the tooling around it. There are thousands of servers and no standard way to tell whether one is any good, safe, or cheap to run. `mcpvitals` gives authors a one-command checkup before they publish, and gives everyone else a way to vet a server before they trust it.
|
|
108
|
+
|
|
109
|
+
Built by [ContextJet.ai](https://www.contextjetai.com). MIT licensed. Contributions welcome, see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
mcpvitals/__init__.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
|
|
2
|
+
mcpvitals/badge.py,sha256=Y3c0237bWrSRLJMF8oTlQjAJaJA-TRbXfgPvU_roJO8,970
|
|
3
|
+
mcpvitals/cli.py,sha256=rFLHMb39EdfQkJPLnyXeRtEcAxAlXMUcUgnCBUiwm8s,3880
|
|
4
|
+
mcpvitals/introspect.py,sha256=AU5pRW9JOj-gmlFByDUlPAyUNe4LvKwgPuwF3vGcF8s,1538
|
|
5
|
+
mcpvitals/lock.py,sha256=hIbHEnwfmYLt2BC5O3YqyKEYNODZa2YsaZY1MvZQux8,1173
|
|
6
|
+
mcpvitals/models.py,sha256=mI11uBTf1FCcj79VhJ12fs78Sy0E3OXjvjjUVfmMT8Q,1441
|
|
7
|
+
mcpvitals/monitor.py,sha256=MFws2iWfyHy-zGbCWJ81QBHnSjuGT6SvQxxwvxHJHcE,888
|
|
8
|
+
mcpvitals/report.py,sha256=A6zismgdhH5hKIafDprFMPcwaFvA2idwft4a0QdM958,869
|
|
9
|
+
mcpvitals/scoring.py,sha256=idK6bGJN8Xv9DsSC8_wiYyZrazCCfIIcQKphLzth9Og,425
|
|
10
|
+
mcpvitals/checks/__init__.py,sha256=CT92THA9L-RcfU68oXcO7sVlVqRt2C8CIUF8F6GHl54,632
|
|
11
|
+
mcpvitals/checks/conformance.py,sha256=j4dAVHqDXmfZB--anUj4bom2vbvHOuFHxPxb2rl7Jmg,717
|
|
12
|
+
mcpvitals/checks/confusion.py,sha256=bhY6Mmw2pGXCDw5ueNixVftNQW6d-58D9UntgGNQkpU,917
|
|
13
|
+
mcpvitals/checks/migration.py,sha256=Z7Cha_ZiLlHEmLMqmtV3dELjK4a7pNxlgCf4cphfYpg,825
|
|
14
|
+
mcpvitals/checks/reliability.py,sha256=URSwDR1Fp1dzZpWOCr9-WJXYBTtUernd0-kTEmo0ago,1021
|
|
15
|
+
mcpvitals/checks/security.py,sha256=8PSbRflXPMezxE2VVpCpZCBC4AOnfgICNNg2uskdc_c,1641
|
|
16
|
+
mcpvitals/checks/tokens.py,sha256=Vyxwjtu9SP0tW_G1PZqZj0KtQUT9Vij6EcG6_YV18bo,1133
|
|
17
|
+
mcpvitals-0.1.0.dist-info/METADATA,sha256=1bqx6yrlDhp3k0sRVL1R7frkxV4wxRgNndUsWncvWmM,4632
|
|
18
|
+
mcpvitals-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
19
|
+
mcpvitals-0.1.0.dist-info/entry_points.txt,sha256=pChmu9QPgpBl6vPBhb9c2kxqY0NKMu-dJgAJRxBE77k,49
|
|
20
|
+
mcpvitals-0.1.0.dist-info/licenses/LICENSE,sha256=u61UZ7kZqsXoGwUc1kAXAz6gxIWEayANbQqng-mrE04,1070
|
|
21
|
+
mcpvitals-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ContextJet.ai
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|