agentcheckup 0.4.0__py3-none-any.whl

agentcheckup/models.py

@@ -0,0 +1,166 @@
+"""Core data model for agentcheckup.
+
+Everything here is deliberately dependency-free so the package keeps working
+across MCP spec revisions without chasing an SDK release.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+class Severity(str, Enum):
+    ERROR = "error"
+    WARNING = "warning"
+    INFO = "info"
+
+    @property
+    def rank(self) -> int:
+        return {"error": 3, "warning": 2, "info": 1}[self.value]
+
+
+@dataclass(frozen=True)
+class Finding:
+    """A single lint result.
+
+    ``rule`` is a stable id (AGT0xx) so CI configs and suppressions never break
+    when wording changes.
+    """
+
+    rule: str
+    severity: Severity
+    message: str
+    tool: str | None = None
+    hint: str | None = None
+    where: str | None = None
+
+    def as_dict(self) -> dict[str, Any]:
+        out: dict[str, Any] = {
+            "rule": self.rule,
+            "severity": self.severity.value,
+            "message": self.message,
+        }
+        if self.tool:
+            out["tool"] = self.tool
+        if self.hint:
+            out["hint"] = self.hint
+        if self.where:
+            out["where"] = self.where
+        return out
+
+
+@dataclass
+class Tool:
+    """One entry from ``tools/list``."""
+
+    name: str
+    description: str = ""
+    input_schema: dict[str, Any] = field(default_factory=dict)
+    annotations: dict[str, Any] = field(default_factory=dict)
+    raw: dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def from_raw(cls, data: dict[str, Any]) -> Tool:
+        schema = data.get("inputSchema") or data.get("input_schema") or {}
+        if not isinstance(schema, dict):
+            schema = {}
+        annotations = data.get("annotations") or {}
+        if not isinstance(annotations, dict):
+            annotations = {}
+        description = data.get("description") or ""
+        if not isinstance(description, str):
+            description = str(description)
+        return cls(
+            name=str(data.get("name", "")),
+            description=description,
+            input_schema=schema,
+            annotations=annotations,
+            raw=data,
+        )
+
+    def properties(self) -> dict[str, Any]:
+        props = self.input_schema.get("properties")
+        return props if isinstance(props, dict) else {}
+
+    def required(self) -> list[str]:
+        req = self.input_schema.get("required")
+        if not isinstance(req, list):
+            return []
+        return [str(x) for x in req]
+
+
+@dataclass
+class ServerConfig:
+    """A server entry lifted from an MCP client config file."""
+
+    name: str
+    command: str | None = None
+    args: list[str] = field(default_factory=list)
+    env: dict[str, str] = field(default_factory=dict)
+    url: str | None = None
+    source: str | None = None
+
+    @classmethod
+    def from_raw(cls, name: str, data: dict[str, Any], source: str | None = None) -> ServerConfig:
+        args = data.get("args") or []
+        if not isinstance(args, list):
+            args = [str(args)]
+        env = data.get("env") or {}
+        if not isinstance(env, dict):
+            env = {}
+        return cls(
+            name=name,
+            command=data.get("command"),
+            args=[str(a) for a in args],
+            env={str(k): str(v) for k, v in env.items()},
+            url=data.get("url") or data.get("serverUrl"),
+            source=source,
+        )
+
+
+def grade_for(score: int) -> str:
+    if score >= 90:
+        return "A"
+    if score >= 80:
+        return "B"
+    if score >= 70:
+        return "C"
+    if score >= 60:
+        return "D"
+    return "F"
+
+
+#: Points charged against a manifest's budget.
+#:
+#: ``info`` is capped rather than free: a rule that fires on 18 of 18 tools in
+#: the official MCP servers is expressing a preference, but a report full of
+#: nudges should not still earn an A either.
+SEVERITY_WEIGHT = {Severity.ERROR: 10, Severity.WARNING: 3, Severity.INFO: 1}
+
+#: Info findings can cost at most this many points in total.
+INFO_CAP = 10
+
+#: Points of budget granted per tool.
+BUDGET_PER_TOOL = 20
+
+
+def score_findings(findings: list[Finding], tool_count: int = 1) -> int:
+    """Score 0-100, normalised per tool so a big manifest is not punished for size.
+
+    Calibrated against four real cases rather than invented: the three official
+    MCP servers (18 tools, 12 warnings, 77 nudges) land at 87; a manifest with a
+    hard collision, an unpinned package and a plaintext token lands at 79; the
+    deliberately broken fixture lands at 35; a clean manifest at 100.
+
+    A flat subtraction scored the official servers 0/100, which tells the reader
+    nothing and makes the tool look broken.
+    """
+    penalty = sum(SEVERITY_WEIGHT[f.severity] for f in findings)
+    infos = sum(1 for f in findings if f.severity is Severity.INFO)
+    if infos:
+        # Replace the per-finding info charge with the capped one.
+        penalty += min(infos, INFO_CAP) - infos
+    budget = BUDGET_PER_TOOL * max(tool_count, 1)
+    return round(100 * max(0.0, 1 - penalty / budget))

agentcheckup/report.py

@@ -0,0 +1,274 @@
+"""Render a Report as text, JSON, or SARIF."""
+
+from __future__ import annotations
+
+import json
+import unicodedata
+from typing import Any
+
+from . import rules as _rules
+from .engine import Report
+from .models import Finding, Severity
+
+__version__ = "0.4.0"
+
+SARIF_LEVEL = {
+    Severity.ERROR: "error",
+    Severity.WARNING: "warning",
+    Severity.INFO: "note",
+}
+
+INVISIBLE = frozenset("\u200b\u200c\u200d\u2060\ufeff")
+
+
+def display(text: str) -> str:
+    """Make invisible characters visible in terminal output.
+
+    A finding about hidden text is useless if the report itself renders it
+    invisibly - the reviewer would see an empty-looking name and move on.
+    """
+    out: list[str] = []
+    for ch in text:
+        if ch in INVISIBLE or unicodedata.category(ch) == "Cf":
+            out.append(f"\\u{ord(ch):04x}")
+        else:
+            out.append(ch)
+    return "".join(out)
+
+
+def _rule_metadata() -> list[dict[str, Any]]:
+    out: list[dict[str, Any]] = []
+    for rule in [*_rules.TOOL_RULES, *_rules.SUITE_RULES, *_rules.CONFIG_RULES]:
+        out.append(
+            {
+                "id": rule.id,
+                "name": rule.id,
+                "shortDescription": {"text": rule.title},
+                "defaultConfiguration": {"level": SARIF_LEVEL[rule.severity]},
+            }
+        )
+    return sorted(out, key=lambda r: r["id"])
+
+
+def _finding_lines(findings: list[Finding], verbose: bool) -> list[str]:
+    out: list[str] = []
+    for finding in findings:
+        target = f"  [{display(finding.tool)}]" if finding.tool else ""
+        out.append(f"    {finding.rule}  {finding.message}{target}")
+        if finding.hint:
+            out.append(f"            fix: {finding.hint}")
+        if verbose and finding.where:
+            out.append(f"            at:  {finding.where}")
+    return out
+
+
+def rule_title(rule_id: str) -> str:
+    for rule in [*_rules.TOOL_RULES, *_rules.SUITE_RULES, *_rules.CONFIG_RULES]:
+        if rule.id == rule_id:
+            return rule.title
+    return ""
+
+
+def _nudge_lines(findings: list[Finding]) -> list[str]:
+    """Collapse info findings into one line per rule.
+
+    Measured on the three official MCP servers: 68 info findings across 18 tools,
+    most from rules that fire on nearly every tool. Printed individually they
+    bury the two findings that matter, and a report nobody reads is a report
+    nobody acts on.
+    """
+    by_rule: dict[str, list[Finding]] = {}
+    for finding in findings:
+        by_rule.setdefault(finding.rule, []).append(finding)
+
+    out: list[str] = []
+    for rule, group in sorted(by_rule.items(), key=lambda kv: (-len(kv[1]), kv[0])):
+        title = rule_title(rule) or group[0].message
+        out.append(f"    {rule}  {title}  ({len(group)})")
+        names = [display(f.tool) for f in group if f.tool]
+        if names:
+            preview = ", ".join(names[:3])
+            if len(names) > 3:
+                preview += f", +{len(names) - 3} more"
+            out.append(f"            {preview}")
+    return out
+
+
+#: A rule that fires on this many tools is one root cause, not N problems.
+AGGREGATE_AT = 5
+
+
+def _serious_lines(findings: list[Finding], verbose: bool) -> list[str]:
+    """Individual findings, except for rules that repeat: those collapse.
+
+    The official git server produced 12 identical "parameter has no description"
+    warnings, one per tool. Twelve near-identical lines hide everything else,
+    while one line naming all twelve tools is both shorter and more actionable.
+    """
+    by_rule: dict[str, list[Finding]] = {}
+    for finding in findings:
+        by_rule.setdefault(finding.rule, []).append(finding)
+
+    ordered = sorted(
+        by_rule.items(),
+        key=lambda kv: (-kv[1][0].severity.rank, -len(kv[1]), kv[0]),
+    )
+
+    out: list[str] = []
+    for rule, group in ordered:
+        if len(group) < AGGREGATE_AT:
+            out.extend(_finding_lines(group, verbose))
+            continue
+
+        title = rule_title(rule) or group[0].message
+        out.append(f"    {rule}  {title}  ({len(group)} tools)")
+        names = [display(f.tool) for f in group if f.tool]
+        if names:
+            preview = ", ".join(names[:4])
+            if len(names) > 4:
+                preview += f", +{len(names) - 4} more"
+            out.append(f"            {preview}")
+        hint = next((f.hint for f in group if f.hint), "")
+        if hint:
+            out.append(f"            fix: {hint}")
+    return out
+
+
+def render_text(report: Report, verbose: bool = False) -> str:
+    lines: list[str] = []
+    if report.kind == "audit":
+        lines.append(
+            f"agentcheckup {__version__}  |  {report.server_count} server(s)  |  "
+            f"{report.tool_count} tool(s)  |  ~{report.tokens:,} tokens per request"
+        )
+    elif report.kind == "config":
+        lines.append(f"agentcheckup {__version__}  |  {report.tool_count} server(s)")
+    else:
+        lines.append(
+            f"agentcheckup {__version__}  |  {report.tool_count} tool(s)  |  "
+            f"~{report.tokens:,} tokens per request"
+        )
+    if report.source:
+        lines.append(f"source: {report.source}")
+
+    counts = report.counts()
+    summary = "  ".join(f"{counts[s.value]} {s.value}" for s in Severity if counts[s.value])
+    lines.append("")
+    lines.append(
+        f"  Grade {report.grade:<2}   Score {report.score:>3}/100   "
+        f"{summary if summary else 'clean'}"
+    )
+    lines.append("")
+
+    if not report.findings:
+        lines.append("  No findings. Ship it.")
+        return "\n".join(lines)
+
+    ordered = report.sorted_findings()
+    serious = [f for f in ordered if f.severity is not Severity.INFO]
+    nudges = [f for f in ordered if f.severity is Severity.INFO]
+
+    if report.kind == "audit":
+        # Two sections, because they answer different questions: "will my agent
+        # pick the wrong tool?" and "is my setup unsafe?".
+        conflicts = [f for f in serious if f.rule.startswith("AGT2")]
+        risks = [f for f in serious if not f.rule.startswith("AGT2")]
+        if conflicts:
+            lines.append(f"  CONFLICTS ({len(conflicts)})")
+            lines.extend(_finding_lines(conflicts, verbose))
+            lines.append("")
+        if risks:
+            lines.append(f"  RISKS ({len(risks)})")
+            lines.extend(_serious_lines(risks, verbose))
+    else:
+        for severity in (Severity.ERROR, Severity.WARNING):
+            group = [f for f in serious if f.severity is severity]
+            if not group:
+                continue
+            lines.append(f"  {severity.value.upper()}")
+            lines.extend(_serious_lines(group, verbose))
+
+    if nudges:
+        rules = len({f.rule for f in nudges})
+        lines.append("")
+        lines.append(f"  NUDGES ({len(nudges)} across {rules} rule(s), no score impact)")
+        lines.extend(_nudge_lines(nudges))
+
+    lines.append("")
+    lines.append("  Rule docs: https://github.com/wanjiaqing/agentcheckup#rules")
+    return "\n".join(lines)
+
+
+def render_json(report: Report) -> str:
+    payload = {
+        "version": __version__,
+        "source": report.source,
+        "server_count": report.server_count,
+        "tool_count": report.tool_count,
+        "estimated_tokens": report.tokens,
+        "score": report.score,
+        "grade": report.grade,
+        "counts": report.counts(),
+        "findings": [f.as_dict() for f in report.sorted_findings()],
+    }
+    return json.dumps(payload, indent=2, ensure_ascii=False)
+
+
+def render_sarif(report: Report) -> str:
+    """SARIF 2.1.0 so findings show up in GitHub code scanning for free."""
+    uri = report.source or "mcp-tools.json"
+    results: list[dict[str, Any]] = []
+    for finding in report.sorted_findings():
+        result: dict[str, Any] = {
+            "ruleId": finding.rule,
+            "level": SARIF_LEVEL[finding.severity],
+            "message": {"text": finding.message + (f" ({finding.tool})" if finding.tool else "")},
+            "locations": [
+                {
+                    "physicalLocation": {
+                        "artifactLocation": {"uri": uri},
+                        "region": {"startLine": 1, "startColumn": 1},
+                    }
+                }
+            ],
+        }
+        if finding.hint:
+            result["fixes"] = []
+            result["properties"] = {"hint": finding.hint}
+        results.append(result)
+
+    payload = {
+        "$schema": "https://json.schemastore.org/sarif-2.1.0.json",
+        "version": "2.1.0",
+        "runs": [
+            {
+                "tool": {
+                    "driver": {
+                        "name": "agentcheckup",
+                        "version": __version__,
+                        "informationUri": "https://github.com/wanjiaqing/agentcheckup",
+                        "rules": _rule_metadata(),
+                    }
+                },
+                "results": results,
+            }
+        ],
+    }
+    return json.dumps(payload, indent=2, ensure_ascii=False)
+
+
+def render_rule_list() -> str:
+    lines = ["Tool rules", "----------"]
+    for rule in sorted(_rules.TOOL_RULES, key=lambda r: r.id):
+        lines.append(f"  {rule.id}  {rule.severity.value:<7}  {rule.title}")
+    lines.append("")
+    lines.append("Suite rules (need the whole manifest)")
+    lines.append("------------------------------------")
+    for rule in sorted(_rules.SUITE_RULES, key=lambda r: r.id):
+        lines.append(f"  {rule.id}  {rule.severity.value:<7}  {rule.title}")
+    lines.append("")
+    lines.append("Config rules")
+    lines.append("------------")
+    for rule in sorted(_rules.CONFIG_RULES, key=lambda r: r.id):
+        lines.append(f"  {rule.id}  {rule.severity.value:<7}  {rule.title}")
+    return "\n".join(lines)

agentcheckup/rules/__init__.py

@@ -0,0 +1,12 @@
+"""Rule registry bootstrap.
+
+Importing this package registers every rule. Order matters only for output
+grouping, not for correctness.
+"""
+
+from __future__ import annotations
+
+from . import config_rules, docs, injection, naming, safety, schema  # noqa: F401
+from .base import CONFIG_RULES, SUITE_RULES, TOOL_RULES
+
+__all__ = ["CONFIG_RULES", "SUITE_RULES", "TOOL_RULES"]

agentcheckup/rules/base.py

@@ -0,0 +1,91 @@
+"""Rule protocol and registry.
+
+Adding a rule should be a single small file plus a decorator - that is the
+whole point, because the fastest way to grow an OSS project is to leave obvious
+room for other people's pull requests.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import NamedTuple, Protocol
+
+from ..models import Finding, Severity
+
+
+class Rule(Protocol):
+    id: str
+    severity: Severity
+    title: str
+
+    def check(self, subject):  # noqa: ANN001 - Tool or ServerConfig
+        ...
+
+
+TOOL_RULES: list[Rule] = []
+CONFIG_RULES: list[Rule] = []
+
+
+class SuiteRule(NamedTuple):
+    """A rule that needs the whole manifest, so it cannot live in a single file."""
+
+    id: str
+    severity: Severity
+    title: str
+
+
+#: Documented here so ``agentcheckup rules`` and the SARIF catalogue stay complete.
+SUITE_RULES: list[SuiteRule] = [
+    SuiteRule("AGT004", Severity.ERROR, "Tool name is duplicated"),
+    SuiteRule("AGT105", Severity.INFO, "Server did not return a tool list"),
+    SuiteRule("AGT201", Severity.ERROR, "Same tool name exposed by two servers"),
+    SuiteRule("AGT202", Severity.WARNING, "Two servers expose the same name once normalised"),
+    SuiteRule("AGT203", Severity.WARNING, "Two servers expose overlapping tools"),
+]
+
+
+def _make_rule(
+    rule_id: str,
+    sev: Severity,
+    rule_title: str,
+    fn: Callable,
+    doc: str | None,
+) -> Rule:
+    class _FnRule:
+        def check(self, subject):  # noqa: ANN001
+            result = fn(subject)
+            if result is None:
+                return []
+            if isinstance(result, Finding):
+                return [result]
+            return list(result)
+
+    _FnRule.id = rule_id
+    _FnRule.severity = sev
+    _FnRule.title = rule_title
+    _FnRule.__doc__ = doc or fn.__doc__
+    return _FnRule()
+
+
+def tool_rule(rule_id: str, severity: Severity, title: str) -> Callable:
+    """Register a tool-level rule.
+
+    The decorated function takes a :class:`Tool` and returns either a single
+    :class:`Finding`, an iterable of them, or ``None``.
+    """
+
+    def decorator(fn: Callable) -> Callable:
+        TOOL_RULES.append(_make_rule(rule_id, severity, title, fn, fn.__doc__))
+        return fn
+
+    return decorator
+
+
+def config_rule(rule_id: str, severity: Severity, title: str) -> Callable:
+    """Register a client-config level rule (operates on ServerConfig)."""
+
+    def decorator(fn: Callable) -> Callable:
+        CONFIG_RULES.append(_make_rule(rule_id, severity, title, fn, fn.__doc__))
+        return fn
+
+    return decorator