agentcheckup 0.4.0__py3-none-any.whl

agentcheckup/__init__.py

@@ -0,0 +1,27 @@
+"""agentcheckup - diagnose your agent setup before your agent does."""
+
+from __future__ import annotations
+
+from .conflicts import Conflict, ToolRef, detect_conflicts, lint_conflicts
+from .discovery import discover, load_servers
+from .engine import Report, lint_audit, lint_servers, lint_tools
+from .models import Finding, ServerConfig, Severity, Tool
+from .report import __version__
+
+__all__ = [
+    "Conflict",
+    "Finding",
+    "Report",
+    "ServerConfig",
+    "Severity",
+    "Tool",
+    "ToolRef",
+    "__version__",
+    "detect_conflicts",
+    "discover",
+    "lint_audit",
+    "lint_conflicts",
+    "lint_servers",
+    "lint_tools",
+    "load_servers",
+]

agentcheckup/__main__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agentcheckup/cli.py

@@ -0,0 +1,242 @@
+"""Command line entry point."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from .discovery import discover, load_servers
+from .engine import Report, lint_audit, lint_servers, lint_tools
+from .loader import LoadError, StdioSession, load_from_file, servers_from_config
+from .models import ServerConfig, Severity
+from .report import __version__, render_json, render_rule_list, render_sarif, render_text
+
+EXIT_OK = 0
+EXIT_FINDINGS = 1
+EXIT_USAGE = 2
+
+THRESHOLDS = {
+    "error": Severity.ERROR,
+    "warning": Severity.WARNING,
+    "info": Severity.INFO,
+}
+
+
+def _add_output_flags(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--format", choices=("text", "json", "sarif"), default="text")
+    parser.add_argument(
+        "--fail-on",
+        choices=("error", "warning", "info", "never"),
+        default="error",
+        help="Exit non-zero at or above this severity (default: error).",
+    )
+    parser.add_argument("--min-score", type=int, default=None, help="Also fail below this score.")
+    parser.add_argument("--verbose", action="store_true", help="Show extra detail per finding.")
+    parser.add_argument(
+        "--json-out",
+        metavar="PATH",
+        default=None,
+        help="Also write the JSON report here, whatever --format says.",
+    )
+    parser.add_argument(
+        "--sarif-out",
+        metavar="PATH",
+        default=None,
+        help="Also write a SARIF report here. One run, three outputs.",
+    )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="agentcheckup",
+        description="Diagnose your agent setup: tool conflicts, risky config, context budget.",
+    )
+    parser.add_argument("--version", action="version", version=f"agentcheckup {__version__}")
+    sub = parser.add_subparsers(dest="command")
+
+    audit = sub.add_parser(
+        "audit",
+        help="Audit every MCP client config found on this machine (default command).",
+    )
+    audit.add_argument(
+        "--config",
+        action="append",
+        default=[],
+        metavar="PATH",
+        help="Extra config file to include (repeatable).",
+    )
+    audit.add_argument(
+        "--timeout",
+        type=float,
+        default=25.0,
+        help="Seconds to wait for each server's tools/list (default: 25).",
+    )
+    audit.add_argument(
+        "--no-spawn",
+        action="store_true",
+        help="Do not start servers; check configs only. Fast, but no conflict detection.",
+    )
+    audit.add_argument(
+        "--no-discover",
+        action="store_true",
+        help="Do not auto-find configs; audit only the ones passed with --config.",
+    )
+    _add_output_flags(audit)
+
+    scan = sub.add_parser("scan", help="Lint one tools/list manifest (file or live server).")
+    scan.add_argument("path", nargs="?", help="JSON file containing a tools array.")
+    scan.add_argument("--cmd", dest="cmd", help="Spawn this stdio server and lint its live tools/list.")
+    _add_output_flags(scan)
+
+    cfg = sub.add_parser("config", help="Audit a single client config file.")
+    cfg.add_argument("path", help="e.g. claude_desktop_config.json or .vscode/mcp.json")
+    _add_output_flags(cfg)
+
+    sub.add_parser("rules", help="List every rule with its id and severity.")
+    return parser
+
+
+def _emit(report: Report, args: argparse.Namespace) -> None:
+    # Side outputs first: CI reads them even when the human view is what printed.
+    if args.json_out:
+        Path(args.json_out).write_text(render_json(report), encoding="utf-8")
+    if args.sarif_out:
+        Path(args.sarif_out).write_text(render_sarif(report), encoding="utf-8")
+
+    if args.format == "json":
+        print(render_json(report))
+    elif args.format == "sarif":
+        print(render_sarif(report))
+    else:
+        print(render_text(report, verbose=args.verbose))
+
+
+def _exit_code(report: Report, fail_on: str, min_score: int | None) -> int:
+    if min_score is not None and report.score < min_score:
+        return EXIT_FINDINGS
+    if fail_on == "never":
+        return EXIT_OK
+    return EXIT_FINDINGS if report.exceeding(THRESHOLDS[fail_on]) else EXIT_OK
+
+
+def _dedupe(servers: list[ServerConfig]) -> list[ServerConfig]:
+    """The same server is often declared in several configs.
+
+    Spawning it twice would double its tools and invent conflicts that do not
+    exist, so identity is the launch spec, not the name.
+    """
+    seen: set[tuple] = set()
+    out: list[ServerConfig] = []
+    for server in servers:
+        key = (
+            (server.command or "").lower(),
+            tuple(a.lower() for a in server.args),
+            (server.url or "").lower(),
+        )
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(server)
+    return out
+
+
+def _run_audit(args: argparse.Namespace) -> int:
+    configs = [] if args.no_discover else discover()
+    for extra in args.config:
+        path = Path(extra)
+        if not path.is_file():
+            print(f"agentcheckup: no such config file: {path}", file=sys.stderr)
+            return EXIT_USAGE
+        configs.append(("explicit", path))
+
+    if not configs:
+        print(
+            "agentcheckup: no MCP client config found.\n"
+            "Looked for Claude Desktop, Claude Code, Cursor, Windsurf, Cline and\n"
+            "project-local .mcp.json / .cursor/mcp.json / .vscode/mcp.json.\n"
+            "Point it at one explicitly: agentcheckup audit --config path/to/config.json",
+            file=sys.stderr,
+        )
+        return EXIT_USAGE
+
+    servers: list[ServerConfig] = []
+    for label, path in configs:
+        servers.extend(load_servers(path, label))
+    servers = _dedupe(servers)
+
+    if not servers:
+        print("agentcheckup: configs found, but none declare an MCP server.", file=sys.stderr)
+        return EXIT_USAGE
+
+    tools_by_server: dict[str, list] = {}
+    failures: dict[str, str] = {}
+
+    if args.no_spawn:
+        # Config-only mode: no tools, so no tool or conflict findings.
+        for server in servers:
+            failures[server.name] = "not started (--no-spawn)"
+    else:
+        for server in servers:
+            if not server.command:
+                failures[server.name] = "remote server (url); cannot be introspected over stdio"
+                continue
+            argv = [server.command, *server.args]
+            try:
+                with StdioSession(argv, timeout=args.timeout, env=server.env) as session:
+                    tools_by_server[server.name] = session.list_tools()
+            except LoadError as exc:
+                failures[server.name] = str(exc).splitlines()[0][:200]
+
+    report = lint_audit(
+        servers,
+        tools_by_server,
+        failures={} if args.no_spawn else failures,
+        source=f"{len(configs)} config(s)",
+    )
+    _emit(report, args)
+    return _exit_code(report, args.fail_on, args.min_score)
+
+
+def main(argv: list[str] | None = None) -> int:
+    if argv is None:
+        argv = sys.argv[1:]
+
+    # `agentcheckup` with no arguments is the whole point: one command, full audit.
+    if not argv or (argv[0].startswith("-") and argv[0] not in ("-h", "--help", "--version")):
+        argv = ["audit", *argv]
+
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    if args.command == "rules":
+        print(render_rule_list())
+        return EXIT_OK
+
+    if args.command == "audit":
+        return _run_audit(args)
+
+    try:
+        if args.command == "scan":
+            if bool(args.path) == bool(args.cmd):
+                parser.error("scan needs exactly one of: a JSON path, or --cmd")
+            if args.cmd:
+                with StdioSession(args.cmd) as session:
+                    tools = session.list_tools()
+                report = lint_tools(tools, source=f"cmd: {args.cmd}")
+            else:
+                path = Path(args.path)
+                report = lint_tools(load_from_file(path), source=str(path))
+        else:
+            path = Path(args.path)
+            report = lint_servers(servers_from_config(path), source=str(path))
+    except LoadError as exc:
+        print(f"agentcheckup: {exc}", file=sys.stderr)
+        return EXIT_USAGE
+
+    _emit(report, args)
+    return _exit_code(report, args.fail_on, args.min_score)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agentcheckup/conflicts.py

@@ -0,0 +1,253 @@
+"""Cross-server tool conflict detection.
+
+This is the reason the tool exists. Every other scanner looks at one server at a
+time; nobody asks what happens when five servers are loaded at once and three of
+them expose something called ``search``.
+
+Two servers cannot see each other. The model can, and it has no way to know that
+``github.search`` and ``docs.search`` are different things - it gets a flat list
+of 47 tools and picks by description similarity. When two descriptions overlap,
+selection becomes a coin flip that looks like model flakiness.
+
+No dependencies: similarity is token-set Jaccard over normalised text. An
+embedding would be better, but it would also mean a model download, a network
+call, or a 200MB wheel - and then the linter stops working on a plane.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from itertools import combinations
+
+from .models import Finding, Severity, Tool
+
+#: Words that carry no discriminating signal in a tool description.
+STOPWORDS = frozenset(
+    {
+        "a", "an", "the", "this", "that", "these", "those",
+        "is", "are", "was", "were", "be", "been", "being",
+        "and", "or", "but", "if", "then", "than", "so", "as",
+        "at", "by", "for", "from", "in", "into", "of", "on", "to", "with",
+        "it", "its", "you", "your", "we", "our", "they", "them", "their",
+        "he", "she", "his", "her",
+        "can", "could", "may", "might", "must", "shall", "should", "will", "would",
+        "do", "does", "did", "done",
+        "make", "makes", "made", "use", "uses", "used", "using",
+        "tool", "tools", "function", "functions", "call", "calls", "called",
+        "return", "returns", "returned", "value", "values", "result", "results",
+        "given", "specify", "specified", "provide", "provides",
+    }
+)
+
+WORD = re.compile(r"[a-z0-9]+")
+
+
+@dataclass(frozen=True)
+class ToolRef:
+    """A tool plus the server it came from."""
+
+    server: str
+    tool: Tool
+
+    @property
+    def qualified(self) -> str:
+        return f"{self.server}/{self.tool.name}"
+
+
+@dataclass(frozen=True)
+class Conflict:
+    rule: str
+    severity: Severity
+    a: ToolRef
+    b: ToolRef
+    similarity: float
+    message: str
+    hint: str
+
+    def as_finding(self) -> Finding:
+        return Finding(
+            rule=self.rule,
+            severity=self.severity,
+            message=self.message,
+            tool=f"{self.a.qualified} <-> {self.b.qualified}",
+            hint=self.hint,
+            where=f"similarity={self.similarity:.2f}",
+        )
+
+
+def _stem(word: str) -> str:
+    """Crude English plural stripping.
+
+    Deliberately not a real stemmer: `read_files` and `read_file` must collapse
+    to the same token, and that is the only job. A full Porter stemmer would be
+    a dependency for no benefit.
+    """
+    if len(word) <= 3:
+        return word
+    if word.endswith("ies") and len(word) > 4:
+        return word[:-3] + "y"
+    if word.endswith(("ses", "xes", "zes", "ches", "shes")):
+        return word[:-2]
+    if word.endswith("s") and not word.endswith("ss"):
+        return word[:-1]
+    return word
+
+
+def tokens(text: str) -> set[str]:
+    """Lowercase word set, stopwords removed, crude plural stemming."""
+    return {
+        _stem(w)
+        for w in WORD.findall(text.lower())
+        if w not in STOPWORDS and len(w) > 1
+    }
+
+
+def jaccard(a: set[str], b: set[str]) -> float:
+    if not a or not b:
+        return 0.0
+    union = a | b
+    return len(a & b) / len(union)
+
+
+def name_tokens(name: str) -> set[str]:
+    return {_stem(w) for w in WORD.findall(name.lower()) if len(w) > 1}
+
+
+def similarity(a: ToolRef, b: ToolRef) -> float:
+    """Blend name and description overlap.
+
+    Name overlap matters more when it is exact, because that is a hard
+    collision rather than a soft one.
+    """
+    name_sim = jaccard(name_tokens(a.tool.name), name_tokens(b.tool.name))
+    desc_sim = jaccard(tokens(a.tool.description), tokens(b.tool.description))
+    return round(0.45 * name_sim + 0.55 * desc_sim, 3)
+
+
+def is_ambiguous(a: ToolRef, b: ToolRef, threshold: float = 0.55) -> tuple[bool, float]:
+    """Decide whether two tools are genuinely confusable.
+
+    Two signals, either is sufficient:
+
+    - the descriptions overlap heavily (>= 0.65) and the names share a head
+      noun: the model reads the description, so this is the primary signal;
+    - the blended score clears ``threshold``.
+
+    Requiring a shared noun is what keeps ``search_repositories`` and
+    ``search_documents`` apart. Without it the false-positive rate made the
+    whole report untrustworthy.
+    """
+    name_sim = jaccard(name_tokens(a.tool.name), name_tokens(b.tool.name))
+    desc_sim = jaccard(tokens(a.tool.description), tokens(b.tool.description))
+    blended = round(0.45 * name_sim + 0.55 * desc_sim, 3)
+
+    if not _same_object(a, b):
+        return False, blended
+    return (desc_sim >= 0.65 or blended >= threshold), blended
+
+
+def _same_object(a: ToolRef, b: ToolRef) -> bool:
+    """Two names sharing a head noun are more likely to collide.
+
+    ``search_docs`` and ``search_code`` both end in a different noun, so they are
+    probably fine. ``search`` and ``search_docs`` share one, so they are not.
+    """
+    ta, tb = name_tokens(a.tool.name), name_tokens(b.tool.name)
+    return bool(ta & tb)
+
+
+def detect_conflicts(
+    refs: list[ToolRef],
+    *,
+    threshold: float = 0.55,
+    max_pairs: int = 60_000,
+) -> list[Conflict]:
+    """Find tool pairs across servers that an agent cannot reliably tell apart."""
+    conflicts: list[Conflict] = []
+    seen: set[tuple[str, str]] = set()
+
+    pairs = list(combinations(refs, 2))
+    if len(pairs) > max_pairs:  # pathological manifests: compare only plausible pairs
+        pairs = [
+            (a, b)
+            for a, b in pairs
+            if a.server != b.server and _same_object(a, b)
+        ]
+
+    for a, b in pairs:
+        if a.server == b.server:
+            continue  # same server: a naming problem, not a conflict problem
+
+        key = tuple(sorted([a.qualified, b.qualified]))
+        if key in seen:
+            continue
+        seen.add(key)
+
+        name_a = a.tool.name.lower()
+        name_b = b.tool.name.lower()
+
+        # Hard collision: identical names, different servers. One is unreachable.
+        if name_a == name_b:
+            conflicts.append(
+                Conflict(
+                    rule="AGT201",
+                    severity=Severity.ERROR,
+                    a=a,
+                    b=b,
+                    similarity=1.0,
+                    message=(
+                        f"Tool name '{a.tool.name}' is exposed by both "
+                        f"'{a.server}' and '{b.server}'."
+                    ),
+                    hint=(
+                        "Hard collision: the client keeps one and the other is "
+                        "unreachable. Rename one, or use a gateway that prefixes."
+                    ),
+                )
+            )
+            continue
+
+        # Same name modulo punctuation/plural: `read_file` vs `readFiles`.
+        if name_tokens(a.tool.name) == name_tokens(b.tool.name):
+            conflicts.append(
+                Conflict(
+                    rule="AGT202",
+                    severity=Severity.WARNING,
+                    a=a,
+                    b=b,
+                    similarity=1.0,
+                    message=(
+                        f"Tools '{a.qualified}' and '{b.qualified}' have the same "
+                        "name once normalised."
+                    ),
+                    hint="Rename one so the model can tell them apart by name alone.",
+                )
+            )
+            continue
+
+        flagged, score = is_ambiguous(a, b, threshold=threshold)
+        if flagged:
+            conflicts.append(
+                Conflict(
+                    rule="AGT203",
+                    severity=Severity.WARNING,
+                    a=a,
+                    b=b,
+                    similarity=score,
+                    message=(
+                        f"'{a.qualified}' and '{b.qualified}' overlap "
+                        f"({score:.0%}); the model picks between them at random."
+                    ),
+                    hint=(
+                        "Add a disambiguation clause to both descriptions: "
+                        '"Use this only for X; use <other tool> for Y."'
+                    ),
+                )
+            )
+
+    return sorted(conflicts, key=lambda c: (-c.severity.rank, -c.similarity))
+
+
+def lint_conflicts(refs: list[ToolRef], threshold: float = 0.55) -> list[Finding]:
+    return [c.as_finding() for c in detect_conflicts(refs, threshold=threshold)]