agent-capmap 0.1.0__py3-none-any.whl

agentcapmap/cli.py

@@ -0,0 +1,392 @@
+"""Command-line interface."""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from pydantic import BaseModel, ValidationError
+
+from agentcapmap import __version__
+from agentcapmap.baseline import Baseline, BaselineError, create_baseline, load_baseline
+from agentcapmap.classify import Classifier, classify_inventory
+from agentcapmap.classify.cache import ClassificationCache
+from agentcapmap.classify.provider import AnthropicProvider
+from agentcapmap.diff import diff_evaluations, render_markdown, should_fail_diff
+from agentcapmap.discovery import discover, user_config_paths
+from agentcapmap.enumerate import DEFAULT_TIMEOUT, enumerate_inventory
+from agentcapmap.findings import Evaluation, Severity
+from agentcapmap.models import Inventory
+from agentcapmap.policy import PolicyError, evaluate, load_policy
+from agentcapmap.redact import RedactingFilter, get_logger, redact_text
+from agentcapmap.report import Format, render, should_fail
+
+log = get_logger("agentcapmap")
+
+ENUMERATE_WARNING = (
+    "warning: --enumerate launches the stdio server commands found in these configs and "
+    "connects to their URLs. Only enumerate configs you trust."
+)
+_MAX_INVENTORY_BYTES = 64 * 1024 * 1024
+
+# Exit codes: 0 success; 1 findings met --fail-on (or something was not assessed with
+# --fail-on-not-assessed); 2 usage or input error.
+EXIT_FINDINGS = 1
+EXIT_ERROR = 2
+
+
+class CliError(Exception):
+    pass
+
+
+def _setup_logging(verbosity: int) -> None:
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(logging.Formatter("%(levelname)s %(name)s: %(message)s"))
+    handler.addFilter(RedactingFilter())
+    root = logging.getLogger("agentcapmap")
+    root.handlers[:] = [handler]
+    root.setLevel(logging.WARNING - 10 * min(verbosity, 2))
+    root.propagate = False
+
+
+def _add_common(p: argparse.ArgumentParser) -> None:
+    p.add_argument("-o", "--output", type=Path, help="write JSON here instead of stdout")
+    p.add_argument("-v", "--verbose", action="count", default=0, help="-v info, -vv debug")
+
+
+def _add_classify_options(p: argparse.ArgumentParser) -> None:
+    p.add_argument(
+        "--offline",
+        action="store_true",
+        help="use cached classifications only; never call the LLM",
+    )
+    p.add_argument("--cache-dir", type=Path, help="classification cache directory")
+    p.add_argument("--model", help="classifier model (default: $AGENTCAPMAP_MODEL or built-in)")
+    p.add_argument(
+        "--classify-concurrency", type=int, default=4, help="parallel LLM requests (default 4)"
+    )
+
+
+def _add_policy_options(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--policy", type=Path, help="policy YAML (default: built-in default policy)")
+    p.add_argument("--baseline", type=Path, help="baseline JSON for rug-pull detection (CAP-006)")
+
+
+def _add_report_options(p: argparse.ArgumentParser) -> None:
+    p.add_argument(
+        "--format",
+        choices=[f.value for f in Format],
+        default=Format.JSON.value,
+        help="output format for findings (default json)",
+    )
+    p.add_argument(
+        "--fail-on",
+        choices=[*(s.value for s in Severity), "none"],
+        default="none",
+        help="exit 1 if any finding is at or above this severity (default none)",
+    )
+    p.add_argument(
+        "--fail-on-not-assessed",
+        action="store_true",
+        help="exit 1 if any server or tool could not be assessed",
+    )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="agentcapmap",
+        description="Inventory MCP servers reachable by AI agents and map their capabilities.",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"agentcapmap {__version__} (installed at {Path(__file__).resolve().parent})",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    scan = sub.add_parser("scan", help="discover MCP configs and output a normalized inventory")
+    scan.add_argument(
+        "paths",
+        nargs="*",
+        type=Path,
+        help="config files or directories to scan (default: current directory, unless --user)",
+    )
+    scan.add_argument(
+        "--user", action="store_true", help="also scan per-user client configs on this machine"
+    )
+    scan.add_argument(
+        "--enumerate",
+        action="store_true",
+        help="connect to each server and list its tools (initialize + tools/list only)",
+    )
+    scan.add_argument(
+        "--timeout",
+        type=float,
+        default=DEFAULT_TIMEOUT,
+        help=f"per-server enumeration timeout in seconds (default {DEFAULT_TIMEOUT:.0f})",
+    )
+    scan.add_argument(
+        "--concurrency", type=int, default=4, help="servers to enumerate in parallel (default 4)"
+    )
+    scan.add_argument(
+        "--classify", action="store_true", help="classify enumerated tools (needs --enumerate)"
+    )
+    scan.add_argument(
+        "--evaluate",
+        action="store_true",
+        help="apply the policy and output findings instead of the inventory",
+    )
+    scan.add_argument(
+        "--save-inventory", type=Path, help="with --evaluate, also write the inventory here"
+    )
+    _add_classify_options(scan)
+    _add_policy_options(scan)
+    _add_report_options(scan)
+    _add_common(scan)
+
+    classify = sub.add_parser("classify", help="classify the tools in a saved inventory JSON")
+    classify.add_argument("inventory", type=Path, help="inventory JSON from `scan --enumerate`")
+    _add_classify_options(classify)
+    _add_common(classify)
+
+    ev = sub.add_parser("evaluate", help="apply a policy to a saved (classified) inventory")
+    ev.add_argument("inventory", type=Path, help="inventory JSON from `scan`")
+    _add_policy_options(ev)
+    _add_report_options(ev)
+    _add_common(ev)
+
+    report = sub.add_parser(
+        "report", help="render a saved evaluation (JSON from `evaluate`) in another format"
+    )
+    report.add_argument("evaluation", type=Path, help="evaluation JSON from `evaluate`")
+    _add_report_options(report)
+    _add_common(report)
+
+    diff = sub.add_parser(
+        "diff", help="compare two evaluations: new and resolved findings, gained capabilities"
+    )
+    diff.add_argument("old", type=Path, help="earlier evaluation JSON")
+    diff.add_argument("new", type=Path, help="later evaluation JSON")
+    diff.add_argument(
+        "--format", choices=["json", "markdown"], default="json", help="output format"
+    )
+    diff.add_argument(
+        "--fail-on",
+        choices=[*(s.value for s in Severity), "none"],
+        default="none",
+        help="exit 1 if a NEW finding is at or above this severity (default none)",
+    )
+    diff.add_argument(
+        "--fail-on-new-capability",
+        action="store_true",
+        help="exit 1 if any agent gained a capability it did not have before",
+    )
+    _add_common(diff)
+
+    baseline = sub.add_parser("baseline", help="manage tool-definition baselines")
+    baseline_sub = baseline.add_subparsers(dest="baseline_command", required=True)
+    create = baseline_sub.add_parser(
+        "create", help="pin the tool definition hashes of an enumerated inventory"
+    )
+    create.add_argument("inventory", type=Path, help="inventory JSON from `scan --enumerate`")
+    _add_common(create)
+    return parser
+
+
+def _read_json_file(path: Path) -> str:
+    try:
+        if path.stat().st_size > _MAX_INVENTORY_BYTES:
+            raise CliError(f"{path} is larger than {_MAX_INVENTORY_BYTES} bytes")
+        return path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise CliError(f"cannot read {path}: {exc.strerror}") from None
+    except UnicodeDecodeError:
+        raise CliError(f"{path} is not UTF-8 text") from None
+
+
+def _load_inventory(path: Path) -> Inventory:
+    try:
+        return Inventory.model_validate_json(_read_json_file(path))
+    except ValidationError as exc:
+        raise CliError(f"{path} is not a valid inventory: {exc.error_count()} error(s)") from None
+
+
+def _load_evaluation(path: Path) -> Evaluation:
+    try:
+        return Evaluation.model_validate_json(_read_json_file(path))
+    except ValidationError as exc:
+        raise CliError(f"{path} is not a valid evaluation: {exc.error_count()} error(s)") from None
+
+
+def _classify(inventory: Inventory, args: argparse.Namespace) -> None:
+    cache = ClassificationCache(args.cache_dir)
+    if args.offline:
+        classifier = Classifier(cache, offline=True)
+    else:
+        try:
+            provider = AnthropicProvider(model=args.model)
+        except Exception as exc:  # SDK raises various errors when credentials are missing
+            # Fail closed: tools without a reviewed reference label or a cached answer
+            # stay unclassified and are reported as not assessed, never as safe.
+            print(
+                f"warning: no usable LLM client ({type(exc).__name__}); set ANTHROPIC_API_KEY "
+                "to classify new tools. Using reviewed reference labels and cached "
+                "classifications only; other tools will be reported as not assessed.",
+                file=sys.stderr,
+            )
+            classifier = Classifier(cache, offline=True)
+        else:
+            classifier = Classifier(cache, provider)
+    classify_inventory(inventory, classifier, concurrency=args.classify_concurrency)
+
+
+def _evaluate(inventory: Inventory, args: argparse.Namespace) -> Evaluation:
+    try:
+        policy = load_policy(args.policy)
+        baseline: Baseline | None = load_baseline(args.baseline) if args.baseline else None
+    except (PolicyError, BaselineError) as exc:
+        raise CliError(str(exc)) from None
+    return evaluate(inventory, policy, baseline)
+
+
+def _write(payload: str, output: Path | None) -> None:
+    if output:
+        output.write_text(payload, encoding="utf-8")
+        log.info("wrote %s", output)
+    else:
+        sys.stdout.write(payload)
+
+
+def _emit(model: BaseModel, output: Path | None) -> None:
+    _write(model.model_dump_json(indent=2) + "\n", output)
+
+
+def _report(evaluation: Evaluation, args: argparse.Namespace) -> int:
+    """Write the evaluation in the requested format; return the exit status."""
+    _write(render(evaluation, Format(args.format)), args.output)
+    fail_on = None if args.fail_on == "none" else Severity(args.fail_on)
+    if should_fail(evaluation, fail_on, args.fail_on_not_assessed):
+        at_or_above = sum(
+            1 for f in evaluation.findings if fail_on and f.severity.rank >= fail_on.rank
+        )
+        log.warning(
+            "failing: %d finding(s) at or above --fail-on %s, %d not assessed",
+            at_or_above,
+            args.fail_on,
+            len(evaluation.not_assessed),
+        )
+        return EXIT_FINDINGS
+    return 0
+
+
+def _print_scan_summary(inventory: Inventory, args: argparse.Namespace) -> None:
+    """One human-readable line on stderr, so an empty result is not a mystery."""
+    servers = sum(len(agent.servers) for agent in inventory.agents)
+    print(
+        f"scanned {len(inventory.agents)} agent config(s): {servers} MCP server(s) found, "
+        f"{len(inventory.errors)} error(s)",
+        file=sys.stderr,
+    )
+    if servers == 0:
+        places = [str(p) for p in user_config_paths()] if args.user else []
+        places += [str(a.config_path) for a in inventory.agents]
+        checked = "; ".join(dict.fromkeys(places)) or "no config files found"
+        print(redact_text(f"no MCP servers found. Looked at: {checked}"), file=sys.stderr)
+    elif not args.enumerate:
+        print(
+            "tools were not listed. Add --enumerate --classify --evaluate to assess what "
+            "each server can do (this starts the configured servers).",
+            file=sys.stderr,
+        )
+
+
+def _cmd_scan(args: argparse.Namespace) -> int:
+    if args.classify and not args.enumerate:
+        raise CliError(
+            "--classify needs tool definitions: add --enumerate, "
+            "or run `agentcapmap classify` on a saved inventory"
+        )
+    if not args.evaluate and (
+        args.save_inventory
+        or args.format != Format.JSON.value
+        or args.fail_on != "none"
+        or args.fail_on_not_assessed
+    ):
+        raise CliError(
+            "--save-inventory, --format, --fail-on and --fail-on-not-assessed need --evaluate"
+        )
+    paths: list[Path] = args.paths or ([] if args.user else [Path.cwd()])
+    inventory = discover(paths, user=args.user, root=Path.cwd())
+    if args.enumerate:
+        print(ENUMERATE_WARNING, file=sys.stderr)
+        enumerate_inventory(inventory, timeout=args.timeout, concurrency=args.concurrency)
+    if args.classify:
+        _classify(inventory, args)
+    if not args.evaluate:
+        _print_scan_summary(inventory, args)
+        _emit(inventory, args.output)
+        return 0
+    if args.save_inventory:
+        _emit(inventory, args.save_inventory)
+    return _report(_evaluate(inventory, args), args)
+
+
+def _cmd_classify(args: argparse.Namespace) -> int:
+    inventory = _load_inventory(args.inventory)
+    _classify(inventory, args)
+    _emit(inventory, args.output)
+    return 0
+
+
+def _cmd_evaluate(args: argparse.Namespace) -> int:
+    return _report(_evaluate(_load_inventory(args.inventory), args), args)
+
+
+def _cmd_report(args: argparse.Namespace) -> int:
+    return _report(_load_evaluation(args.evaluation), args)
+
+
+def _cmd_diff(args: argparse.Namespace) -> int:
+    diff = diff_evaluations(_load_evaluation(args.old), _load_evaluation(args.new))
+    if args.format == "markdown":
+        _write(render_markdown(diff), args.output)
+    else:
+        _emit(diff, args.output)
+    fail_on = None if args.fail_on == "none" else Severity(args.fail_on)
+    if should_fail_diff(diff, fail_on, args.fail_on_new_capability):
+        log.warning(
+            "failing: %d new finding(s), %d gained capabilit(ies)",
+            len(diff.new_findings),
+            len(diff.gained_capabilities),
+        )
+        return EXIT_FINDINGS
+    return 0
+
+
+def _cmd_baseline(args: argparse.Namespace) -> int:
+    baseline = create_baseline(_load_inventory(args.inventory))
+    if not baseline.servers:
+        raise CliError("no successfully enumerated servers to pin (run scan with --enumerate)")
+    _emit(baseline, args.output)
+    return 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+    _setup_logging(getattr(args, "verbose", 0))
+    commands = {
+        "scan": _cmd_scan,
+        "classify": _cmd_classify,
+        "evaluate": _cmd_evaluate,
+        "report": _cmd_report,
+        "diff": _cmd_diff,
+        "baseline": _cmd_baseline,
+    }
+    try:
+        return commands[args.command](args)
+    except CliError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return EXIT_ERROR

agentcapmap/diff.py

@@ -0,0 +1,103 @@
+"""Compare two evaluations: what findings and capabilities appeared or went away.
+
+Findings are matched by their stable fingerprint; capabilities by agent id and the
+`field=value` capability key. Both inputs are evaluation JSON produced by `evaluate`.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from agentcapmap.findings import Evaluation, Finding, Severity
+from agentcapmap.report._text import md_code
+
+_MAX_ROWS = 100
+
+
+class CapabilityChange(BaseModel):
+    agent_id: str
+    capability: str
+    tools: list[str] = Field(default_factory=list)  # "server/tool"
+
+
+class EvaluationDiff(BaseModel):
+    new_findings: list[Finding] = Field(default_factory=list)
+    resolved_findings: list[Finding] = Field(default_factory=list)
+    gained_capabilities: list[CapabilityChange] = Field(default_factory=list)
+    lost_capabilities: list[CapabilityChange] = Field(default_factory=list)
+
+
+def _capabilities(evaluation: Evaluation) -> dict[tuple[str, str], list[str]]:
+    return {
+        (agent.agent_id, capability): sorted({f"{ref.server}/{ref.tool}" for ref in refs})
+        for agent in evaluation.agents
+        for capability, refs in agent.capabilities.items()
+    }
+
+
+def _changes(
+    source: dict[tuple[str, str], list[str]], other: dict[tuple[str, str], list[str]]
+) -> list[CapabilityChange]:
+    return [
+        CapabilityChange(agent_id=agent, capability=capability, tools=tools)
+        for (agent, capability), tools in sorted(source.items())
+        if (agent, capability) not in other
+    ]
+
+
+def diff_evaluations(old: Evaluation, new: Evaluation) -> EvaluationDiff:
+    old_prints = {f.fingerprint for f in old.findings}
+    new_prints = {f.fingerprint for f in new.findings}
+    old_caps, new_caps = _capabilities(old), _capabilities(new)
+    return EvaluationDiff(
+        new_findings=[f for f in new.findings if f.fingerprint not in old_prints],
+        resolved_findings=[f for f in old.findings if f.fingerprint not in new_prints],
+        gained_capabilities=_changes(new_caps, old_caps),
+        lost_capabilities=_changes(old_caps, new_caps),
+    )
+
+
+def should_fail_diff(
+    diff: EvaluationDiff, fail_on: Severity | None, fail_on_new_capability: bool
+) -> bool:
+    """Fail on a *new* finding at or above `fail_on`, or on any gained capability."""
+    if fail_on is not None and any(f.severity.rank >= fail_on.rank for f in diff.new_findings):
+        return True
+    return fail_on_new_capability and bool(diff.gained_capabilities)
+
+
+def _capability_lines(title: str, changes: list[CapabilityChange]) -> list[str]:
+    lines = [f"## {title} ({len(changes)})", ""]
+    if not changes:
+        return [*lines, "None.", ""]
+    for change in changes[:_MAX_ROWS]:
+        tools = ", ".join(md_code(t, 120) for t in change.tools[:10])
+        lines.append(
+            f"- {md_code(change.agent_id, 120)}: {md_code(change.capability, 80)} via {tools}"
+        )
+    if len(changes) > _MAX_ROWS:
+        lines.append(f"- ... and {len(changes) - _MAX_ROWS} more")
+    return [*lines, ""]
+
+
+def _finding_lines(title: str, findings: list[Finding]) -> list[str]:
+    lines = [f"## {title} ({len(findings)})", ""]
+    if not findings:
+        return [*lines, "None.", ""]
+    for finding in findings[:_MAX_ROWS]:
+        lines.append(
+            f"- **{finding.severity.value}** {md_code(finding.rule_id, 40)}: "
+            f"{md_code(finding.message, 300)}"
+        )
+    if len(findings) > _MAX_ROWS:
+        lines.append(f"- ... and {len(findings) - _MAX_ROWS} more")
+    return [*lines, ""]
+
+
+def render_markdown(diff: EvaluationDiff) -> str:
+    lines = ["# agent-capmap diff", ""]
+    lines += _finding_lines("New findings", diff.new_findings)
+    lines += _capability_lines("Gained capabilities", diff.gained_capabilities)
+    lines += _finding_lines("Resolved findings", diff.resolved_findings)
+    lines += _capability_lines("Lost capabilities", diff.lost_capabilities)
+    return "\n".join(lines)