openapi-agent-ready 0.1.0__py3-none-any.whl

agent_ready/__init__.py

@@ -0,0 +1,16 @@
+"""agent-ready: audit OpenAPI specs for AI-agent and MCP readiness."""
+
+__version__ = "0.1.0"
+
+from .auditor import SpecLoadError, extract_endpoints, load_spec, run_audit
+from .rubric import CATEGORY_WEIGHTS, EndpointInfo, Finding
+
+__all__ = [
+    "CATEGORY_WEIGHTS",
+    "EndpointInfo",
+    "Finding",
+    "SpecLoadError",
+    "extract_endpoints",
+    "load_spec",
+    "run_audit",
+]

agent_ready/auditor.py

@@ -0,0 +1,208 @@
+"""
+Core audit engine: parse an OpenAPI spec, run the rubric, produce findings.
+"""
+
+import json
+from collections import defaultdict
+from dataclasses import asdict
+from pathlib import Path
+from urllib.request import urlopen
+
+import yaml
+
+from .rubric import (
+    CATEGORY_WEIGHTS,
+    EndpointInfo,
+    Finding,
+    check_ambiguity,
+    check_auth_clarity,
+    check_description_clarity,
+    check_error_responses,
+    check_parameter_explanation,
+    check_schema_strictness,
+    check_side_effects,
+    check_tool_surface,
+    check_usage_guidelines,
+)
+
+SEVERITY_VALUE = {"pass": 1.0, "warning": 0.5, "fail": 0.0}
+
+
+class SpecLoadError(Exception):
+    """Raised when a spec can't be loaded or parsed."""
+
+
+def load_spec(source: str, timeout: int = 30) -> dict:
+    """Load an OpenAPI spec from a local path or an http(s) URL."""
+    if source.startswith(("http://", "https://")):
+        try:
+            with urlopen(source, timeout=timeout) as resp:
+                raw = resp.read().decode("utf-8")
+        except Exception as exc:
+            raise SpecLoadError(f"Could not fetch spec from {source}: {exc}") from exc
+    else:
+        path = Path(source)
+        if not path.exists():
+            raise SpecLoadError(f"Spec file not found: {source}")
+        raw = path.read_text(encoding="utf-8")
+
+    # YAML is a superset of JSON, so safe_load handles both. Try JSON first for
+    # a clearer error on malformed JSON.
+    try:
+        if source.endswith(".json") or raw.lstrip().startswith("{"):
+            return json.loads(raw)
+        return yaml.safe_load(raw)
+    except Exception as exc:
+        raise SpecLoadError(f"Could not parse spec as JSON or YAML: {exc}") from exc
+
+
+def _resolve_ref(spec: dict, ref: str):
+    """Resolve a local $ref like '#/components/parameters/PageSize'."""
+    if not ref.startswith("#/"):
+        return None
+    node = spec
+    for part in ref[2:].split("/"):
+        part = part.replace("~1", "/").replace("~0", "~")
+        if not isinstance(node, dict) or part not in node:
+            return None
+        node = node[part]
+    return node
+
+
+def extract_endpoints(spec: dict) -> list[EndpointInfo]:
+    """Pull every operation out of the spec, resolving parameter $refs."""
+    endpoints = []
+    paths = spec.get("paths") or {}
+    if not isinstance(paths, dict):
+        return endpoints
+
+    http_methods = {"get", "post", "put", "patch", "delete", "head", "options"}
+
+    for path, methods in paths.items():
+        if not isinstance(methods, dict):
+            continue
+        # Parameters can be declared once at the path level and shared.
+        shared_params = methods.get("parameters", []) or []
+
+        for method, op in methods.items():
+            if method.lower() not in http_methods or not isinstance(op, dict):
+                continue
+
+            raw_params = list(shared_params) + list(op.get("parameters", []) or [])
+            resolved = []
+            for p in raw_params:
+                if isinstance(p, dict) and "$ref" in p:
+                    target = _resolve_ref(spec, p["$ref"])
+                    if isinstance(target, dict):
+                        resolved.append(target)
+                elif isinstance(p, dict):
+                    resolved.append(p)
+
+            endpoints.append(
+                EndpointInfo(
+                    path=path,
+                    method=method,
+                    summary=op.get("summary") or "",
+                    description=op.get("description") or "",
+                    parameters=resolved,
+                    responses=op.get("responses") or {},
+                    request_body=op.get("requestBody") or {},
+                )
+            )
+    return endpoints
+
+
+def run_audit(spec: dict) -> dict:
+    """Run every rubric check and aggregate into a scored result."""
+    endpoints = extract_endpoints(spec)
+    security_schemes = (spec.get("components") or {}).get("securitySchemes") or {}
+
+    findings: list[Finding] = []
+    for ep in endpoints:
+        findings += check_description_clarity(ep)
+        findings += check_side_effects(ep)
+        findings += check_error_responses(ep)
+        findings += check_auth_clarity(ep, security_schemes)
+        findings += check_parameter_explanation(ep)
+        findings += check_usage_guidelines(ep)
+        findings += check_schema_strictness(ep)
+    findings += check_ambiguity(endpoints)
+    findings += check_tool_surface(endpoints)
+
+    # Score per category. Spec-level findings (e.g. "this catalogue has 587
+    # tools") describe the whole API, so averaging them in alongside hundreds of
+    # endpoint-level findings would drown them out entirely. Where both exist,
+    # the category score is a 50/50 blend of the two scopes.
+    endpoint_vals = defaultdict(list)
+    spec_vals = defaultdict(list)
+    for f in findings:
+        bucket = spec_vals if f.scope == "spec" else endpoint_vals
+        bucket[f.category].append(SEVERITY_VALUE[f.severity])
+
+    category_summary = {}
+    for cat, weight in CATEGORY_WEIGHTS.items():
+        ep_scores = endpoint_vals.get(cat, [])
+        sp_scores = spec_vals.get(cat, [])
+        ep_avg = sum(ep_scores) / len(ep_scores) if ep_scores else None
+        sp_avg = sum(sp_scores) / len(sp_scores) if sp_scores else None
+
+        if ep_avg is not None and sp_avg is not None:
+            avg = 0.5 * ep_avg + 0.5 * sp_avg
+        elif ep_avg is not None:
+            avg = ep_avg
+        elif sp_avg is not None:
+            avg = sp_avg
+        else:
+            avg = 0.0
+
+        category_summary[cat] = {"score": round(avg * 100, 1), "weight": weight}
+
+    overall = round(
+        sum(category_summary[c]["score"] * w for c, w in CATEGORY_WEIGHTS.items()), 1
+    )
+
+    return {
+        "endpoints": endpoints,
+        "endpoint_count": len(endpoints),
+        "overall_score": overall,
+        "category_summary": category_summary,
+        "gaps": [f for f in findings if f.severity in ("fail", "warning")],
+        "passes": [f for f in findings if f.severity == "pass"],
+        "fail_count": sum(1 for f in findings if f.severity == "fail"),
+        "warning_count": sum(1 for f in findings if f.severity == "warning"),
+    }
+
+
+def result_to_dict(spec: dict, result: dict) -> dict:
+    """Serialise an audit result to plain JSON-safe data (for --format json)."""
+    info = spec.get("info") or {}
+    return {
+        "api": {
+            "title": info.get("title", "Untitled API"),
+            "version": str(info.get("version", "unknown")),
+        },
+        "overall_score": result["overall_score"],
+        "endpoint_count": result["endpoint_count"],
+        "fail_count": result["fail_count"],
+        "warning_count": result["warning_count"],
+        "categories": result["category_summary"],
+        "findings": [asdict(f) for f in result["gaps"]],
+    }
+
+
+def endpoints_ready_for_mcp(
+    result: dict,
+    blocking_categories=("description_clarity", "parameter_explanation"),
+) -> list[EndpointInfo]:
+    """
+    Endpoints with no FAIL findings in the blocking categories — i.e. safe to
+    wrap as MCP tools. Description clarity and parameter explanation are the
+    blockers because they most directly cause wrong tool selection and
+    malformed arguments.
+    """
+    blocked = {
+        f.endpoint
+        for f in result["gaps"]
+        if f.severity == "fail" and f.category in blocking_categories
+    }
+    return [ep for ep in result["endpoints"] if ep.id not in blocked]

agent_ready/cli.py

@@ -0,0 +1,175 @@
+"""
+Command-line interface for agent-ready.
+
+Exit codes (designed for CI use):
+    0  audit passed all configured gates
+    1  audit ran, but a gate failed (score below --min-score, or
+       --max-fails exceeded)
+    2  the spec could not be loaded or parsed
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from .auditor import (
+    SpecLoadError,
+    endpoints_ready_for_mcp,
+    load_spec,
+    result_to_dict,
+    run_audit,
+)
+from .mcp_generator import generate_mcp_scaffold
+from .report_generator import generate_report
+
+EXIT_OK = 0
+EXIT_GATE_FAILED = 1
+EXIT_SPEC_ERROR = 2
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="agent-ready",
+        description=(
+            "Audit an OpenAPI spec for AI-agent readiness and optionally "
+            "generate an MCP server scaffold."
+        ),
+        epilog=(
+            "CI example:\n"
+            "  agent-ready openapi.yaml --min-score 70 --max-fails 0 --quiet\n"
+        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("spec", help="Path or http(s) URL to an OpenAPI spec (.json/.yaml)")
+    parser.add_argument(
+        "--format",
+        choices=["markdown", "json", "text"],
+        default="text",
+        help="Output format (default: text summary to stdout)",
+    )
+    parser.add_argument("--out", help="Write the report to this file instead of stdout")
+    parser.add_argument(
+        "--min-score",
+        type=float,
+        help="Exit 1 if the overall score is below this value (CI gate)",
+    )
+    parser.add_argument(
+        "--max-fails",
+        type=int,
+        help="Exit 1 if the number of FAIL findings exceeds this value (CI gate)",
+    )
+    parser.add_argument(
+        "--generate-mcp",
+        metavar="PATH",
+        nargs="?",
+        const="mcp_server_scaffold.py",
+        help="Generate an MCP server scaffold for the endpoints that pass",
+    )
+    parser.add_argument("--quiet", action="store_true", help="Suppress non-essential output")
+    parser.add_argument("--version", action="version", version="%(prog)s 0.1.0")
+    return parser
+
+
+def _text_summary(spec: dict, result: dict) -> str:
+    info = spec.get("info") or {}
+    lines = [
+        f"{info.get('title', 'Untitled API')} (v{info.get('version', 'unknown')})",
+        f"AI-readiness score: {result['overall_score']}/100",
+        (
+            f"Endpoints: {result['endpoint_count']}  "
+            f"Fails: {result['fail_count']}  Warnings: {result['warning_count']}"
+        ),
+        "",
+        "Category scores:",
+    ]
+    for cat, summary in result["category_summary"].items():
+        label = cat.replace("_", " ").title()
+        lines.append(f"  {label:<28} {summary['score']:>5}/100")
+    return "\n".join(lines)
+
+
+def main(argv=None) -> int:
+    try:
+        return _run(argv)
+    except BrokenPipeError:
+        # Downstream command closed the pipe (e.g. `agent-ready spec.yaml | head`).
+        # Suppress Python's noisy teardown warning and exit quietly.
+        try:
+            sys.stdout.close()
+        except BrokenPipeError:
+            pass
+        return EXIT_OK
+    except KeyboardInterrupt:
+        print("\ninterrupted", file=sys.stderr)
+        return 130
+
+
+def _run(argv=None) -> int:
+    args = build_parser().parse_args(argv)
+
+    try:
+        spec = load_spec(args.spec)
+    except SpecLoadError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return EXIT_SPEC_ERROR
+
+    if not isinstance(spec, dict) or not spec.get("paths"):
+        print(
+            "error: no 'paths' found — this does not look like an OpenAPI spec.",
+            file=sys.stderr,
+        )
+        return EXIT_SPEC_ERROR
+
+    result = run_audit(spec)
+
+    if result["endpoint_count"] == 0:
+        print("error: spec contains no operations to audit.", file=sys.stderr)
+        return EXIT_SPEC_ERROR
+
+    if args.format == "json":
+        output = json.dumps(result_to_dict(spec, result), indent=2)
+    elif args.format == "markdown":
+        output = generate_report(spec, result)
+    else:
+        output = _text_summary(spec, result)
+
+    if args.out:
+        Path(args.out).write_text(output, encoding="utf-8")
+        if not args.quiet:
+            print(f"Report written to {args.out}")
+            print(f"AI-readiness score: {result['overall_score']}/100")
+    else:
+        print(output)
+
+    if args.generate_mcp:
+        ready = endpoints_ready_for_mcp(result)
+        generate_mcp_scaffold(spec, ready, args.generate_mcp)
+        if not args.quiet:
+            print(
+                f"MCP scaffold written to {args.generate_mcp} "
+                f"({len(ready)}/{result['endpoint_count']} endpoints included)"
+            )
+
+    # CI gates
+    gate_failed = False
+    if args.min_score is not None and result["overall_score"] < args.min_score:
+        print(
+            f"GATE FAILED: score {result['overall_score']} is below "
+            f"minimum {args.min_score}",
+            file=sys.stderr,
+        )
+        gate_failed = True
+    if args.max_fails is not None and result["fail_count"] > args.max_fails:
+        print(
+            f"GATE FAILED: {result['fail_count']} FAIL finding(s) exceeds "
+            f"maximum {args.max_fails}",
+            file=sys.stderr,
+        )
+        gate_failed = True
+
+    return EXIT_GATE_FAILED if gate_failed else EXIT_OK
+
+
+if __name__ == "__main__":
+    sys.exit(main())

agent_ready/mcp_generator.py

@@ -0,0 +1,119 @@
+"""
+Generates a runnable FastMCP server scaffold from the endpoints that
+passed the AI-readiness audit. Each OpenAPI operation becomes an MCP
+tool with a model-facing description (distinct from the raw API docs).
+"""
+
+import re
+
+
+def _to_snake_case(path: str, method: str) -> str:
+    name = re.sub(r"[{}]", "", path)
+    name = re.sub(r"[^a-zA-Z0-9]+", "_", name).strip("_").lower()
+    return f"{method.lower()}_{name}"
+
+
+def _to_python_identifier(name: str) -> str:
+    """Sanitize an OpenAPI parameter name (e.g. 'Idempotency-Key') into a valid Python identifier."""
+    ident = re.sub(r"[^a-zA-Z0-9_]", "_", name).lower()
+    if ident and ident[0].isdigit():
+        ident = f"_{ident}"
+    return ident or "param"
+
+
+def _model_facing_description(ep) -> str:
+    """
+    Rewrites the raw OpenAPI description into something aimed at a model's
+    tool-selection decision: what it's for, and when to use it vs. not.
+    """
+    base = ep.description.strip() or ep.summary.strip() or f"{ep.method.upper()} {ep.path}"
+    write_methods = {"post", "put", "patch", "delete"}
+    if ep.method.lower() in write_methods:
+        note = " NOTE: this endpoint has side effects — it changes data, it is not read-only."
+    else:
+        note = " This is a read-only lookup — safe to call speculatively."
+    return base + note
+
+
+def generate_mcp_scaffold(spec: dict, endpoints: list, out_path: str):
+    base_url = ""
+    servers = spec.get("servers", [])
+    if servers:
+        base_url = servers[0].get("url", "")
+
+    lines = []
+    lines.append('"""')
+    lines.append("Auto-generated MCP server scaffold.")
+    lines.append("Generated by api-readiness-auditor from an OpenAPI spec.")
+    lines.append("Review every tool description before shipping — these are a first draft,")
+    lines.append("written to be model-facing rather than copied straight from API docs.")
+    lines.append('"""')
+    lines.append("")
+    lines.append("import httpx")
+    lines.append("from mcp.server.fastmcp import FastMCP")
+    lines.append("")
+    lines.append(f'BASE_URL = "{base_url}"')
+    lines.append("")
+    lines.append('mcp = FastMCP("generated-api-server")')
+    lines.append("")
+
+    for ep in endpoints:
+        func_name = _to_snake_case(ep.path, ep.method)
+        description = _model_facing_description(ep)
+        params = [p for p in ep.parameters if isinstance(p, dict) and p.get("name")]
+
+        path_params, query_params, header_params = [], [], []
+        for p in params:
+            location = p.get("in", "query")
+            entry = (p["name"], _to_python_identifier(p["name"]))
+            if location == "path":
+                path_params.append(entry)
+            elif location == "header":
+                header_params.append(entry)
+            else:
+                query_params.append(entry)
+
+        all_params = path_params + query_params + header_params
+        sig_params = ", ".join(f"{ident}: str = None" for _, ident in all_params)
+
+        url_path = ep.path
+        for original, ident in path_params:
+            url_path = url_path.replace("{" + original + "}", "{" + ident + "}")
+
+        lines.append("@mcp.tool()")
+        lines.append(f"def {func_name}({sig_params}):")
+        lines.append(f'    """{description}"""')
+        if path_params:
+            lines.append(f'    url = BASE_URL + f"{url_path}"')
+        else:
+            lines.append(f'    url = BASE_URL + "{url_path}"')
+
+        if query_params:
+            q_entries = ", ".join(f'"{orig}": {ident}' for orig, ident in query_params)
+            lines.append(f"    params = {{{q_entries}}}")
+            lines.append("    params = {k: v for k, v in params.items() if v is not None}")
+        else:
+            lines.append("    params = {}")
+
+        if header_params:
+            h_entries = ", ".join(f'"{orig}": {ident}' for orig, ident in header_params)
+            lines.append(f"    headers = {{{h_entries}}}")
+            lines.append("    headers = {k: v for k, v in headers.items() if v is not None}")
+        else:
+            lines.append("    headers = {}")
+
+        method = ep.method.lower()
+        if method == "get":
+            lines.append("    resp = httpx.get(url, params=params, headers=headers)")
+        else:
+            lines.append(f"    resp = httpx.{method}(url, params=params, headers=headers, json={{}})")
+        lines.append("    resp.raise_for_status()")
+        lines.append("    return resp.json()")
+        lines.append("")
+
+    lines.append('if __name__ == "__main__":')
+    lines.append("    mcp.run()")
+    lines.append("")
+
+    with open(out_path, "w") as f:
+        f.write("\n".join(lines))

agent_ready/report_generator.py

@@ -0,0 +1,93 @@
+"""Generates a markdown AI-readiness report from an audit result."""
+
+CATEGORY_LABELS = {
+    "description_clarity": "1. Capability Description Clarity",
+    "side_effects": "2. Side-Effect Signaling",
+    "error_responses": "3. Actionable Error Responses",
+    "auth_clarity": "4. Auth/Scope Clarity for Agents",
+    "ambiguity": "5. Endpoint Ambiguity",
+    "parameter_explanation": "6. Parameter Explanation",
+    "usage_guidelines": "7. Usage Guidelines (When to Call)",
+    "tool_surface": "8. Tool Surface & Schema Strictness",
+}
+
+SEVERITY_ICON = {"fail": "🔴", "warning": "🟡", "pass": "🟢"}
+
+
+def generate_report(spec: dict, result: dict) -> str:
+    title = spec.get("info", {}).get("title", "Untitled API")
+    version = spec.get("info", {}).get("version", "unknown")
+
+    lines = []
+    lines.append(f"# AI-Readiness Report: {title} (v{version})")
+    lines.append("")
+    lines.append(f"**Overall score: {result['overall_score']}/100**  ")
+    lines.append(f"**Endpoints analyzed:** {result['endpoint_count']}")
+    lines.append("")
+    lines.append("## Category Scores")
+    lines.append("")
+    lines.append("| Category | Score | Weight |")
+    lines.append("|---|---|---|")
+    for cat, label in CATEGORY_LABELS.items():
+        summary = result["category_summary"].get(cat, {"score": 0, "weight": 0})
+        lines.append(f"| {label} | {summary['score']}/100 | {int(summary['weight']*100)}% |")
+    lines.append("")
+
+    lines.append("## Gaps Found")
+    lines.append("")
+    if not result["gaps"]:
+        lines.append("No gaps found — every endpoint passed every category.")
+    else:
+        by_category = {}
+        for f in result["gaps"]:
+            by_category.setdefault(f.category, []).append(f)
+        for cat, label in CATEGORY_LABELS.items():
+            findings = by_category.get(cat, [])
+            if not findings:
+                continue
+            lines.append(f"### {label}")
+            lines.append("")
+            for f in findings:
+                icon = SEVERITY_ICON[f.severity]
+                lines.append(f"- {icon} **{f.endpoint}** — {f.message}")
+            lines.append("")
+
+    lines.append("## What Passed")
+    lines.append("")
+    pass_counts = {}
+    for f in result["passes"]:
+        pass_counts[f.category] = pass_counts.get(f.category, 0) + 1
+    for cat, label in CATEGORY_LABELS.items():
+        count = pass_counts.get(cat, 0)
+        lines.append(f"- {label}: {count} endpoint(s) passed cleanly")
+    lines.append("")
+
+    lines.append("## Recommended Next Step")
+    lines.append("")
+    lines.append(
+        "Don't try to fix every gap at once. Prioritize the endpoints with 🔴 fails in "
+        "**Capability Description Clarity** and **Parameter Explanation** first — these "
+        "most directly cause wrong tool selection and malformed arguments, which matters "
+        "more early on than auth polish or ambiguity cleanup."
+    )
+    lines.append("")
+    lines.append("### A caveat on maximising this score")
+    lines.append("")
+    lines.append(
+        "This score is a diagnostic, not a target. Hasan et al. (arXiv:2602.14878) found "
+        "that augmenting *all* description components raised task success by a median 5.85 "
+        "percentage points — but increased execution steps by 67.46% and regressed "
+        "performance in 16.67% of cases, because richer descriptions consume context window "
+        "and raise cost. Their ablations found shorter, targeted descriptions often performed "
+        "equivalently. Fix the fails; don't gold-plate the passes."
+    )
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("*Rubric categories 1–5 derived independently from MCP tool-selection mechanics "
+                 "and API design principles; categories 6–7 added after validation against "
+                 "arXiv:2602.14878 (856 tools / 103 servers) and arXiv:2602.18914 (10,831 servers). "
+                 "Category weights are judgement calls, not measured effect sizes.*")
+    lines.append("")
+
+    return "\n".join(lines)