codi-api-agent 0.3.1__py3-none-any.whl

api_agent/openapi_loader.py

@@ -0,0 +1,560 @@
+"""Load an OpenAPI 3.x spec and turn it into catalog Tools.
+
+This is the mechanism by which a *real* API spec becomes agent-callable tools —
+the same path your own APIs will use. It produces the same `Tool` objects the
+hand-built catalog uses, so the rest of the pipeline is unchanged.
+
+Design choices that mirror the architecture:
+  * READ-ONLY by default — only GET/HEAD operations are exposed. This enforces
+    the agreed "read-only / Tier-2 GET-only" guardrail at the loader boundary.
+  * One operation -> one tool (operationId -> name, summary/description -> the
+    NL the model routes on, parameters -> the tool's args, server -> base URL).
+
+NOT yet implemented (deferred to Phase-3 hardening): SSRF egress controls on the
+outbound HTTP, and per-user credential injection. The loader currently fetches
+the spec URL and calls endpoints as given — fine for dev against trusted specs
+(e.g. Petstore), but it must be gated before pointing it at untrusted specs.
+"""
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import replace
+from urllib.parse import urlparse
+
+import requests
+
+from .catalog import Catalog, Tool
+
+_UA = {"User-Agent": "api-agent-demo/0.1"}
+READ_METHODS = {"get", "head"}  # only these become CALLABLE tools (read-only guardrail)
+_ALL_METHODS = {"get", "head", "post", "put", "patch", "delete"}  # documented, not all called
+_SCALARS = {"string", "integer", "number", "boolean"}
+# Infrastructure / auth headers are handled by the agent (UA set here, auth injected from
+# the Auth field) — never expose them as model-fillable parameters, or the model invents a
+# value (e.g. a fake Cookie) that clobbers the real one.
+_NOISE_HEADERS = {
+    "cookie", "authorization", "user-agent", "content-type", "accept",
+    "accept-encoding", "content-length", "host", "connection",
+}
+
+# Privacy guardrail: personal / contact details must never reach the user. We redact such
+# values in API responses BEFORE they reach the model (so it physically cannot leak them).
+# Redaction is by field NAME (precise — won't blank numeric ids) plus email-pattern masking
+# for free-text fields. Business data (names, ids, statuses, counts) flows through untouched.
+# Sensitive identifiers whose VALUE is itself the PII — always redact, even when the name
+# ends in `_id` (e.g. tax_id, national_id).
+_PII_ALWAYS_RE = re.compile(
+    r"\bssn\b|social[-_ ]?security|tax[-_ ]?id|\bein\b|national[-_ ]?id|passport|"
+    r"date[-_ ]?of[-_ ]?birth|\bdob\b|birth[-_ ]?date|birthday",
+    re.IGNORECASE,
+)
+# Other contact-detail fields — redact unless the name is a `_id` reference (address_id is a
+# foreign key, not the address itself). Phone is handled separately (value-aware) below.
+_PII_KEY_RE = re.compile(
+    r"e[-_ ]?mail|address|street|\bzip\b|postal",
+    re.IGNORECASE,
+)
+# Phone-ish fields — but only redact when the VALUE is a real phone NUMBER. A country calling /
+# dialing code ("+33", "91") is general public info, NOT personal, so it must NOT be redacted.
+_PHONE_KEY_RE = re.compile(r"phone|mobile|\bcell\b|\bfax\b|telephone|whatsapp", re.IGNORECASE)
+# ...unless the field is explicitly a CODE/prefix (country/dialing/area code) — never PII.
+_CODE_KEY_RE = re.compile(
+    r"(?:phone|dial(?:ing)?|call(?:ing)?|country|area|std|idd)[-_ ]?(?:code|prefix)|"
+    r"prefix|callingcode|dialcode|countrycode",
+    re.IGNORECASE,
+)
+_EMAIL_RE = re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+")
+_REDACTED = "[redacted]"
+
+# Some APIs answer an expired/invalid session with HTTP 200 and a "please log in" body (a login
+# redirect) instead of 401. We detect that in small responses and treat it as an auth failure,
+# so it's reported clearly ("re-authenticate") instead of as a vague "no data".
+_AUTH_FAIL_BODY = re.compile(
+    r"session (?:has )?expired|please log\s?in|log\s?in again|not (?:authenticated|authorized|logged)|"
+    r"authentication (?:failed|required)|unauthenticated|invalid (?:session|token|credential)|"
+    r"expired (?:token|session)|login required|\"state\"\s*:\s*\"login\"",
+    re.IGNORECASE,
+)
+
+
+def _is_pii_key(key: str) -> bool:
+    k = key.lower()
+    if _PII_ALWAYS_RE.search(k):
+        return True
+    if k == "id" or k.endswith("_id"):  # a reference (e.g. address_id), not the PII itself
+        return False
+    return bool(_PII_KEY_RE.search(k))
+
+
+def _has_phone_number(v) -> bool:
+    """True if a value looks like a real phone NUMBER (>=7 digits) rather than a short country /
+    dialing code. So `phone: "33"` (a calling code) is kept, but `phone: "+1 202-555-0143"` is
+    redacted. Recurses into lists (e.g. a country's list of dialing codes)."""
+    if isinstance(v, (list, tuple)):
+        return any(_has_phone_number(x) for x in v)
+    return len(re.sub(r"\D", "", str(v))) >= 7
+
+
+def _is_phone_field(key: str) -> bool:
+    k = key.lower()
+    return bool(_PHONE_KEY_RE.search(k)) and not _CODE_KEY_RE.search(k)
+
+
+def _redact_text(s: str) -> str:
+    """Mask email addresses anywhere in a string (catches PII in free-text fields)."""
+    return _EMAIL_RE.sub(_REDACTED, s)
+
+
+# --------------------------------------------------------------------------- #
+# Loading / parsing
+# --------------------------------------------------------------------------- #
+def load_openapi(source: str, methods: set[str] | None = None,
+                 max_operations: int = 1000, timeout: int = 20,
+                 auth_header: str | None = None, auth_value: str | None = None) -> Catalog:
+    """Load a spec (URL or local path) and build a read-only Catalog from it.
+
+    Optionally inject an auth header into every call — covers API-key-in-header
+    and bearer tokens, e.g. ``auth_header='Authorization', auth_value='Bearer <t>'``
+    or ``auth_header='X-API-Key', auth_value='<key>'``.
+    """
+    spec, origin = load_spec(source, timeout)
+    auth = (auth_header, auth_value) if auth_header and auth_value else None
+    return build_catalog(spec, origin, methods=methods or READ_METHODS,
+                         max_operations=max_operations, auth=auth)
+
+
+def load_spec(source: str, timeout: int = 20) -> tuple[dict, str | None]:
+    if source.startswith(("http://", "https://")):
+        resp = requests.get(source, timeout=timeout, headers=_UA)
+        resp.raise_for_status()
+        return _parse(resp.text), source
+    with open(source, encoding="utf-8") as f:
+        return _parse(f.read()), None
+
+
+def _parse(text: str) -> dict:
+    if text.lstrip().startswith("{"):
+        return json.loads(text)
+    try:
+        import yaml
+    except ImportError as e:  # pragma: no cover
+        raise RuntimeError("PyYAML is required for YAML specs: pip install pyyaml") from e
+    data = yaml.safe_load(text)
+    if not isinstance(data, dict):
+        raise ValueError("Spec did not parse to an object.")
+    return data
+
+
+# --------------------------------------------------------------------------- #
+# Spec -> Catalog
+# --------------------------------------------------------------------------- #
+def build_catalog(spec: dict, origin: str | None = None,
+                  methods: set[str] = READ_METHODS, max_operations: int = 1000,
+                  auth: tuple[str, str] | None = None) -> Catalog:
+    base = _base_url(spec, origin)
+    tools: list[Tool] = []
+    reference: list[dict] = []  # documentation for ALL ops (incl. writes); never called
+    seen: set[str] = set()
+    for path, item in (spec.get("paths") or {}).items():
+        if not isinstance(item, dict):
+            continue
+        shared = item.get("parameters") or []
+        for method, op in item.items():
+            m = method.lower()
+            if m not in _ALL_METHODS or not isinstance(op, dict):
+                continue
+            params = list(shared) + list(op.get("parameters") or [])
+            if len(reference) < 2000:
+                reference.append(_reference_entry(spec, path, m, op, params))
+            if m in methods and len(tools) < max_operations:
+                tools.append(_build_tool(spec, base, path, m, op, params, seen, auth))
+    return Catalog(tools, reference=reference)
+
+
+def _base_url(spec: dict, origin: str | None) -> str:
+    servers = spec.get("servers") or []
+    url = (servers[0].get("url") if servers and isinstance(servers[0], dict) else "") or ""
+    # Swagger 2.0 has no `servers`; it uses host + basePath + schemes.
+    if not url and spec.get("host"):
+        scheme = (spec.get("schemes") or ["https"])[0]
+        return f"{scheme}://{spec['host']}{spec.get('basePath', '')}".rstrip("/")
+    if url.startswith(("http://", "https://")):
+        return url.rstrip("/")
+    if origin:  # resolve a relative server url (e.g. "/api/v3") against the spec origin
+        p = urlparse(origin)
+        root = f"{p.scheme}://{p.netloc}"
+        return (root + "/" + url.lstrip("/")).rstrip("/") if url else root
+    return url.rstrip("/")
+
+
+def _build_tool(spec: dict, base: str, path: str, method: str,
+                op: dict, params: list, seen: set[str],
+                auth: tuple[str, str] | None = None) -> Tool:
+    name = _unique(_sanitize(op.get("operationId") or f"{method}_{path}"), seen)
+    summary = (op.get("summary") or "").strip()
+    desc = (op.get("description") or "").strip()
+    description = ((summary + ("\n" + desc if desc else "")).strip() or f"{method.upper()} {path}")[:300]
+
+    properties: dict = {}
+    required: list[str] = []
+    locations: list[tuple[str, str]] = []  # (name, in)
+    for p in params:
+        p = _deref(spec, p)
+        if not isinstance(p, dict):
+            continue
+        pname, loc = p.get("name"), p.get("in")
+        if not pname or loc not in ("query", "path", "header"):
+            continue
+        if loc == "header" and pname.lower() in _NOISE_HEADERS:
+            continue  # auth/infra header — the agent sets these, not the model
+        schema = _deref(spec, p.get("schema") or {})
+        jtype = schema.get("type", "string") if isinstance(schema, dict) else "string"
+        if jtype not in _SCALARS:
+            jtype = "string"
+        # Include the parameter's example so the model knows the expected FORMAT — crucial
+        # for things like a daterange (YYYY-MM-DD|YYYY-MM-DD vs a relative token vs unix ts).
+        pdesc = (p.get("description") or "").strip()
+        ex = p.get("example", schema.get("example") if isinstance(schema, dict) else None)
+        if ex not in (None, ""):
+            pdesc = (pdesc + f" (example: {ex})").strip()
+        properties[pname] = {"type": jtype, "description": pdesc[:160]}
+        if p.get("required") or loc == "path":
+            required.append(pname)
+        locations.append((pname, loc))
+
+    parameters = {"type": "object", "properties": properties, "required": required}
+    return Tool(
+        name=name,
+        description=description,
+        parameters=parameters,
+        data_type="structured",
+        fn=_make_executor(base, path, method, dict(locations), auth),
+    )
+
+
+def _reference_entry(spec: dict, path: str, method: str, op: dict, params: list) -> dict:
+    """A documentation record for ONE operation (any method) — its signature and params,
+    so the agent can DESCRIBE it (including write ops) without ever calling it."""
+    entry = {
+        "name": _sanitize(op.get("operationId") or f"{method}_{path}"),
+        "method": method.upper(),
+        "path": path,
+        "summary": (op.get("summary") or "").strip(),
+        "description": (op.get("description") or "").strip()[:300],
+        "params": [],
+        "body": [],
+    }
+    for p in params:
+        p = _deref(spec, p)
+        if not isinstance(p, dict):
+            continue
+        pname, loc = p.get("name"), p.get("in")
+        if not pname or loc not in ("query", "path", "header"):
+            continue
+        if loc == "header" and pname.lower() in _NOISE_HEADERS:
+            continue
+        schema = _deref(spec, p.get("schema") or {})
+        jtype = schema.get("type", "string") if isinstance(schema, dict) else "string"
+        entry["params"].append({
+            "name": pname, "in": loc, "required": bool(p.get("required") or loc == "path"),
+            "type": jtype,
+            "example": p.get("example", schema.get("example") if isinstance(schema, dict) else None),
+        })
+    # Request body (for writes) — pull fields from the schema's properties, or from its example
+    # (the Postman-converted specs usually only carry an example object).
+    rb = _deref(spec, op.get("requestBody") or {})
+    content = rb.get("content") if isinstance(rb, dict) else None
+    jsonc = (content.get("application/json") if isinstance(content, dict) else None) or {}
+    schema = _deref(spec, jsonc.get("schema") or {}) if isinstance(jsonc, dict) else {}
+    req = set(schema.get("required") or []) if isinstance(schema, dict) else set()
+    props = schema.get("properties") if isinstance(schema, dict) else None
+    example = schema.get("example") if isinstance(schema, dict) else None
+    if isinstance(props, dict):
+        for fname, fdef in props.items():
+            fdef = _deref(spec, fdef) if isinstance(fdef, dict) else {}
+            entry["body"].append({
+                "name": fname, "required": fname in req,
+                "type": (fdef.get("type") if isinstance(fdef, dict) else "") or "",
+                "example": fdef.get("example") if isinstance(fdef, dict) else None,
+            })
+    elif isinstance(example, dict):
+        for fname, val in example.items():
+            entry["body"].append({"name": fname, "required": fname in req, "type": "", "example": val})
+    return entry
+
+
+def format_reference(entry: dict) -> str:
+    """Render a reference entry as a readable block (used as evidence for documentation answers)."""
+    label = f"[{entry['label']}] " if entry.get("label") else ""
+    head = f"{label}{entry['method']} {entry['path']}"
+    if entry.get("summary"):
+        head += f" — {entry['summary']}"
+    lines = [head]
+    if entry.get("description"):
+        lines.append(entry["description"])
+    if entry["params"]:
+        lines.append("Parameters:")
+        for p in entry["params"]:
+            meta = ", ".join([p["in"]] + ([p["type"]] if p["type"] else []) + (["required"] if p["required"] else []))
+            ex = f" — example: {p['example']}" if p.get("example") not in (None, "") else ""
+            lines.append(f"  - {p['name']} ({meta}){ex}")
+    if entry["body"]:
+        # Note: many converted specs don't mark which body fields are required, so we only
+        # label the ones explicitly marked required — never assert "optional" (we don't know).
+        lines.append("Request body fields:")
+        for b in entry["body"]:
+            tags = ", ".join(([b["type"]] if b["type"] else []) + (["required"] if b["required"] else []))
+            tags = f" ({tags})" if tags else ""
+            ex = f" — example: {b['example']}" if b.get("example") not in (None, "") else ""
+            lines.append(f"  - {b['name']}{tags}{ex}")
+    if not entry["params"] and not entry["body"]:
+        lines.append("(no parameters)")
+    return "\n".join(lines)
+
+
+def _make_executor(base: str, path: str, method: str, locations: dict[str, str],
+                   auth: tuple[str, str] | None = None):
+    # Tolerate the model mangling array/odd param names: it often emits `clusterIds` (or
+    # `clusterids`) for a spec param literally named `clusterIds[]`. Map a normalized form
+    # (trailing [] stripped, lowercased) back to the REAL param name so the value still gets sent.
+    _norm = {re.sub(r"\[\]$", "", k).lower(): k for k in locations}
+
+    def fn(args: dict, timeout: int, list_limit: int | None = None) -> dict:
+        n = list_limit or 50  # how many items of a list response to keep (per-query override)
+        url = base + path
+        query: dict = {}
+        headers = dict(_UA)
+        if auth:
+            headers[auth[0]] = auth[1]
+        for pname, value in (args or {}).items():
+            real = pname if pname in locations else _norm.get(re.sub(r"\[\]$", "", pname).lower())
+            loc = locations.get(real) if real else None
+            if loc == "path":
+                url = url.replace("{" + real + "}", str(value))
+            elif loc == "query":
+                query[real] = value  # real keeps the spec's exact name, e.g. clusterIds[]
+            elif loc == "header":
+                # The configured auth header wins — a model-supplied value (e.g. an invented
+                # Cookie) must never override the real credential injected from the Auth field.
+                if auth and real.lower() == auth[0].lower():
+                    continue
+                headers[real] = str(value)
+        # A required path param the model left out (classically GitHub's `path` for a repo's ROOT
+        # contents) is filled with empty string so we list the collection/root rather than send a
+        # literal `{path}` and 404. Strip only within the PATH — NEVER the scheme/host: a base like
+        # `https://{host}` (a tenant placeholder) must not collapse to `https:///` with no host.
+        _hp = re.match(r"(https?://[^/]+)(.*)", url)
+        if _hp:
+            url = (_hp.group(1) + re.sub(r"\{[^}]+\}", "", _hp.group(2))).rstrip("/")
+        else:
+            url = re.sub(r"\{[^}]+\}", "", url).rstrip("/")
+        try:
+            resp = requests.request(method.upper(), url, params=query, headers=headers, timeout=timeout)
+        except Exception as e:
+            return {"ok": False, "error": str(e), "content": "", "source": url}
+        src = resp.url
+        if resp.status_code >= 400:
+            return {"ok": False, "error": f"HTTP {resp.status_code}", "content": resp.text[:500], "source": src}
+        # 200-but-not-logged-in: a small "session expired / please log in" body means the auth
+        # (cookie/token) is invalid. Flag it as an auth failure (the agent keys on "401").
+        body = resp.text
+        if len(body) < 800 and _AUTH_FAIL_BODY.search(body):
+            return {"ok": False, "error": "HTTP 401 — session/authentication expired",
+                    "content": body[:300], "source": src}
+        content = None
+        if "json" in resp.headers.get("content-type", "").lower():
+            try:
+                payload = resp.json()
+            except Exception:
+                payload = None
+            if isinstance(payload, (dict, list)):
+                emsg = _error_envelope(payload)
+                if emsg:  # HTTP 200, but the body is an application error envelope (status:"failure"…)
+                    return {"ok": False, "error": emsg,
+                            "content": _compact_json(payload, max_items=n), "source": src}
+                content = _compact_json(payload, max_items=n)  # redacts PII field-by-field
+        if content is None:
+            content = _redact_text(resp.text)
+        return {"ok": True, "content": content[: max(8000, n * 600)], "source": src}
+
+    return fn
+
+
+# Keys that merely WRAP a response's primary collection (REST envelopes + GraphQL + the ARQ
+# `{status, data}` shape) — a list under one of these is the main payload, not an incidental
+# sub-list, so it should get the full item cap rather than the small nested-list cap.
+_LIST_ENVELOPE_KEYS = frozenset({
+    "data", "result", "results", "items", "rows", "records", "payload", "list", "edges", "nodes",
+})
+
+_ERROR_STATUS = frozenset({"failure", "fail", "failed", "error"})
+
+
+def _error_envelope(payload) -> str | None:
+    """Detect an application-level error returned with a 200 body instead of an HTTP error status.
+
+    Some backends wrap failures in an envelope (e.g. the ARQ shape
+    ``{"status_code": 502, "status": "failure", "message": "Warehouse query failed: …"}``) and return
+    it with HTTP 200 — so a failed call would otherwise be counted as a SUCCESS with an error message
+    buried in its body, and the real detail never surfaces. Return the human-readable error message
+    when the payload is clearly such an envelope, else ``None``.
+
+    Conservative on purpose (a real data record whose own ``status`` field is "failed" must NOT be
+    mistaken for an API error): trigger only on a top-level ``status_code`` ≥ 400, OR an error-ish
+    ``status`` that comes WITH an error message and WITHOUT any data payload.
+    """
+    if not isinstance(payload, dict):
+        return None
+    code = payload.get("status_code")
+    code_bad = isinstance(code, int) and not isinstance(code, bool) and code >= 400
+    status = str(payload.get("status", "")).strip().lower()
+    msg = ""
+    for k in ("message", "error", "detail", "msg", "error_message", "reason"):
+        v = payload.get(k)
+        if isinstance(v, str) and v.strip():
+            msg = v.strip()
+            break
+    if not msg and isinstance(payload.get("errors"), list) and payload["errors"]:
+        first = payload["errors"][0]
+        msg = (first if isinstance(first, str) else str(first)).strip()
+    has_data = any(k in payload for k in ("data", "rows", "results", "items", "records"))
+    status_err = status in _ERROR_STATUS and bool(msg) and not has_data
+    if not (code_bad or status_err):
+        return None
+    prefix = f"HTTP {code}: " if code_bad else ""
+    # Keep enough of the message that the ACTUAL cause survives — a DB/SQL error puts the useful part
+    # (the missing signature + the "HINT: No function matches …" tail) well past 300 chars.
+    return (prefix + msg)[:800] if (msg or prefix) else f"upstream error (status={status or code})"
+
+
+def _compact_json(data, max_items: int = 30, max_fields: int = 20,
+                  max_str: int = 300, max_depth: int = 3) -> str:
+    """Make a JSON response digestible WITHOUT losing the meaningful fields.
+
+    The bulk of a typical API record is noise — every object carries ~dozens of
+    ``*_url`` links — while the substance is often *nested* (a commit's ``message``
+    lives under ``commit.message``, an author's ``login`` under ``author.login``).
+    So we keep scalar values at any nesting level up to ``max_depth`` and drop the
+    noise (``*_url`` link fields; ``url``/``html_url`` are kept as the one canonical
+    link). That lets MANY records fit the budget instead of one bloated one, while
+    still surfacing nested substance like commit messages. Single objects are slimmed
+    the same way (detail preserved)."""
+    def is_noise(key: str) -> bool:
+        k = key.lower()
+        return k.endswith("_url") and k != "html_url"
+
+    # `eff` is the list-nesting level for the cap, and it does NOT count "envelope" wrapper keys —
+    # a response's PRIMARY collection is often wrapped (`{"data":{"rows":[...]}}`, `{"results":[...]}`,
+    # GraphQL `{"data":{...}}`, or the app's `{"status","data":{...}}`). Without this those primary
+    # rows land at object-depth ≥2 and get truncated to 5. Real incidental sub-lists (an order's
+    # `steps`) sit under a NON-envelope key, so they still cap at 5.
+    def slim_list(items: list, depth: int, eff: int):
+        cap = max_items if eff <= 1 else 5
+        out = [slim(x, depth + 1, eff + 1) for x in items[:cap]]
+        # Always keep the TOTAL when truncating — even for a nested list — so counts/aggregates
+        # (e.g. "which continent has the most countries" over each continent's nested `countries`)
+        # can read the real size instead of silently seeing only the first few.
+        if len(items) > cap:
+            return {"total": len(items), "showing": cap, "items": out}
+        return out
+
+    def slim(val, depth: int, eff: int):
+        if isinstance(val, dict):
+            out: dict = {}
+            for k, v in val.items():
+                if len(out) >= max_fields:
+                    break
+                if is_noise(k):
+                    continue
+                if _is_pii_key(k):  # personal/contact detail — never surface its value
+                    out[k] = _REDACTED
+                    continue
+                if _is_phone_field(k) and _has_phone_number(v):  # a real phone NUMBER (not a code)
+                    out[k] = _REDACTED
+                    continue
+                child_eff = eff if str(k).lower() in _LIST_ENVELOPE_KEYS else eff + 1
+                if isinstance(v, str):
+                    out[k] = _redact_text(v[:max_str])
+                elif isinstance(v, (int, float, bool)) or v is None:
+                    out[k] = v
+                elif isinstance(v, dict) and depth < max_depth:
+                    s = slim(v, depth + 1, child_eff)
+                    if s:
+                        out[k] = s
+                elif isinstance(v, list) and depth < max_depth:
+                    out[k] = slim_list(v, depth + 1, child_eff)
+            return out
+        if isinstance(val, list):
+            return slim_list(val, depth, eff)
+        return val[:max_str] if isinstance(val, str) else val
+
+    return json.dumps(slim(data, 0, 0), ensure_ascii=False)
+
+
+# --------------------------------------------------------------------------- #
+# helpers
+# --------------------------------------------------------------------------- #
+def _deref(spec: dict, node):
+    """Resolve a single local ``$ref`` (``#/components/...``); else return node."""
+    if isinstance(node, dict) and "$ref" in node:
+        ref = node["$ref"]
+        if not ref.startswith("#/"):
+            return {}
+        target: object = spec
+        for part in ref[2:].split("/"):
+            part = part.replace("~1", "/").replace("~0", "~")
+            target = target.get(part, {}) if isinstance(target, dict) else {}
+        return target
+    return node
+
+
+def _sanitize(name: str) -> str:
+    return (re.sub(r"[^a-zA-Z0-9_-]", "_", name)[:64]) or "op"
+
+
+def _unique(name: str, seen: set[str]) -> str:
+    out, i = name, 2
+    while out in seen:
+        out = f"{name}_{i}"[:64]
+        i += 1
+    seen.add(out)
+    return out
+
+
+# --------------------------------------------------------------------------- #
+# Multiple specs
+# --------------------------------------------------------------------------- #
+def spec_label(source: str) -> str:
+    """A short, readable name for a spec source (used to tag merged operations)."""
+    base = source.rstrip("/").split("/")[-1] or source
+    base = re.sub(r"\.(openapi|swagger)?\.?(ya?ml|json|graphql|gql|graphqls|sdl)$", "", base,
+                  flags=re.IGNORECASE)
+    return base or source
+
+
+def merge_catalogs(labeled: list[tuple[str, Catalog]]) -> Catalog:
+    """Combine several loaded catalogs into one so the agent can answer queries that
+    span multiple APIs. Each tool keeps its own base URL and auth (both are baked into
+    its executor), so specs with different credentials work side by side. Tool names are
+    made unique across specs, and — when more than one spec is loaded — each operation's
+    description is tagged with its source so the router/model knows which API it belongs to.
+    """
+    multi = len(labeled) > 1
+    merged: dict[str, Tool] = {}
+    reference: list[dict] = []
+    for label, cat in labeled:
+        tag = _sanitize(label)
+        for name, tool in cat.tools.items():
+            new_name = name
+            if new_name in merged:  # collision across specs — disambiguate by source
+                new_name = _unique(f"{name}__{tag}", set(merged))
+            desc = f"[{label}] {tool.description}" if multi else tool.description
+            merged[new_name] = replace(tool, name=new_name, description=desc)
+        for entry in getattr(cat, "reference", []):
+            reference.append({**entry, "label": label if multi else entry.get("label", "")})
+    out = Catalog(list(merged.values()), reference=reference)
+    # Carry the location resolver across the merge. Without this, loading the warehouse ALONGSIDE
+    # an HTTP spec silently disables location binding — the UI's normal multi-source setup.
+    out.locations = next((c.locations for _l, c in labeled if getattr(c, "locations", None)), None)
+    return out

api_agent/prompts/__init__.py

@@ -0,0 +1,59 @@
+"""Every prompt in the system, grouped by the stage that uses it.
+
+Extracted from `agent.py`, where 59,000 characters of instructions sat inline among the
+control flow. They are re-exported here so `from api_agent.agent import SYNTHESIS_SYSTEM`
+keeps working for existing callers and tests.
+"""
+from __future__ import annotations
+
+from .synthesis import (  # noqa: F401
+    _TABLES_NOTE,
+    SYNTHESIS_SYSTEM,
+    _REPORT_MODE_NOTE,
+    _ANALYSIS_MODE_NOTE,
+    _JUDGEMENT_ALLOWED_NOTE,
+    _VOICE_NOTE,
+    _BRIEFING_MODE_NOTE,
+    _NO_TABLES_NOTE,
+)
+from .advisory import (  # noqa: F401
+    ADVISORY_SYSTEM,
+)
+from .executor import (  # noqa: F401
+    EXECUTOR_SYSTEM,
+    PLAN_SYSTEM,
+    SECTION_PLAN_SYSTEM,
+)
+from .judges import (  # noqa: F401
+    JUDGE_SYSTEM,
+    RESPONSIVENESS_SYSTEM,
+    COMMUNICATION_SYSTEM,
+    REVIEW_SYSTEM,
+    DOC_JUDGE_SYSTEM,
+)
+from .support import (  # noqa: F401
+    DOC_SYSTEM,
+    CACHE_MATCH_SYSTEM,
+)
+
+__all__ = [
+    "_TABLES_NOTE",
+    "SYNTHESIS_SYSTEM",
+    "_REPORT_MODE_NOTE",
+    "_ANALYSIS_MODE_NOTE",
+    "_JUDGEMENT_ALLOWED_NOTE",
+    "_VOICE_NOTE",
+    "_BRIEFING_MODE_NOTE",
+    "_NO_TABLES_NOTE",
+    "ADVISORY_SYSTEM",
+    "EXECUTOR_SYSTEM",
+    "PLAN_SYSTEM",
+    "SECTION_PLAN_SYSTEM",
+    "JUDGE_SYSTEM",
+    "RESPONSIVENESS_SYSTEM",
+    "COMMUNICATION_SYSTEM",
+    "REVIEW_SYSTEM",
+    "DOC_JUDGE_SYSTEM",
+    "DOC_SYSTEM",
+    "CACHE_MATCH_SYSTEM",
+]

api_agent/prompts/advisory.py

@@ -0,0 +1,52 @@
+"""The ADVISORY prompt — what to DO about findings already established."""
+
+ADVISORY_SYSTEM = """You are advising a business leader on WHAT TO DO about findings that were
+already established earlier in this conversation. The PRIOR FINDINGS below are the analysis you are
+acting on — they are the only facts you have.
+
+Write a short, decisive action plan:
+1. SITUATION — 1-2 sentences naming the specific problem the findings show, with the figure that
+   establishes it. Do not re-report the whole analysis; the reader has just seen it.
+2. RECOMMENDATIONS — 2-4 of them, most urgent first. Each MUST have all four of:
+   - the ACTION, stated concretely enough to start on Monday (not "improve performance");
+   - the FINDING it responds to, naming the specific location / category / metric and its figure
+     from the PRIOR FINDINGS. Quote the figures that ARE there and name the entities involved —
+     "costs are up at several sites" is weak where "Dental Supplies -$228,188.73" is checkable.
+     But NEVER reach for a number to make a recommendation look better evidenced: if the findings
+     support one figure, cite one; if they support none, name the entity and cite none. An invented
+     figure destroys the reader's ability to trust ANY of them, which costs far more than a thinly
+     evidenced recommendation ever could;
+   - an OWNER — the ROLE accountable (COO, CFO, Regional VP, Office Manager, …), never a person's
+     name;
+   - a TIMEFRAME (e.g. "within 2 weeks", "this quarter", "next 30 days").
+   State an EXPECTED IMPACT where the findings support one.
+3. WHAT WE DON'T KNOW — anything you'd need in order to act with more confidence, when the findings
+   genuinely don't cover it. Omit this section entirely if there is nothing real to say.
+
+THE RULES THAT MATTER MOST — an advisor who invents numbers is worse than one who admits a gap:
+- EVERY FIGURE YOU STATE MUST APPEAR IN THE PRIOR FINDINGS. Never introduce a new number. Do not
+  compute a new one — no sums, no percentages, no run-rates, no projections of your own.
+- NEVER PUT A NUMBER ON AN OUTCOME THAT HASN'T HAPPENED. Do not write "this will recover $2.5M",
+  "expected savings: $250K+", or "restoring these locations would add ~$160K per month". You have
+  no basis for any of those, and a confident fabricated projection is exactly the failure this
+  agent exists to avoid. Express expected impact QUALITATIVELY ("stops the largest single monthly
+  EBITDA drain", "removes the biggest contributor to the YTD miss") or by naming the EXISTING
+  figure at stake ("addresses the -$138,029.32 monthly variance at Lansing MI (Lake)").
+- If you genuinely must estimate, label it inline as an estimate AND state the basis in the same
+  sentence ("an estimate, assuming the -$66,105.44 monthly variance simply stops"). Never present
+  an estimate as a finding.
+- NEVER INVENT A CAUSE. The findings show WHAT moved, rarely WHY. Do not assert a provider left, a
+  contract lapsed, or demand softened unless the findings say so. Where the cause is unknown, make
+  DIAGNOSIS the first action ("determine why …") — that is a legitimate, honest recommendation.
+- Recommend only what the data supports. 2 well-founded recommendations beat 4 padded ones.
+- Do not invent location, person, vendor or account names — use only names in the PRIOR FINDINGS.
+
+Format as Markdown. Bold the action of each recommendation, and put its owner, timeframe and the
+finding it responds to on their own short lines beneath it. No tables.
+NEVER USE AN EM DASH OR EN DASH ("—", "–"), in any position. Use a comma, a colon, a semicolon,
+brackets, or two sentences; for a range use a plain hyphen ("2024-2026"). Write
+"**Owner**: Regional Director", never "**Owner** — Regional Director".
+Respond with ONLY a JSON object of the form:
+{"answer": "<Markdown text>", "status": "answered|partial",
+ "citations": [{"evidence_id": "F1"}]}"""
+