codi-api-agent 0.3.1__py3-none-any.whl

api_agent/doc_extract.py

@@ -0,0 +1,431 @@
+"""Turn a prose API **reference document** into a draft OpenAPI spec (doc support).
+
+This is the LLM analog of `spec_convert.convert_to_openapi` (which shells out to npx for
+Postman/RAML/API-Blueprint): here an LLM extracts the endpoints a human wrote in prose. The output
+is a normal OpenAPI dict that flows through `openapi_loader.build_catalog` and the rest of the
+pipeline unchanged.
+
+TRUST MODEL: extraction is a *draft*. The caller (the UI review gate) shows the endpoints to a
+person who approves/prunes before anything becomes callable — so the agent's "never invent
+endpoints" guarantee holds even though an LLM did the parsing. Each op carries `x-source` (the doc
+snippet it came from) so the reviewer can check provenance.
+
+Phase 2: large docs are split into heading-bounded chunks, extracted per chunk, and merged/deduped
+by (method, path) — so a 200 KB reference doesn't have to fit one context window. A doc URL is
+fetched (HTML stripped to text) before extraction.
+"""
+from __future__ import annotations
+
+import html as _html
+import re
+from urllib.parse import urlparse
+
+import requests
+
+from .llm import extract_json
+
+_HTTP_METHODS = {"get", "post", "put", "delete", "patch", "head", "options"}
+_UA = {"User-Agent": "api-agent/doc-extract"}
+
+
+DOC_EXTRACT_SYSTEM = """You convert an API REFERENCE DOCUMENT (prose, markdown, curl examples,
+parameter tables) into a strict JSON description of its HTTP endpoints.
+
+Rules — follow EXACTLY:
+- Extract ONLY endpoints that the document explicitly documents with a concrete HTTP METHOD and a
+  concrete PATH (e.g. "GET /api/v1/orders/{id}", "DELETE app/iotool/comments/comment", or a curl to
+  that path). Paths may be RELATIVE (e.g. "app/iotool/orders/order") — copy them exactly as written.
+  If a method+path is not written in the document, DO NOT output it. Never guess or invent. Omitting
+  a real endpoint is far better than inventing one.
+- BE EXHAUSTIVE. A chunk may document MANY endpoints (a dozen or more) — extract EVERY one you find,
+  do not stop after the first few.
+- SKIP GraphQL operations. A GraphQL query/mutation/subscription (a `query {…}`/`mutation {…}` sent
+  to a single GraphQL endpoint, or client hook names like `useGetTasksQuery`, `loginMutation`) is
+  NOT a REST endpoint and has no method+path — do not output it (a separate loader handles GraphQL).
+  Only output real REST HTTP endpoints (METHOD + URL path).
+- For each endpoint capture: method, path (with {braces} for path params), a short summary, and its
+  parameters. For each parameter give: name; `in` (one of "query", "path", "header"); type
+  ("string"/"integer"/"number"/"boolean"/"array"); required (true/false); enum (array of allowed
+  values IF the doc lists them, else omit); a short description; and an example value if shown.
+- If the endpoint takes a request body, put a short description or a small example in "body".
+- Capture the document's base URL (the server, e.g. "https://api.example.com") in "base_url" if
+  stated, else "". Capture the auth scheme in "auth": {scheme: "bearer"|"apiKey"|"basic"|"none",
+  header: "<header name if any>", format: "<e.g. 'Bearer <token>'>"} — only if the doc states it.
+- "source": copy the SHORT exact doc snippet (a line or two) the endpoint came from, so a human can
+  verify it. This is required for every endpoint.
+- The text may be ONE CHUNK of a larger document — just extract whatever endpoints appear in it.
+
+Return ONLY this JSON (no prose):
+{
+  "title": "<api name if stated, else ''>",
+  "base_url": "<server url or ''>",
+  "auth": {"scheme": "bearer|apiKey|basic|none", "header": "", "format": ""},
+  "endpoints": [
+    {"method": "GET", "path": "/api/v1/orders/{id}", "summary": "...",
+     "params": [{"name": "id", "in": "path", "type": "integer", "required": true,
+                 "description": "...", "example": 1}],
+     "body": "", "source": "GET /api/v1/orders/{id} — ..."}
+  ]
+}"""
+
+
+# --------------------------------------------------------------------------- #
+# Chunking + extraction + merge (map-reduce)
+# --------------------------------------------------------------------------- #
+def _chunk_doc(text: str, max_chars: int) -> list[str]:
+    """Split a doc into <=max_chars chunks WITHOUT cutting through a section: break on markdown
+    headings, then greedily pack whole sections; a single oversized section is hard-split."""
+    if len(text) <= max_chars:
+        return [text]
+    sections: list[str] = []
+    cur: list[str] = []
+    for line in text.split("\n"):
+        if re.match(r"^#{1,6}\s", line) and cur:  # a heading opens a new section
+            sections.append("\n".join(cur))
+            cur = [line]
+        else:
+            cur.append(line)
+    if cur:
+        sections.append("\n".join(cur))
+
+    chunks: list[str] = []
+    buf = ""
+    for sec in sections:
+        if len(sec) > max_chars:  # a giant section: flush, then hard-split it
+            if buf:
+                chunks.append(buf)
+                buf = ""
+            chunks.extend(sec[i:i + max_chars] for i in range(0, len(sec), max_chars))
+            continue
+        if buf and len(buf) + len(sec) + 1 > max_chars:
+            chunks.append(buf)
+            buf = sec
+        else:
+            buf = (buf + "\n" + sec) if buf else sec
+    if buf:
+        chunks.append(buf)
+    return chunks or [text[:max_chars]]
+
+
+def _extract_data(chunk: str, llm, model: str) -> dict:
+    """One extractor LLM call → the raw JSON dict. Best-effort: any failure yields no endpoints so a
+    single bad chunk never aborts the whole document."""
+    try:
+        resp = llm.complete(
+            [{"role": "system", "content": DOC_EXTRACT_SYSTEM},
+             {"role": "user", "content": f"API REFERENCE DOCUMENT (chunk):\n\n{chunk}"}],
+            model=model, json_mode=True,
+        )
+        return extract_json(resp.choices[0].message.content) or {}
+    except Exception:
+        return {}
+
+
+def _merge_data(datas: list[dict]) -> dict:
+    """Merge per-chunk extractions: union endpoints deduped by (method, path) — keeping the RICHER
+    duplicate (more params) — and take the first non-empty title / base_url / auth."""
+    merged: dict = {"title": "", "base_url": "", "auth": {}, "endpoints": []}
+    index: dict[tuple[str, str], int] = {}
+    for d in datas:
+        if not isinstance(d, dict):
+            continue
+        if not merged["title"] and d.get("title"):
+            merged["title"] = d["title"]
+        if not merged["base_url"] and (d.get("base_url") or "").strip():
+            merged["base_url"] = d["base_url"]
+        auth = d.get("auth") or {}
+        if not merged["auth"] and auth.get("scheme") and auth.get("scheme") != "none":
+            merged["auth"] = auth
+        for ep in d.get("endpoints") or []:
+            if not isinstance(ep, dict):
+                continue
+            key = (str(ep.get("method", "")).lower(), str(ep.get("path", "")))
+            if key in index:
+                prev = merged["endpoints"][index[key]]
+                if len(ep.get("params") or []) > len(prev.get("params") or []):
+                    merged["endpoints"][index[key]] = ep  # keep the more detailed one
+            else:
+                index[key] = len(merged["endpoints"])
+                merged["endpoints"].append(ep)
+    return merged
+
+
+def looks_graphql_heavy(text: str) -> tuple[bool, int]:
+    """Heuristic count of GraphQL-operation markers (useXxxQuery/Mutation hooks, `query {`/`mutation {`
+    blocks). Returns (is_heavy, count). Used to warn that GraphQL ops need the GraphQL loader, not
+    prose→OpenAPI extraction (they have no REST method+path so they can't be captured here)."""
+    t = text or ""
+    ops = set(re.findall(r"\buse[A-Z][A-Za-z0-9]*(?:Query|Mutation)\b", t))
+    blocks = len(re.findall(r"(?m)^\s*(?:query|mutation|subscription)\b[\s\w()]*\{", t))
+    n = len(ops) + blocks
+    return (n >= 10, n)
+
+
+def _llm_extract_data(text: str, llm, model: str, max_chars: int = 16000, progress=None) -> dict:
+    """Chunk → per-chunk LLM extract → merge, returning the raw endpoint `data` (pre-assembly)."""
+    chunks = _chunk_doc(text or "", max_chars)
+    datas: list[dict] = []
+    for i, chunk in enumerate(chunks):
+        if progress:
+            try:
+                progress(i + 1, len(chunks))
+            except Exception:
+                pass
+        datas.append(_extract_data(chunk, llm, model))
+    return _merge_data(datas)
+
+
+def extract_openapi_from_doc(text: str, llm, model: str, max_chars: int = 16000,
+                             progress=None) -> dict:
+    """LLM-ONLY extraction path (kept for callers/tests that specifically want it). Prefer
+    `build_doc_openapi` which is deterministic-first."""
+    return _assemble_openapi(_llm_extract_data(text, llm, model, max_chars, progress))
+
+
+# --------------------------------------------------------------------------- #
+# Structural (regex) extraction — the DEFAULT. Free, exhaustive, no hallucinated endpoints.
+# --------------------------------------------------------------------------- #
+# `METHOD <path>` anywhere: curl blocks, method-path headings, or a table cell. Quotes/backticks/
+# pipes are treated as separators (so `GET "https://…"` yields the URL without the quote).
+_METHOD_PATH_RE = re.compile(r"""\b(GET|POST|PUT|DELETE|PATCH|HEAD)\b[\s|`"']+([^\s`()<>|"']+)""", re.I)
+_HEADING_RE = re.compile(r"^#{1,6}\s+(.*)")
+
+
+def _normalize_path(raw: str) -> str:
+    """Clean a captured path token into an absolute OpenAPI path, or '' if it isn't one."""
+    p = raw.strip().strip("`\"'").rstrip(".,;:)\"'")
+    if "://" in p:                                 # absolute URL (any host/placeholder) → keep the path
+        p = urlparse(p).path
+    else:
+        p = re.sub(r"^\{\{[^}]+\}\}", "", p)       # {{baseUrl}}/{{host}} placeholder prefix
+        p = re.sub(r"^\$\w+", "", p)               # $HOST shell-var prefix
+    p = p.replace("${", "{")                       # JS template literal `${id}` → `{id}`
+    p = re.sub(r":(\w+)", r"{\1}", p)              # :id → {id}
+    if "/" not in p or p.count("{") != p.count("}"):
+        return ""                                  # not a path, or a token cut mid-param (`{dynamic`)
+    if not p.startswith("/"):
+        p = "/" + p
+    p = re.sub(r"/{2,}", "/", p)
+    return p.rstrip("/") or "/"
+
+
+def _nearest_heading(lines: list[str], idx: int) -> str:
+    for j in range(idx, max(-1, idx - 60), -1):
+        m = _HEADING_RE.match(lines[j])
+        if m:
+            return re.split(r"\s[—–-]\s", m.group(1))[0].strip()[:120]  # drop a "— opName" suffix
+    return ""
+
+
+def _find_base_url(text: str) -> str:
+    m = re.search(r"(?i)base\s*url[^\n]*?(https?://[^\s`\"'<>]+)", text)
+    if not m:
+        return ""
+    url = m.group(1).rstrip("/`.,")
+    # A base URL with a placeholder (`https://{host}`, `{{baseUrl}}`) isn't a real server — return ""
+    # so the loader doesn't store it (the user supplies the real host in the UI). Otherwise the
+    # executor would strip the placeholder and call `https:///…`.
+    return "" if ("{" in url or "}" in url) else url
+
+
+def _find_auth(text: str) -> dict:
+    if re.search(r"(?i)authorization:\s*bearer", text):
+        return {"scheme": "bearer", "header": "Authorization", "format": "Bearer <token>"}
+    if re.search(r"(?i)\bcookie:\s*\S", text):
+        return {"scheme": "apiKey", "header": "Cookie", "format": "<cookie>"}
+    if re.search(r"(?i)x-api-key", text):
+        return {"scheme": "apiKey", "header": "X-API-Key", "format": "<key>"}
+    return {}
+
+
+def _find_title(text: str) -> str:
+    for line in (text or "").split("\n"):
+        m = _HEADING_RE.match(line)
+        if m:
+            return m.group(1).strip()[:80]
+    return ""
+
+
+def structural_extract(text: str) -> dict:
+    """Deterministic endpoint discovery: scan for literal `METHOD path` tokens, deriving path params
+    from `{braces}` and query params from `?a=b` example URLs. Free, no LLM — and it CANNOT invent an
+    endpoint: every one is a literal string copied from the doc. Returns the same `data` shape the
+    LLM path does, so it flows through `_assemble_openapi`."""
+    lines = (text or "").split("\n")
+    found: dict[tuple[str, str], dict] = {}
+    for i, line in enumerate(lines):
+        for m in _METHOD_PATH_RE.finditer(line):
+            method = m.group(1).upper()
+            raw_path, _, query = m.group(2).partition("?")
+            path = _normalize_path(raw_path)
+            if not path:
+                continue
+            key = (method, path)
+            ep = found.get(key)
+            if ep is None:
+                ep = found[key] = {"method": method, "path": path,
+                                   "summary": _nearest_heading(lines, i),
+                                   "params": {}, "source": line.strip()[:200]}
+            for pp in re.findall(r"\{(\w+)\}", path):
+                ep["params"].setdefault(pp, {"name": pp, "in": "path", "required": True})
+            for qp in re.findall(r"[?&]([\w\[\]]+)=", "?" + query):
+                ep["params"].setdefault(qp, {"name": qp, "in": "query", "required": False})
+    endpoints = _dedupe_concrete_ids(
+        [{**ep, "params": list(ep["params"].values())} for ep in found.values()])
+    return {"title": _find_title(text), "base_url": _find_base_url(text),
+            "auth": _find_auth(text), "endpoints": endpoints}
+
+
+def _dedupe_concrete_ids(endpoints: list[dict]) -> list[dict]:
+    """Docs often show an endpoint BOTH templated (`/x/{id}`) and as a concrete example (`/x/4821`).
+    Drop the concrete twin when a templated sibling exists — keeping the useful `{id}` form. A slot
+    only counts as an id if the concrete segment contains a digit, so real sub-resources like
+    `/x/templates` are NOT mistaken for an id value."""
+    templated = [e for e in endpoints if "{" in e["path"]]
+
+    def is_instance(concrete: str, template: str) -> bool:
+        cs, ts = concrete.split("/"), template.split("/")
+        if len(cs) != len(ts):
+            return False
+        for c, t in zip(cs, ts):
+            if t.startswith("{") and t.endswith("}"):
+                if not re.search(r"\d", c):      # the {param} slot must look like an id (has a digit)
+                    return False
+            elif c != t:
+                return False
+        return True
+
+    out = []
+    for e in endpoints:
+        if "{" not in e["path"] and any(
+                e["method"] == t["method"] and is_instance(e["path"], t["path"]) for t in templated):
+            continue                              # concrete instance of a templated sibling → drop
+        out.append(e)
+    return out
+
+
+def build_doc_openapi(text: str, llm=None, model: str | None = None, use_llm: bool = False,
+                      max_chars: int = 16000, progress=None) -> dict:
+    """PRIMARY entry for doc support. Deterministic **structural** extraction is the default (free,
+    exhaustive, cannot hallucinate an endpoint). With `use_llm=True` the LLM ALSO runs and its output
+    is merged in — enriching params on the structurally-found endpoints, and filling in a prose-only
+    doc where regex found nothing. With `use_llm=False` the result is 100% from the doc's literal
+    method+path lines."""
+    datas = [structural_extract(text)]
+    if use_llm and llm is not None:
+        datas.append(_llm_extract_data(text, llm, model, max_chars, progress))
+    # structural first → its endpoints are authoritative; merge keeps the richer params per (method,path)
+    return _assemble_openapi(_merge_data(datas))
+
+
+# --------------------------------------------------------------------------- #
+# Assemble the extractor JSON into an OpenAPI dict build_catalog understands
+# --------------------------------------------------------------------------- #
+def _assemble_openapi(data: dict) -> dict:
+    """Assemble the extractor's JSON into an OpenAPI 3 dict `build_catalog` understands. Skips any
+    endpoint without a concrete method + `/path`. Enum values fold into the param description (the
+    tool schema only carries type+description). `x-source`/`x-auth` are informational extensions."""
+    spec: dict = {
+        "openapi": "3.0.0",
+        "info": {"title": (data.get("title") or "Extracted API").strip(), "version": "1.0"},
+        "paths": {},
+    }
+    base = (data.get("base_url") or "").strip()
+    if base:
+        spec["servers"] = [{"url": base}]
+    auth = data.get("auth") or {}
+    if auth.get("scheme") and auth.get("scheme") != "none":
+        spec["x-auth"] = {"scheme": auth.get("scheme"), "header": auth.get("header", ""),
+                          "format": auth.get("format", "")}
+
+    for ep in data.get("endpoints") or []:
+        if not isinstance(ep, dict):
+            continue
+        method = str(ep.get("method", "")).strip().lower()
+        path = str(ep.get("path", "")).strip()
+        if method not in _HTTP_METHODS or not path:
+            continue
+        # Docs often write RELATIVE paths ("app/iotool/orders/order") — make them absolute so they
+        # aren't dropped. A slashless single token (e.g. a GraphQL op name "loginMutation") is NOT a
+        # REST path → skip it. Strip any query string the model left on the path.
+        path = path.split("?", 1)[0].strip()
+        if not path.startswith("/"):
+            if "/" in path:
+                path = "/" + path
+            else:
+                continue
+        params = []
+        for p in ep.get("params") or []:
+            if not isinstance(p, dict):
+                continue
+            name, loc = p.get("name"), p.get("in")
+            if not name or loc not in ("query", "path", "header"):
+                continue
+            desc = (p.get("description") or "").strip()
+            enum = p.get("enum")
+            if isinstance(enum, list) and enum:
+                desc = (desc + f" (one of: {', '.join(str(e) for e in enum)})").strip()
+            ptype = p.get("type") if p.get("type") in ("string", "integer", "number", "boolean", "array") else "string"
+            param = {"name": name, "in": loc,
+                     "required": bool(p.get("required")) or loc == "path",
+                     "schema": {"type": ptype}, "description": desc}
+            if p.get("example") not in (None, ""):
+                param["example"] = p["example"]
+            params.append(param)
+        op = {
+            "summary": (ep.get("summary") or "").strip(),
+            "parameters": params,
+            "x-source": str(ep.get("source") or "")[:500],
+        }
+        if ep.get("body"):
+            op["requestBody"] = {"description": str(ep["body"])[:300],
+                                 "content": {"application/json": {"schema": {"type": "object"}}}}
+        spec["paths"].setdefault(path, {})[method] = op
+    return spec
+
+
+def endpoint_rows(spec: dict) -> list[tuple[str, str, dict]]:
+    """(method, path, op) rows for the review UI, in a stable order."""
+    rows = []
+    for path, methods in (spec.get("paths") or {}).items():
+        for method, op in methods.items():
+            if isinstance(op, dict) and method.lower() in _HTTP_METHODS:
+                rows.append((method.lower(), path, op))
+    return rows
+
+
+def prune_openapi(spec: dict, keep: set[tuple[str, str]]) -> dict:
+    """Return a copy of `spec` keeping only the (method, path) operations the reviewer approved."""
+    out = {k: v for k, v in spec.items() if k != "paths"}
+    out["paths"] = {}
+    for path, methods in (spec.get("paths") or {}).items():
+        kept = {m: op for m, op in methods.items() if (m.lower(), path) in keep}
+        if kept:
+            out["paths"][path] = kept
+    return out
+
+
+# --------------------------------------------------------------------------- #
+# Doc fetching (URL → text)
+# --------------------------------------------------------------------------- #
+def _html_to_text(html_text: str) -> str:
+    """Cheap HTML → text (no extra dependency): drop script/style, turn block tags into newlines,
+    strip remaining tags, unescape entities, collapse whitespace. Good enough for LLM extraction."""
+    t = re.sub(r"(?is)<(script|style|noscript)\b.*?</\1>", " ", html_text)
+    t = re.sub(r"(?i)<br\s*/?>", "\n", t)
+    t = re.sub(r"(?i)</(p|div|li|tr|h[1-6]|section|article|pre)>", "\n", t)
+    t = re.sub(r"<[^>]+>", " ", t)
+    t = _html.unescape(t)
+    t = re.sub(r"[ \t]+", " ", t)
+    return re.sub(r"\n{3,}", "\n\n", t).strip()
+
+
+def fetch_doc_text(source: str, timeout: int = 20) -> str:
+    """Load a reference doc's text from a URL (HTML stripped to text) or a local file path."""
+    if source.startswith(("http://", "https://")):
+        resp = requests.get(source, timeout=timeout, headers=_UA)
+        resp.raise_for_status()
+        ctype = resp.headers.get("content-type", "").lower()
+        return _html_to_text(resp.text) if "html" in ctype else resp.text
+    with open(source, encoding="utf-8") as f:
+        return f.read()