liquid-api 0.2.0__py3-none-any.whl

liquid/discovery/browser.py

@@ -0,0 +1,175 @@
+"""Browser-based API discovery using Playwright.
+
+This is the last-resort discovery strategy (Level 4). It launches a headless
+browser, navigates the target URL, captures network requests, and uses an LLM
+to classify discovered endpoints.
+
+Requires the `browser` extra: pip install liquid[browser]
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+from liquid.exceptions import DiscoveryError
+from liquid.models.schema import APISchema, AuthRequirement, Endpoint
+
+if TYPE_CHECKING:
+    from liquid.protocols import LLMBackend
+
+logger = logging.getLogger(__name__)
+
+_PLAYWRIGHT_AVAILABLE = False
+try:
+    from playwright.async_api import async_playwright  # type: ignore[import-untyped]
+
+    _PLAYWRIGHT_AVAILABLE = True
+except ImportError:
+    pass
+
+
+class BrowserDiscovery:
+    """Discovers APIs by browsing the target URL and capturing network traffic.
+
+    This strategy:
+    1. Launches a headless Chromium browser
+    2. Navigates to the target URL
+    3. Captures all XHR/Fetch network requests
+    4. Uses LLM to classify captured requests into API endpoints
+    """
+
+    def __init__(self, llm: LLMBackend, timeout_ms: int = 30000) -> None:
+        self.llm = llm
+        self.timeout_ms = timeout_ms
+
+    async def discover(self, url: str) -> APISchema | None:
+        if not _PLAYWRIGHT_AVAILABLE:
+            logger.debug("Playwright not installed, skipping BrowserDiscovery")
+            return None
+
+        try:
+            captured = await self._browse_and_capture(url)
+            if not captured:
+                return None
+            return await self._classify_with_llm(url, captured)
+        except DiscoveryError:
+            raise
+        except Exception as e:
+            raise DiscoveryError(f"Browser discovery failed for {url}: {e}") from e
+
+    async def _browse_and_capture(self, url: str) -> list[dict[str, Any]]:
+        captured: list[dict[str, Any]] = []
+
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context()
+            page = await context.new_page()
+
+            async def on_response(response):
+                request = response.request
+                if request.resource_type in ("xhr", "fetch"):
+                    content_type = response.headers.get("content-type", "")
+                    if "json" in content_type or "xml" in content_type:
+                        try:
+                            body = await response.text()
+                            captured.append(
+                                {
+                                    "url": request.url,
+                                    "method": request.method,
+                                    "status": response.status,
+                                    "content_type": content_type,
+                                    "body_preview": body[:500],
+                                }
+                            )
+                        except Exception:
+                            pass
+
+            page.on("response", on_response)
+
+            try:
+                await page.goto(url, wait_until="networkidle", timeout=self.timeout_ms)
+                # Scroll to trigger lazy-loaded requests
+                await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+                await page.wait_for_timeout(2000)
+            except Exception as e:
+                logger.warning("Browser navigation error for %s: %s", url, e)
+            finally:
+                await browser.close()
+
+        return captured
+
+    async def _classify_with_llm(self, url: str, captured: list[dict[str, Any]]) -> APISchema:
+        from liquid.models.llm import Message
+
+        captures_summary = "\n".join(
+            f"- {c['method']} {c['url']} ({c['status']}): {c['body_preview'][:150]}" for c in captured[:20]
+        )
+
+        messages = [
+            Message(
+                role="system",
+                content=(
+                    "You are an API analyst. Given captured network requests from browsing a website, "
+                    "identify the API endpoints. Respond with JSON: "
+                    '{"service_name": "...", "endpoints": [{"path": "...", "method": "...", "description": "..."}], '
+                    '"auth_type": "oauth2|api_key|bearer|basic|custom"}'
+                ),
+            ),
+            Message(
+                role="user",
+                content=f"URL: {url}\n\nCaptured requests:\n{captures_summary}",
+            ),
+        ]
+
+        response = await self.llm.chat(messages)
+        return self._parse_response(response.content or "{}", url, captured)
+
+    def _parse_response(self, content: str, url: str, captured: list[dict[str, Any]]) -> APISchema:
+        import json
+        from urllib.parse import urlparse
+
+        try:
+            data = json.loads(content)
+        except json.JSONDecodeError:
+            data = {}
+
+        endpoints: list[Endpoint] = []
+        for ep in data.get("endpoints", []):
+            if isinstance(ep, dict) and "path" in ep:
+                endpoints.append(
+                    Endpoint(
+                        path=ep["path"],
+                        method=ep.get("method", "GET").upper(),
+                        description=ep.get("description", ""),
+                    )
+                )
+
+        if not endpoints:
+            # Fallback: create endpoints from captured URLs
+            seen_paths: set[str] = set()
+            for c in captured:
+                parsed = urlparse(c["url"])
+                if parsed.path not in seen_paths:
+                    seen_paths.add(parsed.path)
+                    endpoints.append(
+                        Endpoint(
+                            path=parsed.path,
+                            method=c["method"],
+                            description=f"Captured via browser ({c['status']})",
+                        )
+                    )
+
+        auth_type = data.get("auth_type", "custom")
+        valid_types = {"oauth2", "api_key", "bearer", "basic", "custom"}
+        if auth_type not in valid_types:
+            auth_type = "custom"
+        tier = "A" if auth_type in ("oauth2", "bearer") else "C"
+
+        return APISchema(
+            source_url=url,
+            service_name=data.get("service_name", urlparse(url).hostname or "Unknown"),
+            discovery_method="browser",
+            endpoints=endpoints,
+            auth=AuthRequirement(type=auth_type, tier=tier),
+        )

liquid/discovery/diff.py

@@ -0,0 +1,66 @@
+"""Schema diff utility for detecting API changes."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from liquid.models.schema import APISchema, Endpoint, SchemaDiff
+
+
+def diff_schemas(old: APISchema, new: APISchema) -> SchemaDiff:
+    """Compare two API schemas and return a structured diff."""
+    old_ep_map = {(ep.path, ep.method): ep for ep in old.endpoints}
+    new_ep_map = {(ep.path, ep.method): ep for ep in new.endpoints}
+
+    old_keys = set(old_ep_map.keys())
+    new_keys = set(new_ep_map.keys())
+
+    added_endpoints = [new_ep_map[k] for k in sorted(new_keys - old_keys)]
+    removed_endpoints = [old_ep_map[k] for k in sorted(old_keys - new_keys)]
+    unchanged_endpoints = [new_ep_map[k] for k in sorted(old_keys & new_keys)]
+
+    old_fields = _extract_all_fields(old.endpoints)
+    new_fields = _extract_all_fields(new.endpoints)
+
+    added_fields = sorted(new_fields - old_fields)
+    removed_fields = sorted(old_fields - new_fields)
+    unchanged_fields = sorted(old_fields & new_fields)
+
+    has_breaking = bool(removed_endpoints or removed_fields)
+
+    return SchemaDiff(
+        added_endpoints=added_endpoints,
+        removed_endpoints=removed_endpoints,
+        unchanged_endpoints=unchanged_endpoints,
+        added_fields=added_fields,
+        removed_fields=removed_fields,
+        unchanged_fields=unchanged_fields,
+        has_breaking_changes=has_breaking,
+    )
+
+
+def _extract_all_fields(endpoints: list[Endpoint]) -> set[str]:
+    """Extract all field paths from endpoint response schemas."""
+    fields: set[str] = set()
+    for ep in endpoints:
+        if ep.response_schema:
+            _collect_fields(ep.response_schema, "", fields)
+    return fields
+
+
+def _collect_fields(schema: dict[str, Any], prefix: str, fields: set[str]) -> None:
+    schema_type = schema.get("type", "")
+
+    if schema_type == "object":
+        properties = schema.get("properties", {})
+        for prop_name, prop_schema in properties.items():
+            full_path = f"{prefix}.{prop_name}" if prefix else prop_name
+            fields.add(full_path)
+            if isinstance(prop_schema, dict):
+                _collect_fields(prop_schema, full_path, fields)
+
+    elif schema_type == "array":
+        items = schema.get("items", {})
+        item_prefix = f"{prefix}[]" if prefix else "[]"
+        if isinstance(items, dict):
+            _collect_fields(items, item_prefix, fields)

liquid/discovery/graphql.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import httpx
+
+from liquid.exceptions import DiscoveryError
+from liquid.models.schema import (
+    APISchema,
+    AuthRequirement,
+    Endpoint,
+    Parameter,
+    ParameterLocation,
+)
+
+logger = logging.getLogger(__name__)
+
+_INTROSPECTION_QUERY = """
+query IntrospectionQuery {
+  __schema {
+    queryType { name }
+    mutationType { name }
+    types {
+      kind
+      name
+      description
+      fields {
+        name
+        description
+        args {
+          name
+          description
+          type { kind name ofType { kind name ofType { kind name } } }
+          defaultValue
+        }
+        type { kind name ofType { kind name ofType { kind name } } }
+      }
+    }
+  }
+}
+"""
+
+_GRAPHQL_PATHS = ["/graphql", "/api/graphql", "/graphql/v1", "/gql"]
+
+
+class GraphQLDiscovery:
+    """Discovers APIs by running a GraphQL introspection query."""
+
+    def __init__(self, http_client: httpx.AsyncClient | None = None) -> None:
+        self._external_client = http_client
+
+    async def discover(self, url: str) -> APISchema | None:
+        client = self._external_client or httpx.AsyncClient()
+        try:
+            introspection = await self._run_introspection(client, url)
+            if introspection is None:
+                return None
+            return self._parse_introspection(introspection, url)
+        finally:
+            if not self._external_client:
+                await client.aclose()
+
+    async def _run_introspection(
+        self,
+        client: httpx.AsyncClient,
+        base_url: str,
+    ) -> dict[str, Any] | None:
+        base = base_url.rstrip("/")
+        for path in _GRAPHQL_PATHS:
+            try:
+                resp = await client.post(
+                    f"{base}{path}",
+                    json={"query": _INTROSPECTION_QUERY},
+                    headers={"Content-Type": "application/json"},
+                    timeout=10.0,
+                )
+                if resp.is_success:
+                    data = resp.json()
+                    if "data" in data and "__schema" in data["data"]:
+                        logger.info("GraphQL introspection succeeded at %s%s", base, path)
+                        return data["data"]["__schema"]
+            except Exception:
+                continue
+        return None
+
+    def _parse_introspection(self, schema: dict[str, Any], source_url: str) -> APISchema:
+        try:
+            endpoints = self._extract_endpoints(schema)
+        except Exception as e:
+            raise DiscoveryError(f"Failed to parse GraphQL introspection: {e}") from e
+
+        return APISchema(
+            source_url=source_url,
+            service_name=self._infer_service_name(source_url),
+            discovery_method="graphql",
+            endpoints=endpoints,
+            auth=AuthRequirement(type="bearer", tier="A"),
+        )
+
+    def _extract_endpoints(self, schema: dict[str, Any]) -> list[Endpoint]:
+        endpoints: list[Endpoint] = []
+        types_map = {t["name"]: t for t in schema.get("types", []) if isinstance(t, dict)}
+
+        query_type_name = (schema.get("queryType") or {}).get("name", "Query")
+        mutation_type_name = (schema.get("mutationType") or {}).get("name", "Mutation")
+
+        for type_name, method in [(query_type_name, "POST"), (mutation_type_name, "POST")]:
+            type_def = types_map.get(type_name)
+            if not type_def:
+                continue
+            for field in type_def.get("fields", []):
+                if not isinstance(field, dict):
+                    continue
+                name = field.get("name", "")
+                if name.startswith("_"):
+                    continue
+
+                params = [
+                    Parameter(
+                        name=arg["name"],
+                        location=ParameterLocation.BODY,
+                        required=arg.get("type", {}).get("kind") == "NON_NULL",
+                        description=arg.get("description"),
+                    )
+                    for arg in field.get("args", [])
+                    if isinstance(arg, dict)
+                ]
+
+                op_type = "query" if type_name == query_type_name else "mutation"
+                endpoints.append(
+                    Endpoint(
+                        path=f"/graphql#{op_type}.{name}",
+                        method=method,
+                        description=field.get("description", "") or "",
+                        parameters=params,
+                        response_schema=self._type_to_schema(field.get("type", {})),
+                    )
+                )
+
+        return endpoints
+
+    def _type_to_schema(self, gql_type: dict[str, Any]) -> dict[str, Any]:
+        kind = gql_type.get("kind", "")
+        name = gql_type.get("name", "")
+
+        if kind == "NON_NULL":
+            return self._type_to_schema(gql_type.get("ofType", {}))
+        if kind == "LIST":
+            return {"type": "array", "items": self._type_to_schema(gql_type.get("ofType", {}))}
+        if kind == "SCALAR":
+            return {"type": _scalar_to_json_type(name)}
+        if kind in ("OBJECT", "INTERFACE"):
+            return {"type": "object", "title": name}
+        if kind == "ENUM":
+            return {"type": "string", "title": name}
+        return {"type": "object"}
+
+    def _infer_service_name(self, url: str) -> str:
+        from urllib.parse import urlparse
+
+        parsed = urlparse(url)
+        host = parsed.hostname or "unknown"
+        parts = host.split(".")
+        if len(parts) >= 2:
+            return parts[-2].capitalize()
+        return host.capitalize()
+
+
+def _scalar_to_json_type(name: str) -> str:
+    mapping = {
+        "String": "string",
+        "Int": "integer",
+        "Float": "number",
+        "Boolean": "boolean",
+        "ID": "string",
+        "DateTime": "string",
+        "Date": "string",
+    }
+    return mapping.get(name, "string")

liquid/discovery/mcp.py

@@ -0,0 +1,159 @@
+"""MCP-based API discovery.
+
+If the service publishes an MCP server, tools and resources are already
+structured with types and descriptions — the cheapest and most reliable
+discovery method (Level 1).
+
+Requires the `mcp` extra: pip install liquid[mcp]
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from liquid.exceptions import DiscoveryError
+from liquid.models.schema import (
+    APISchema,
+    AuthRequirement,
+    Endpoint,
+    Parameter,
+    ParameterLocation,
+)
+
+logger = logging.getLogger(__name__)
+
+_MCP_AVAILABLE = False
+try:
+    from mcp import ClientSession  # type: ignore[import-untyped]
+    from mcp.client.streamable_http import streamable_http_client  # type: ignore[import-untyped]
+
+    _MCP_AVAILABLE = True
+except ImportError:
+    pass
+
+
+class MCPDiscovery:
+    """Discovers APIs by connecting to an MCP server.
+
+    MCP servers publish tools and resources with structured types
+    and descriptions. This strategy connects via Streamable HTTP,
+    lists available tools/resources, and maps them to APISchema.
+
+    Falls back gracefully if the `mcp` package is not installed
+    or the URL doesn't expose an MCP endpoint.
+    """
+
+    def __init__(self, mcp_path: str = "/mcp") -> None:
+        self.mcp_path = mcp_path
+
+    async def discover(self, url: str) -> APISchema | None:
+        if not _MCP_AVAILABLE:
+            logger.debug("MCP SDK not installed, skipping MCPDiscovery")
+            return None
+
+        mcp_url = f"{url.rstrip('/')}{self.mcp_path}"
+        try:
+            return await self._connect_and_discover(mcp_url, url)
+        except DiscoveryError:
+            raise
+        except Exception as e:
+            logger.debug("MCP discovery failed for %s: %s", mcp_url, e)
+            return None
+
+    async def _connect_and_discover(self, mcp_url: str, source_url: str) -> APISchema | None:
+        async with streamable_http_client(mcp_url) as (read, write), ClientSession(read, write) as session:
+            await session.initialize()
+
+            tools_result = await session.list_tools()
+            resources_result = await session.list_resources()
+
+            tools = tools_result.tools if tools_result else []
+            resources = resources_result.resources if resources_result else []
+
+            if not tools and not resources:
+                return None
+
+            endpoints = self._tools_to_endpoints(tools)
+            resource_endpoints = self._resources_to_endpoints(resources)
+            endpoints.extend(resource_endpoints)
+
+            service_name = self._infer_service_name(source_url)
+
+            return APISchema(
+                source_url=source_url,
+                service_name=service_name,
+                discovery_method="mcp",
+                endpoints=endpoints,
+                auth=AuthRequirement(type="bearer", tier="A"),
+            )
+
+    def _tools_to_endpoints(self, tools: list[Any]) -> list[Endpoint]:
+        endpoints: list[Endpoint] = []
+        for tool in tools:
+            name = getattr(tool, "name", str(tool))
+            description = getattr(tool, "description", "") or ""
+            input_schema = getattr(tool, "inputSchema", None) or {}
+
+            params = self._schema_to_parameters(input_schema)
+
+            endpoints.append(
+                Endpoint(
+                    path=f"/mcp/tools/{name}",
+                    method="POST",
+                    description=description[:500],
+                    parameters=params,
+                    response_schema={"type": "object"},
+                )
+            )
+        return endpoints
+
+    def _resources_to_endpoints(self, resources: list[Any]) -> list[Endpoint]:
+        endpoints: list[Endpoint] = []
+        for resource in resources:
+            uri = str(getattr(resource, "uri", resource))
+            name = getattr(resource, "name", uri)
+            description = getattr(resource, "description", "") or ""
+            mime_type = getattr(resource, "mimeType", "application/json")
+
+            endpoints.append(
+                Endpoint(
+                    path=f"/mcp/resources/{name}",
+                    method="GET",
+                    description=description[:500] or f"Resource: {uri}",
+                    response_schema={"type": "object", "mimeType": mime_type},
+                )
+            )
+        return endpoints
+
+    def _schema_to_parameters(self, input_schema: dict[str, Any]) -> list[Parameter]:
+        if not isinstance(input_schema, dict):
+            return []
+
+        properties = input_schema.get("properties", {})
+        required_fields = set(input_schema.get("required", []))
+        params: list[Parameter] = []
+
+        for prop_name, prop_schema in properties.items():
+            if not isinstance(prop_schema, dict):
+                continue
+            params.append(
+                Parameter(
+                    name=prop_name,
+                    location=ParameterLocation.BODY,
+                    required=prop_name in required_fields,
+                    schema=prop_schema,
+                    description=prop_schema.get("description"),
+                )
+            )
+        return params
+
+    def _infer_service_name(self, url: str) -> str:
+        from urllib.parse import urlparse
+
+        parsed = urlparse(url)
+        host = parsed.hostname or "unknown"
+        parts = host.split(".")
+        if len(parts) >= 2:
+            return parts[-2].capitalize()
+        return host.capitalize()