liquid-api 0.2.0__py3-none-any.whl

liquid/discovery/openapi.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import httpx
+import yaml
+
+from liquid.exceptions import DiscoveryError
+from liquid.models.schema import (
+    APISchema,
+    AuthRequirement,
+    Endpoint,
+    PaginationType,
+    Parameter,
+    ParameterLocation,
+    RateLimits,
+)
+
+logger = logging.getLogger(__name__)
+
+_SPEC_PATHS = [
+    "/openapi.json",
+    "/openapi.yaml",
+    "/swagger.json",
+    "/swagger/v1/swagger.json",
+    "/api-docs",
+    "/api/swagger.json",
+    "/.well-known/openapi.yaml",
+    "/.well-known/openapi.json",
+    "/v3/api-docs",
+]
+
+
+class OpenAPIDiscovery:
+    """Discovers APIs by finding and parsing OpenAPI/Swagger specifications."""
+
+    def __init__(self, http_client: httpx.AsyncClient | None = None) -> None:
+        self._external_client = http_client
+
+    async def discover(self, url: str) -> APISchema | None:
+        async with self._get_client() as client:
+            spec = await self._find_spec(client, url)
+            if spec is None:
+                return None
+
+            try:
+                return self._parse_spec(spec, url)
+            except Exception as e:
+                raise DiscoveryError(f"Failed to parse OpenAPI spec from {url}: {e}") from e
+
+    async def _find_spec(self, client: httpx.AsyncClient, base_url: str) -> dict[str, Any] | None:
+        base = base_url.rstrip("/")
+        for path in _SPEC_PATHS:
+            try:
+                resp = await client.get(f"{base}{path}", follow_redirects=True, timeout=10.0)
+                if resp.is_success:
+                    content_type = resp.headers.get("content-type", "")
+                    text = resp.text
+                    is_yaml = "yaml" in content_type or path.endswith(".yaml")
+                    spec = yaml.safe_load(text) if is_yaml else resp.json()
+                    if isinstance(spec, dict) and ("openapi" in spec or "swagger" in spec):
+                        logger.info("Found OpenAPI spec at %s%s", base, path)
+                        return spec
+            except Exception:
+                continue
+        return None
+
+    def _parse_spec(self, spec: dict[str, Any], source_url: str) -> APISchema:
+        version = spec.get("openapi", spec.get("swagger", ""))
+        is_v3 = str(version).startswith("3")
+
+        info = spec.get("info", {})
+        service_name = info.get("title", "Unknown")
+
+        endpoints = self._extract_endpoints(spec, is_v3)
+        auth = self._extract_auth(spec, is_v3)
+        rate_limits = self._extract_rate_limits(spec)
+
+        return APISchema(
+            source_url=source_url,
+            service_name=service_name,
+            discovery_method="openapi",
+            endpoints=endpoints,
+            auth=auth,
+            rate_limits=rate_limits,
+        )
+
+    def _extract_endpoints(self, spec: dict[str, Any], is_v3: bool) -> list[Endpoint]:
+        endpoints: list[Endpoint] = []
+        paths = spec.get("paths", {})
+
+        for path, path_item in paths.items():
+            if not isinstance(path_item, dict):
+                continue
+            for method in ("get", "post", "put", "patch", "delete"):
+                operation = path_item.get(method)
+                if not isinstance(operation, dict):
+                    continue
+                if operation.get("deprecated", False):
+                    continue
+
+                params = self._extract_parameters(path_item.get("parameters", []) + operation.get("parameters", []))
+                response_schema = self._extract_response_schema(operation, is_v3)
+                description = operation.get("summary", operation.get("description", ""))
+                pagination = self._infer_pagination(params)
+
+                endpoints.append(
+                    Endpoint(
+                        path=path,
+                        method=method.upper(),
+                        description=str(description)[:500] if description else "",
+                        parameters=params,
+                        response_schema=response_schema,
+                        pagination=pagination,
+                    )
+                )
+
+        return endpoints
+
+    def _extract_parameters(self, raw_params: list[dict[str, Any]]) -> list[Parameter]:
+        params: list[Parameter] = []
+        for p in raw_params:
+            if not isinstance(p, dict):
+                continue
+            name = p.get("name", "")
+            if not name:
+                continue
+
+            location_str = p.get("in", "query")
+            try:
+                location = ParameterLocation(location_str)
+            except ValueError:
+                location = ParameterLocation.QUERY
+
+            raw_schema = p.get("schema")
+            if raw_schema is None:
+                type_str = p.get("type")
+                raw_schema = {"type": type_str} if type_str else None
+
+            params.append(
+                Parameter(
+                    name=name,
+                    location=location,
+                    required=bool(p.get("required", False)),
+                    schema=raw_schema,
+                    description=p.get("description"),
+                )
+            )
+        return params
+
+    def _extract_response_schema(self, operation: dict[str, Any], is_v3: bool) -> dict[str, Any]:
+        responses = operation.get("responses", {})
+        success_resp = responses.get("200", responses.get("201", {}))
+        if not isinstance(success_resp, dict):
+            return {}
+
+        if is_v3:
+            content = success_resp.get("content", {})
+            json_content = content.get("application/json", {})
+            return json_content.get("schema", {})
+        else:
+            return success_resp.get("schema", {})
+
+    def _extract_auth(self, spec: dict[str, Any], is_v3: bool) -> AuthRequirement:
+        if is_v3:
+            components = spec.get("components", {})
+            security_schemes = components.get("securitySchemes", {})
+        else:
+            security_schemes = spec.get("securityDefinitions", {})
+
+        if not security_schemes:
+            return AuthRequirement(type="custom", tier="C")
+
+        for _name, scheme in security_schemes.items():
+            if not isinstance(scheme, dict):
+                continue
+            scheme_type = scheme.get("type", "").lower()
+
+            if scheme_type == "oauth2":
+                return AuthRequirement(type="oauth2", tier="A")
+            if scheme_type == "apikey":
+                return AuthRequirement(type="api_key", tier="C")
+            if scheme_type == "http":
+                bearer_scheme = scheme.get("scheme", "").lower()
+                if bearer_scheme == "bearer":
+                    return AuthRequirement(type="bearer", tier="A")
+                if bearer_scheme == "basic":
+                    return AuthRequirement(type="basic", tier="C")
+
+        return AuthRequirement(type="custom", tier="C")
+
+    def _extract_rate_limits(self, spec: dict[str, Any]) -> RateLimits | None:
+        extensions = {k: v for k, v in spec.items() if k.startswith("x-")}
+        rate_limit = extensions.get("x-rateLimit-limit") or extensions.get("x-rate-limit")
+        if rate_limit:
+            return RateLimits(requests_per_minute=float(rate_limit) if isinstance(rate_limit, int | float) else None)
+        return None
+
+    def _infer_pagination(self, params: list[Parameter]) -> PaginationType | None:
+        param_names = {p.name.lower() for p in params}
+        if "cursor" in param_names or "after" in param_names or "before" in param_names:
+            return PaginationType.CURSOR
+        if "offset" in param_names:
+            return PaginationType.OFFSET
+        if "page" in param_names or "page_number" in param_names:
+            return PaginationType.PAGE_NUMBER
+        return None
+
+    def _get_client(self) -> httpx.AsyncClient:
+        if self._external_client:
+
+            class _NoOpContext:
+                async def __aenter__(self):
+                    return self
+
+                async def __aexit__(self, *args):
+                    pass
+
+                def __getattr__(self, name):
+                    return getattr(self._client, name)
+
+                def __init__(self, client):
+                    self._client = client
+
+            return _NoOpContext(self._external_client)  # type: ignore[return-value]
+        return httpx.AsyncClient()

liquid/discovery/rest_heuristic.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+import httpx
+
+from liquid.exceptions import DiscoveryError
+from liquid.models.schema import APISchema, AuthRequirement, Endpoint
+
+if TYPE_CHECKING:
+    from liquid.protocols import LLMBackend
+
+logger = logging.getLogger(__name__)
+
+_PROBE_PATHS = [
+    "/api",
+    "/api/v1",
+    "/api/v2",
+    "/v1",
+    "/v2",
+    "/docs",
+    "/api-docs",
+    "/rest",
+]
+
+_COMMON_RESOURCE_PATHS = [
+    "/users",
+    "/items",
+    "/orders",
+    "/products",
+    "/accounts",
+    "/events",
+    "/webhooks",
+]
+
+
+class RESTHeuristicDiscovery:
+    """Discovers REST APIs by probing common patterns and using LLM to interpret."""
+
+    def __init__(
+        self,
+        llm: LLMBackend,
+        http_client: httpx.AsyncClient | None = None,
+    ) -> None:
+        self.llm = llm
+        self._external_client = http_client
+
+    async def discover(self, url: str) -> APISchema | None:
+        client = self._external_client or httpx.AsyncClient()
+        try:
+            found_endpoints = await self._probe_endpoints(client, url)
+            if not found_endpoints:
+                return None
+
+            return await self._interpret_with_llm(url, found_endpoints)
+        except DiscoveryError:
+            raise
+        except Exception as e:
+            raise DiscoveryError(f"REST heuristic discovery failed: {e}") from e
+        finally:
+            if not self._external_client:
+                await client.aclose()
+
+    async def _probe_endpoints(
+        self,
+        client: httpx.AsyncClient,
+        base_url: str,
+    ) -> list[dict]:
+        base = base_url.rstrip("/")
+        found: list[dict] = []
+
+        all_paths = _PROBE_PATHS + [f"/api/v1{p}" for p in _COMMON_RESOURCE_PATHS]
+
+        for path in all_paths:
+            try:
+                resp = await client.get(f"{base}{path}", timeout=5.0, follow_redirects=True)
+                if resp.is_success:
+                    content_type = resp.headers.get("content-type", "")
+                    if "json" in content_type:
+                        found.append(
+                            {
+                                "path": path,
+                                "status": resp.status_code,
+                                "content_type": content_type,
+                                "body_preview": resp.text[:500],
+                            }
+                        )
+            except Exception:
+                continue
+
+        return found
+
+    async def _interpret_with_llm(self, url: str, probed: list[dict]) -> APISchema:
+        from liquid.models.llm import Message
+
+        probe_summary = "\n".join(f"- {p['path']} ({p['status']}): {p['body_preview'][:200]}" for p in probed)
+
+        messages = [
+            Message(
+                role="system",
+                content=(
+                    "You are an API analyst. Given probe results from an unknown REST API, "
+                    "identify the likely endpoints, HTTP methods, and data structure. "
+                    "Respond with a JSON object containing: service_name (string), "
+                    "endpoints (array of {path, method, description}), "
+                    "auth_type (oauth2|api_key|bearer|basic|custom)."
+                ),
+            ),
+            Message(
+                role="user",
+                content=f"Base URL: {url}\n\nProbe results:\n{probe_summary}",
+            ),
+        ]
+
+        response = await self.llm.chat(messages)
+        return self._parse_llm_response(response.content or "{}", url, probed)
+
+    def _parse_llm_response(self, content: str, url: str, probed: list[dict]) -> APISchema:
+        import json
+
+        try:
+            data = json.loads(content)
+        except json.JSONDecodeError:
+            data = {}
+
+        endpoints = []
+        for ep in data.get("endpoints", []):
+            if isinstance(ep, dict) and "path" in ep:
+                endpoints.append(
+                    Endpoint(
+                        path=ep["path"],
+                        method=ep.get("method", "GET").upper(),
+                        description=ep.get("description", ""),
+                    )
+                )
+
+        if not endpoints:
+            endpoints = [
+                Endpoint(path=p["path"], method="GET", description=f"Discovered via probe ({p['status']})")
+                for p in probed
+            ]
+
+        auth_type = data.get("auth_type", "custom")
+        valid_auth_types = {"oauth2", "api_key", "bearer", "basic", "custom"}
+        if auth_type not in valid_auth_types:
+            auth_type = "custom"
+
+        tier = "A" if auth_type in ("oauth2", "bearer") else "C"
+
+        return APISchema(
+            source_url=url,
+            service_name=data.get("service_name", "Unknown"),
+            discovery_method="rest_heuristic",
+            endpoints=endpoints,
+            auth=AuthRequirement(type=auth_type, tier=tier),
+        )

liquid/events.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Protocol, runtime_checkable
+
+from pydantic import BaseModel, Field
+
+from liquid.models.schema import SchemaDiff  # noqa: TC001
+from liquid.models.sync import SyncError, SyncResult  # noqa: TC001
+
+
+class Event(BaseModel):
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    adapter_id: str | None = None
+
+
+class SyncCompleted(Event):
+    result: SyncResult
+
+
+class SyncFailed(Event):
+    error: SyncError
+    consecutive_failures: int = 1
+
+
+class ReDiscoveryNeeded(Event):
+    reason: str
+
+
+class AdapterRepaired(Event):
+    diff: SchemaDiff
+    auto_approved: bool = False
+
+
+@runtime_checkable
+class EventHandler(Protocol):
+    async def handle(self, event: Event) -> None: ...

liquid/exceptions.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+
+class LiquidError(Exception):
+    pass
+
+
+class DiscoveryError(LiquidError):
+    pass
+
+
+class AuthSetupError(LiquidError):
+    pass
+
+
+class MappingError(LiquidError):
+    pass
+
+
+class SyncRuntimeError(LiquidError):
+    pass
+
+
+class FieldNotFoundError(SyncRuntimeError):
+    pass
+
+
+class AuthError(SyncRuntimeError):
+    pass
+
+
+class RateLimitError(SyncRuntimeError):
+    def __init__(self, message: str = "Rate limit exceeded", retry_after: float | None = None) -> None:
+        super().__init__(message)
+        self.retry_after = retry_after
+
+
+class ServiceDownError(SyncRuntimeError):
+    pass
+
+
+class EndpointGoneError(SyncRuntimeError):
+    pass
+
+
+class ReDiscoveryNeededError(LiquidError):
+    pass
+
+
+class VaultError(LiquidError):
+    pass

liquid/mapping/__init__.py

@@ -0,0 +1,9 @@
+from liquid.mapping.learning import MappingLearner
+from liquid.mapping.proposer import MappingProposer
+from liquid.mapping.reviewer import MappingReview
+
+__all__ = [
+    "MappingLearner",
+    "MappingProposer",
+    "MappingReview",
+]

liquid/mapping/learning.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from liquid.models.adapter import FieldMapping
+    from liquid.protocols import KnowledgeStore
+
+logger = logging.getLogger(__name__)
+
+
+class MappingLearner:
+    """Records corrections and retrieves known mappings for learning."""
+
+    def __init__(self, knowledge: KnowledgeStore | None = None) -> None:
+        self.knowledge = knowledge
+
+    async def record_corrections(
+        self,
+        service: str,
+        target_model: str,
+        corrections: list[tuple[FieldMapping, FieldMapping]],
+    ) -> None:
+        """Store corrected mappings for future use."""
+        if not self.knowledge or not corrections:
+            return
+
+        corrected_mappings = [corrected for _original, corrected in corrections]
+        existing = await self.knowledge.find_mapping(service, target_model)
+
+        merged = self._merge_mappings(existing, corrected_mappings) if existing else corrected_mappings
+
+        await self.knowledge.store_mapping(service, target_model, merged)
+        logger.info(
+            "Stored %d corrected mappings for %s -> %s",
+            len(corrected_mappings),
+            service,
+            target_model,
+        )
+
+    async def get_known_mappings(
+        self,
+        service: str,
+        target_model: str,
+    ) -> list[FieldMapping] | None:
+        if not self.knowledge:
+            return None
+        return await self.knowledge.find_mapping(service, target_model)
+
+    def _merge_mappings(
+        self,
+        existing: list[FieldMapping],
+        new: list[FieldMapping],
+    ) -> list[FieldMapping]:
+        """Merge new corrections into existing mappings, preferring corrections."""
+        by_target: dict[str, FieldMapping] = {}
+        for m in existing:
+            by_target[m.target_field] = m
+        for m in new:
+            by_target[m.target_field] = m
+        return list(by_target.values())

liquid/mapping/proposer.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+import json
+import logging
+from typing import TYPE_CHECKING, Any
+
+from liquid.exceptions import MappingError
+from liquid.models.adapter import FieldMapping
+
+if TYPE_CHECKING:
+    from liquid.models.schema import APISchema
+    from liquid.protocols import KnowledgeStore, LLMBackend
+
+logger = logging.getLogger(__name__)
+
+
+class MappingProposer:
+    """Proposes field mappings using KnowledgeStore (if available) then LLM."""
+
+    def __init__(self, llm: LLMBackend, knowledge: KnowledgeStore | None = None) -> None:
+        self.llm = llm
+        self.knowledge = knowledge
+
+    async def propose(
+        self,
+        schema: APISchema,
+        target_model: dict[str, Any],
+        existing_mappings: list[FieldMapping] | None = None,
+        removed_fields: list[str] | None = None,
+    ) -> list[FieldMapping]:
+        if existing_mappings and removed_fields is not None:
+            return await self._selective_repropose(
+                schema,
+                target_model,
+                existing_mappings,
+                set(removed_fields),
+            )
+
+        if self.knowledge:
+            known = await self.knowledge.find_mapping(schema.service_name, json.dumps(target_model))
+            if known:
+                logger.info("Found existing mappings for %s in knowledge store", schema.service_name)
+                return known
+
+        return await self._propose_with_llm(schema, target_model)
+
+    async def _selective_repropose(
+        self,
+        schema: APISchema,
+        target_model: dict[str, Any],
+        existing: list[FieldMapping],
+        removed: set[str],
+    ) -> list[FieldMapping]:
+        kept: list[FieldMapping] = []
+        broken_targets: list[str] = []
+
+        for m in existing:
+            if m.source_path in removed:
+                broken_targets.append(m.target_field)
+                logger.info("Mapping %s → %s dropped (field removed)", m.source_path, m.target_field)
+            else:
+                kept.append(
+                    FieldMapping(
+                        source_path=m.source_path,
+                        target_field=m.target_field,
+                        transform=m.transform,
+                        confidence=1.0,
+                    )
+                )
+
+        if broken_targets:
+            new_proposals = await self._propose_with_llm(schema, target_model)
+            for proposal in new_proposals:
+                if proposal.target_field in broken_targets and not any(
+                    k.target_field == proposal.target_field for k in kept
+                ):
+                    kept.append(proposal)
+
+        return kept
+
+    async def _propose_with_llm(
+        self,
+        schema: APISchema,
+        target_model: dict[str, Any],
+    ) -> list[FieldMapping]:
+        from liquid.models.llm import Message
+
+        endpoints_desc = "\n".join(f"- {ep.method} {ep.path}: {ep.description}" for ep in schema.endpoints[:20])
+        response_schemas = "\n".join(
+            f"  {ep.path}: {json.dumps(ep.response_schema)[:300]}" for ep in schema.endpoints[:10] if ep.response_schema
+        )
+
+        messages = [
+            Message(
+                role="system",
+                content=(
+                    "You are a data mapping expert. Given an API schema and a target data model, "
+                    "propose field mappings. Respond with a JSON array of objects, each with: "
+                    "source_path (string, dot-notation), target_field (string), "
+                    "transform (string expression or null), confidence (float 0-1)."
+                ),
+            ),
+            Message(
+                role="user",
+                content=(
+                    f"API: {schema.service_name}\n"
+                    f"Endpoints:\n{endpoints_desc}\n\n"
+                    f"Response schemas:\n{response_schemas}\n\n"
+                    f"Target model:\n{json.dumps(target_model, indent=2)}\n\n"
+                    "Propose field mappings as a JSON array."
+                ),
+            ),
+        ]
+
+        try:
+            response = await self.llm.chat(messages)
+            return self._parse_mappings(response.content or "[]")
+        except Exception as e:
+            raise MappingError(f"LLM mapping proposal failed: {e}") from e
+
+    def _parse_mappings(self, content: str) -> list[FieldMapping]:
+        try:
+            start = content.find("[")
+            end = content.rfind("]") + 1
+            if start == -1 or end == 0:
+                return []
+            raw = json.loads(content[start:end])
+        except json.JSONDecodeError:
+            logger.warning("Failed to parse LLM mapping response")
+            return []
+
+        mappings: list[FieldMapping] = []
+        for item in raw:
+            if not isinstance(item, dict):
+                continue
+            if "source_path" not in item or "target_field" not in item:
+                continue
+            try:
+                mappings.append(
+                    FieldMapping(
+                        source_path=item["source_path"],
+                        target_field=item["target_field"],
+                        transform=item.get("transform"),
+                        confidence=float(item.get("confidence", 0.5)),
+                    )
+                )
+            except (ValueError, TypeError):
+                continue
+
+        return mappings