liquid-api 0.2.0__py3-none-any.whl

liquid/__init__.py

@@ -0,0 +1,60 @@
+"""Liquid — AI discovers APIs. Code syncs data. No adapters to write."""
+
+__version__ = "0.2.0"
+
+from liquid.client import Liquid
+from liquid.exceptions import (
+    AuthError,
+    AuthSetupError,
+    DiscoveryError,
+    EndpointGoneError,
+    FieldNotFoundError,
+    LiquidError,
+    MappingError,
+    RateLimitError,
+    ReDiscoveryNeededError,
+    ServiceDownError,
+    SyncRuntimeError,
+    VaultError,
+)
+from liquid.models import (
+    AdapterConfig,
+    APISchema,
+    AuthRequirement,
+    DeliveryResult,
+    Endpoint,
+    FieldMapping,
+    MappedRecord,
+    SyncConfig,
+    SyncResult,
+)
+from liquid.protocols import DataSink, KnowledgeStore, LLMBackend, Vault
+
+__all__ = [
+    "APISchema",
+    "AdapterConfig",
+    "AuthError",
+    "AuthRequirement",
+    "AuthSetupError",
+    "DataSink",
+    "DeliveryResult",
+    "DiscoveryError",
+    "Endpoint",
+    "EndpointGoneError",
+    "FieldMapping",
+    "FieldNotFoundError",
+    "KnowledgeStore",
+    "LLMBackend",
+    "Liquid",
+    "LiquidError",
+    "MappedRecord",
+    "MappingError",
+    "RateLimitError",
+    "ReDiscoveryNeededError",
+    "ServiceDownError",
+    "SyncConfig",
+    "SyncResult",
+    "SyncRuntimeError",
+    "Vault",
+    "VaultError",
+]

liquid/_defaults.py

@@ -0,0 +1,58 @@
+"""Default in-memory implementations of protocols for testing and quick starts."""
+
+from __future__ import annotations
+
+from liquid.exceptions import VaultError
+from liquid.models.adapter import FieldMapping  # noqa: TC001
+from liquid.models.llm import DeliveryResult, MappedRecord
+
+
+class InMemoryVault:
+    """Dict-based vault for testing. Not for production."""
+
+    def __init__(self) -> None:
+        self._data: dict[str, str] = {}
+
+    async def store(self, key: str, value: str) -> None:
+        self._data[key] = value
+
+    async def get(self, key: str) -> str:
+        if key not in self._data:
+            raise VaultError(f"Key not found: {key}")
+        return self._data[key]
+
+    async def delete(self, key: str) -> None:
+        self._data.pop(key, None)
+
+
+class InMemoryKnowledgeStore:
+    """Dict-based knowledge store for testing."""
+
+    def __init__(self) -> None:
+        self._data: dict[str, list[FieldMapping]] = {}
+
+    async def find_mapping(self, service: str, target_model: str) -> list[FieldMapping] | None:
+        return self._data.get(f"{service}:{target_model}")
+
+    async def store_mapping(self, service: str, target_model: str, mappings: list[FieldMapping]) -> None:
+        self._data[f"{service}:{target_model}"] = mappings
+
+
+class StdoutSink:
+    """Prints records to stdout. For debugging only."""
+
+    async def deliver(self, records: list[MappedRecord]) -> DeliveryResult:
+        for record in records:
+            print(f"[StdoutSink] {record.source_endpoint}: {record.mapped_data}")
+        return DeliveryResult(delivered=len(records))
+
+
+class CollectorSink:
+    """Collects records in memory. Useful for testing."""
+
+    def __init__(self) -> None:
+        self.records: list[MappedRecord] = []
+
+    async def deliver(self, records: list[MappedRecord]) -> DeliveryResult:
+        self.records.extend(records)
+        return DeliveryResult(delivered=len(records))

liquid/auth/__init__.py

@@ -0,0 +1,8 @@
+from liquid.auth.classifier import AuthClassifier, EscalationInfo
+from liquid.auth.manager import AuthManager
+
+__all__ = [
+    "AuthClassifier",
+    "AuthManager",
+    "EscalationInfo",
+]

liquid/auth/classifier.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel
+
+if TYPE_CHECKING:
+    from liquid.models.schema import AuthRequirement
+
+
+class EscalationInfo(BaseModel):
+    tier: str
+    action_required: str
+    docs_url: str | None = None
+    instructions: str = ""
+
+
+class AuthClassifier:
+    """Classifies auth requirements and generates escalation info."""
+
+    def classify(self, auth: AuthRequirement) -> EscalationInfo:
+        match auth.tier:
+            case "A":
+                return EscalationInfo(
+                    tier="A",
+                    action_required="none",
+                    docs_url=auth.docs_url,
+                    instructions="OAuth flow can proceed automatically. Redirect user to authorize.",
+                )
+            case "B":
+                return EscalationInfo(
+                    tier="B",
+                    action_required="admin_registration",
+                    docs_url=auth.docs_url,
+                    instructions=(
+                        "This service requires creating a developer application first. "
+                        "An admin needs to register the app at the service's developer portal, "
+                        "then provide client_id and client_secret."
+                    ),
+                )
+            case "C":
+                return self._classify_tier_c(auth)
+            case _:
+                return EscalationInfo(
+                    tier=auth.tier,
+                    action_required="manual",
+                    docs_url=auth.docs_url,
+                    instructions="Unknown auth tier. Manual configuration required.",
+                )
+
+    def _classify_tier_c(self, auth: AuthRequirement) -> EscalationInfo:
+        match auth.type:
+            case "api_key":
+                return EscalationInfo(
+                    tier="C",
+                    action_required="provide_api_key",
+                    docs_url=auth.docs_url,
+                    instructions="This service requires an API key. Obtain it from the service dashboard.",
+                )
+            case "basic":
+                return EscalationInfo(
+                    tier="C",
+                    action_required="provide_credentials",
+                    docs_url=auth.docs_url,
+                    instructions="This service requires username and password for Basic auth.",
+                )
+            case _:
+                return EscalationInfo(
+                    tier="C",
+                    action_required="manual_configuration",
+                    docs_url=auth.docs_url,
+                    instructions="This service requires custom authentication. Contact support for setup.",
+                )

liquid/auth/manager.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+import httpx
+
+from liquid.exceptions import AuthError, VaultError
+
+if TYPE_CHECKING:
+    from liquid.models.schema import AuthRequirement, OAuthConfig
+    from liquid.protocols import Vault
+
+logger = logging.getLogger(__name__)
+
+
+class AuthManager:
+    """Manages credentials storage and auth header generation."""
+
+    def __init__(self, vault: Vault) -> None:
+        self.vault = vault
+
+    async def store_credentials(self, adapter_id: str, credentials: dict[str, Any]) -> str:
+        """Store credentials in vault with per-adapter isolation. Returns vault key prefix."""
+        prefix = f"liquid/{adapter_id}"
+        try:
+            for key, value in credentials.items():
+                await self.vault.store(f"{prefix}/{key}", str(value))
+        except Exception as e:
+            raise VaultError(f"Failed to store credentials for {adapter_id}: {e}") from e
+        return prefix
+
+    async def get_auth_headers(self, auth: AuthRequirement, vault_key: str) -> dict[str, str]:
+        """Build HTTP headers for the given auth type."""
+        try:
+            match auth.type:
+                case "bearer" | "oauth2":
+                    token = await self.vault.get(f"{vault_key}/access_token")
+                    return {"Authorization": f"Bearer {token}"}
+                case "api_key":
+                    key = await self.vault.get(f"{vault_key}/api_key")
+                    return {"X-API-Key": key}
+                case "basic":
+                    username = await self.vault.get(f"{vault_key}/username")
+                    password = await self.vault.get(f"{vault_key}/password")
+                    import base64
+
+                    encoded = base64.b64encode(f"{username}:{password}".encode()).decode()
+                    return {"Authorization": f"Basic {encoded}"}
+                case _:
+                    token = await self.vault.get(f"{vault_key}/token")
+                    return {"Authorization": f"Bearer {token}"}
+        except VaultError:
+            raise
+        except Exception as e:
+            raise AuthError(f"Failed to build auth headers: {e}") from e
+
+    async def refresh_oauth_token(
+        self,
+        vault_key: str,
+        oauth_config: OAuthConfig,
+        http_client: httpx.AsyncClient | None = None,
+    ) -> str:
+        """Refresh an OAuth2 access token using the stored refresh token."""
+        try:
+            refresh_token = await self.vault.get(f"{vault_key}/refresh_token")
+            client_id = await self.vault.get(f"{vault_key}/client_id")
+            client_secret = await self.vault.get(f"{vault_key}/client_secret")
+        except Exception as e:
+            raise AuthError(f"Missing OAuth credentials for refresh: {e}") from e
+
+        client = http_client or httpx.AsyncClient()
+        try:
+            resp = await client.post(
+                oauth_config.token_url,
+                data={
+                    "grant_type": "refresh_token",
+                    "refresh_token": refresh_token,
+                    "client_id": client_id,
+                    "client_secret": client_secret,
+                },
+            )
+
+            if not resp.is_success:
+                raise AuthError(f"Token refresh failed ({resp.status_code}): {resp.text[:200]}")
+
+            token_data = resp.json()
+            new_access_token = token_data["access_token"]
+            await self.vault.store(f"{vault_key}/access_token", new_access_token)
+
+            if "refresh_token" in token_data:
+                await self.vault.store(f"{vault_key}/refresh_token", token_data["refresh_token"])
+
+            logger.info("OAuth token refreshed for %s", vault_key)
+            return new_access_token
+        finally:
+            if not http_client:
+                await client.aclose()
+
+    async def delete_credentials(self, adapter_id: str, keys: list[str] | None = None) -> None:
+        """Remove stored credentials for an adapter."""
+        prefix = f"liquid/{adapter_id}"
+        key_names = keys or ["access_token", "refresh_token", "client_id", "client_secret", "api_key"]
+        import contextlib
+
+        for key in key_names:
+            with contextlib.suppress(Exception):
+                await self.vault.delete(f"{prefix}/{key}")

liquid/client.py

@@ -0,0 +1,213 @@
+"""Liquid — the main orchestrator tying all phases together."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import httpx
+
+from liquid.auth.classifier import AuthClassifier, EscalationInfo
+from liquid.auth.manager import AuthManager
+from liquid.discovery.base import DiscoveryPipeline
+from liquid.discovery.browser import BrowserDiscovery
+from liquid.discovery.diff import diff_schemas
+from liquid.discovery.graphql import GraphQLDiscovery
+from liquid.discovery.mcp import MCPDiscovery
+from liquid.discovery.openapi import OpenAPIDiscovery
+from liquid.discovery.rest_heuristic import RESTHeuristicDiscovery
+from liquid.mapping.learning import MappingLearner
+from liquid.mapping.proposer import MappingProposer
+from liquid.mapping.reviewer import MappingReview
+from liquid.models.adapter import AdapterConfig, FieldMapping, SyncConfig
+from liquid.models.schema import APISchema  # noqa: TC001
+from liquid.sync.engine import SyncEngine
+from liquid.sync.fetcher import Fetcher
+from liquid.sync.mapper import RecordMapper
+
+if TYPE_CHECKING:
+    from liquid.events import EventHandler
+    from liquid.models.sync import SyncResult
+    from liquid.protocols import DataSink, KnowledgeStore, LLMBackend, Vault
+    from liquid.sync.retry import RetryPolicy
+
+
+class Liquid:
+    """Main entry point for the Liquid library.
+
+    Orchestrates: discover → classify auth → propose mappings → sync.
+    """
+
+    def __init__(
+        self,
+        llm: LLMBackend,
+        vault: Vault,
+        sink: DataSink,
+        knowledge: KnowledgeStore | None = None,
+        event_handler: EventHandler | None = None,
+        http_client: httpx.AsyncClient | None = None,
+        retry_policy: RetryPolicy | None = None,
+    ) -> None:
+        self.llm = llm
+        self.vault = vault
+        self.sink = sink
+        self.knowledge = knowledge
+        self.event_handler = event_handler
+        self._http_client = http_client
+        self._retry_policy = retry_policy
+
+        self._auth_classifier = AuthClassifier()
+        self._auth_manager = AuthManager(vault)
+        self._mapping_proposer = MappingProposer(llm, knowledge)
+        self._mapping_learner = MappingLearner(knowledge)
+
+    async def discover(self, url: str) -> APISchema:
+        """Phase 1: Discover the API at the given URL."""
+        client = self._http_client or httpx.AsyncClient()
+        try:
+            pipeline = DiscoveryPipeline(
+                [
+                    MCPDiscovery(),
+                    OpenAPIDiscovery(http_client=client),
+                    GraphQLDiscovery(http_client=client),
+                    RESTHeuristicDiscovery(llm=self.llm, http_client=client),
+                    BrowserDiscovery(llm=self.llm),
+                ]
+            )
+            return await pipeline.discover(url)
+        finally:
+            if not self._http_client:
+                await client.aclose()
+
+    def classify_auth(self, schema: APISchema) -> EscalationInfo:
+        """Phase 2: Classify auth requirements and return escalation info."""
+        return self._auth_classifier.classify(schema.auth)
+
+    async def store_credentials(self, adapter_id: str, credentials: dict[str, Any]) -> str:
+        """Phase 2b: Store credentials after human provides them."""
+        return await self._auth_manager.store_credentials(adapter_id, credentials)
+
+    async def propose_mappings(
+        self,
+        schema: APISchema,
+        target_model: dict[str, Any],
+    ) -> MappingReview:
+        """Phase 3: AI proposes field mappings for human review."""
+        proposals = await self._mapping_proposer.propose(schema, target_model)
+        return MappingReview(proposals)
+
+    async def create_adapter(
+        self,
+        schema: APISchema,
+        auth_ref: str,
+        mappings: list[FieldMapping],
+        sync_config: SyncConfig,
+        verified_by: str | None = None,
+    ) -> AdapterConfig:
+        """Phase 3b: Create the final adapter config after human approval."""
+        from datetime import UTC, datetime
+
+        return AdapterConfig(
+            schema=schema,
+            auth_ref=auth_ref,
+            mappings=mappings,
+            sync=sync_config,
+            verified_by=verified_by,
+            verified_at=datetime.now(UTC) if verified_by else None,
+        )
+
+    async def sync(self, config: AdapterConfig, cursor: str | None = None) -> SyncResult:
+        """Phase 4: Run a deterministic sync cycle."""
+        client = self._http_client or httpx.AsyncClient()
+        try:
+            fetcher = Fetcher(http_client=client, vault=self.vault)
+            mapper = RecordMapper(config.mappings)
+            engine = SyncEngine(
+                fetcher=fetcher,
+                mapper=mapper,
+                sink=self.sink,
+                event_handler=self.event_handler,
+                retry_policy=self._retry_policy,
+            )
+            return await engine.run(config, cursor)
+        finally:
+            if not self._http_client:
+                await client.aclose()
+
+    async def repair_adapter(
+        self,
+        config: AdapterConfig,
+        target_model: dict[str, Any],
+        auto_approve: bool = False,
+        confidence_threshold: float = 0.8,
+    ) -> AdapterConfig | MappingReview:
+        """Re-discover API, diff schemas, selectively re-map broken fields.
+
+        Returns AdapterConfig if auto_approve=True and all mappings are confident,
+        otherwise returns MappingReview for human review.
+        """
+        from liquid.events import AdapterRepaired
+
+        new_schema = await self.discover(config.schema_.source_url)
+        diff = diff_schemas(config.schema_, new_schema)
+
+        if not diff.has_breaking_changes:
+            updated = config.model_copy(update={"schema_": new_schema, "version": config.version + 1})
+            if self.event_handler:
+                await self.event_handler.handle(
+                    AdapterRepaired(
+                        adapter_id=config.config_id,
+                        diff=diff,
+                        auto_approved=True,
+                    )
+                )
+            return updated
+
+        proposals = await self._mapping_proposer.propose(
+            new_schema,
+            target_model,
+            existing_mappings=config.mappings,
+            removed_fields=diff.removed_fields,
+        )
+
+        review = MappingReview(proposals)
+
+        if auto_approve and all(m.confidence >= confidence_threshold for m in proposals):
+            review.approve_all()
+            mappings = review.finalize()
+            updated = AdapterConfig(
+                config_id=config.config_id,
+                schema=new_schema,
+                auth_ref=config.auth_ref,
+                mappings=mappings,
+                sync=config.sync,
+                verified_by=config.verified_by,
+                version=config.version + 1,
+            )
+            if self.event_handler:
+                await self.event_handler.handle(
+                    AdapterRepaired(
+                        adapter_id=config.config_id,
+                        diff=diff,
+                        auto_approved=True,
+                    )
+                )
+            return updated
+
+        return review
+
+    async def learn_from_review(
+        self,
+        schema: APISchema,
+        target_model: dict[str, Any],
+        review: MappingReview,
+    ) -> None:
+        """Record corrections from a mapping review for future learning."""
+        import json
+
+        corrections = review.corrections()
+        if corrections:
+            await self._mapping_learner.record_corrections(
+                schema.service_name,
+                json.dumps(target_model),
+                corrections,
+            )

liquid/discovery/__init__.py

@@ -0,0 +1,18 @@
+from liquid.discovery.base import DiscoveryPipeline, DiscoveryStrategy
+from liquid.discovery.browser import BrowserDiscovery
+from liquid.discovery.diff import diff_schemas
+from liquid.discovery.graphql import GraphQLDiscovery
+from liquid.discovery.mcp import MCPDiscovery
+from liquid.discovery.openapi import OpenAPIDiscovery
+from liquid.discovery.rest_heuristic import RESTHeuristicDiscovery
+
+__all__ = [
+    "BrowserDiscovery",
+    "DiscoveryPipeline",
+    "DiscoveryStrategy",
+    "GraphQLDiscovery",
+    "MCPDiscovery",
+    "OpenAPIDiscovery",
+    "RESTHeuristicDiscovery",
+    "diff_schemas",
+]

liquid/discovery/base.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import logging
+from typing import Protocol, runtime_checkable
+
+from liquid.exceptions import DiscoveryError
+from liquid.models.schema import APISchema  # noqa: TC001
+
+logger = logging.getLogger(__name__)
+
+
+@runtime_checkable
+class DiscoveryStrategy(Protocol):
+    async def discover(self, url: str) -> APISchema | None:
+        """Try to discover the API at the given URL.
+
+        Returns APISchema on success, None if this strategy doesn't apply.
+        Raises DiscoveryError on unexpected failures.
+        """
+        ...
+
+
+class DiscoveryPipeline:
+    """Tries discovery strategies in order, returns first success."""
+
+    def __init__(self, strategies: list[DiscoveryStrategy]) -> None:
+        self.strategies = strategies
+
+    async def discover(self, url: str) -> APISchema:
+        errors: list[tuple[str, Exception]] = []
+
+        for strategy in self.strategies:
+            strategy_name = type(strategy).__name__
+            logger.info("Trying discovery strategy: %s for %s", strategy_name, url)
+            try:
+                result = await strategy.discover(url)
+                if result is not None:
+                    logger.info("Discovery succeeded with %s", strategy_name)
+                    return result
+                logger.debug("Strategy %s returned None, trying next", strategy_name)
+            except DiscoveryError as e:
+                logger.warning("Strategy %s failed: %s", strategy_name, e)
+                errors.append((strategy_name, e))
+            except Exception as e:
+                logger.warning("Strategy %s unexpected error: %s", strategy_name, e)
+                errors.append((strategy_name, e))
+
+        error_summary = "; ".join(f"{name}: {err}" for name, err in errors)
+        raise DiscoveryError(
+            f"All discovery strategies failed for {url}. Errors: {error_summary}"
+            if errors
+            else f"No discovery strategy could handle {url}"
+        )