liquid-api 0.2.0__py3-none-any.whl

liquid/mapping/reviewer.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from enum import StrEnum
+
+from liquid.models.adapter import FieldMapping
+
+
+class MappingStatus(StrEnum):
+    PENDING = "pending"
+    APPROVED = "approved"
+    REJECTED = "rejected"
+    CORRECTED = "corrected"
+
+
+class MappingReview:
+    """Manages the human review workflow for proposed field mappings."""
+
+    def __init__(self, proposed: list[FieldMapping]) -> None:
+        self._proposed = list(proposed)
+        self._statuses: list[MappingStatus] = [MappingStatus.PENDING] * len(proposed)
+        self._corrections: list[FieldMapping | None] = [None] * len(proposed)
+
+    @property
+    def proposed(self) -> list[FieldMapping]:
+        return list(self._proposed)
+
+    def __len__(self) -> int:
+        return len(self._proposed)
+
+    def approve(self, index: int) -> None:
+        self._check_index(index)
+        self._statuses[index] = MappingStatus.APPROVED
+
+    def reject(self, index: int) -> None:
+        self._check_index(index)
+        self._statuses[index] = MappingStatus.REJECTED
+
+    def correct(
+        self,
+        index: int,
+        target_field: str | None = None,
+        transform: str | None = None,
+        source_path: str | None = None,
+    ) -> None:
+        self._check_index(index)
+        original = self._proposed[index]
+        self._corrections[index] = FieldMapping(
+            source_path=source_path or original.source_path,
+            target_field=target_field or original.target_field,
+            transform=transform if transform is not None else original.transform,
+            confidence=1.0,
+        )
+        self._statuses[index] = MappingStatus.CORRECTED
+
+    def approve_all(self) -> None:
+        for i in range(len(self._statuses)):
+            if self._statuses[i] == MappingStatus.PENDING:
+                self._statuses[i] = MappingStatus.APPROVED
+
+    def finalize(self) -> list[FieldMapping]:
+        """Return only approved and corrected mappings."""
+        result: list[FieldMapping] = []
+        for i, status in enumerate(self._statuses):
+            if status == MappingStatus.APPROVED:
+                result.append(self._proposed[i])
+            elif status == MappingStatus.CORRECTED and self._corrections[i]:
+                result.append(self._corrections[i])
+        return result
+
+    def corrections(self) -> list[tuple[FieldMapping, FieldMapping]]:
+        """Return (original, corrected) pairs for learning."""
+        pairs: list[tuple[FieldMapping, FieldMapping]] = []
+        for i, status in enumerate(self._statuses):
+            if status == MappingStatus.CORRECTED and self._corrections[i]:
+                pairs.append((self._proposed[i], self._corrections[i]))
+        return pairs
+
+    def status(self, index: int) -> MappingStatus:
+        self._check_index(index)
+        return self._statuses[index]
+
+    def _check_index(self, index: int) -> None:
+        if index < 0 or index >= len(self._proposed):
+            raise IndexError(f"Mapping index {index} out of range (0-{len(self._proposed) - 1})")

liquid/models/__init__.py

@@ -0,0 +1,36 @@
+from liquid.models.adapter import AdapterConfig, FieldMapping, SyncConfig
+from liquid.models.llm import DeliveryResult, LLMResponse, MappedRecord, Message, Tool, ToolCall
+from liquid.models.schema import (
+    APISchema,
+    AuthRequirement,
+    Endpoint,
+    OAuthConfig,
+    PaginationType,
+    Parameter,
+    ParameterLocation,
+    RateLimits,
+)
+from liquid.models.sync import SyncError, SyncErrorType, SyncResult
+
+__all__ = [
+    "APISchema",
+    "AdapterConfig",
+    "AuthRequirement",
+    "DeliveryResult",
+    "Endpoint",
+    "FieldMapping",
+    "LLMResponse",
+    "MappedRecord",
+    "Message",
+    "OAuthConfig",
+    "PaginationType",
+    "Parameter",
+    "ParameterLocation",
+    "RateLimits",
+    "SyncConfig",
+    "SyncError",
+    "SyncErrorType",
+    "SyncResult",
+    "Tool",
+    "ToolCall",
+]

liquid/models/adapter.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from datetime import datetime  # noqa: TC003
+from uuid import uuid4
+
+from pydantic import BaseModel, Field
+
+from liquid.models.schema import APISchema  # noqa: TC001
+
+
+class FieldMapping(BaseModel):
+    source_path: str
+    target_field: str
+    transform: str | None = None
+    confidence: float = Field(default=1.0, ge=0.0, le=1.0)
+
+
+class SyncConfig(BaseModel):
+    endpoints: list[str]
+    schedule: str = "0 */6 * * *"
+    cursor_field: str | None = None
+    batch_size: int = 100
+
+
+class AdapterConfig(BaseModel):
+    config_id: str = Field(default_factory=lambda: uuid4().hex)
+    schema_: APISchema = Field(alias="schema")
+    auth_ref: str
+    mappings: list[FieldMapping]
+    sync: SyncConfig
+    verified_by: str | None = None
+    verified_at: datetime | None = None
+    version: int = 1
+
+    model_config = {"populate_by_name": True}

liquid/models/llm.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class ToolCall(BaseModel):
+    id: str
+    name: str
+    arguments: dict[str, Any] = Field(default_factory=dict)
+
+
+class Message(BaseModel):
+    role: Literal["system", "user", "assistant", "tool"]
+    content: str
+    tool_call_id: str | None = None
+    tool_calls: list[ToolCall] | None = None
+
+
+class Tool(BaseModel):
+    name: str
+    description: str
+    parameters: dict[str, Any] = Field(default_factory=dict)
+
+
+class LLMResponse(BaseModel):
+    content: str | None = None
+    tool_calls: list[ToolCall] | None = None
+
+
+class MappedRecord(BaseModel):
+    source_endpoint: str
+    source_data: dict[str, Any]
+    mapped_data: dict[str, Any]
+    mapping_errors: list[str] | None = None
+
+
+class DeliveryResult(BaseModel):
+    delivered: int = 0
+    failed: int = 0
+    errors: list[str] | None = None

liquid/models/schema.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class PaginationType(StrEnum):
+    CURSOR = "cursor"
+    OFFSET = "offset"
+    PAGE_NUMBER = "page_number"
+    LINK_HEADER = "link_header"
+    NONE = "none"
+
+
+class ParameterLocation(StrEnum):
+    QUERY = "query"
+    PATH = "path"
+    HEADER = "header"
+    BODY = "body"
+
+
+class Parameter(BaseModel):
+    name: str
+    location: ParameterLocation
+    required: bool = False
+    schema_: dict[str, Any] | None = Field(default=None, alias="schema")
+    description: str | None = None
+
+    model_config = {"populate_by_name": True}
+
+
+class OAuthConfig(BaseModel):
+    authorize_url: str
+    token_url: str
+    scopes: list[str] = Field(default_factory=list)
+    client_registration_url: str | None = None
+
+
+class RateLimits(BaseModel):
+    requests_per_second: float | None = None
+    requests_per_minute: float | None = None
+    burst: int | None = None
+    retry_after_header: str | None = None
+
+
+class Endpoint(BaseModel):
+    path: str
+    method: str = "GET"
+    description: str = ""
+    parameters: list[Parameter] = Field(default_factory=list)
+    response_schema: dict[str, Any] = Field(default_factory=dict)
+    pagination: PaginationType | None = None
+
+
+class AuthRequirement(BaseModel):
+    type: Literal["oauth2", "api_key", "basic", "bearer", "custom"]
+    tier: Literal["A", "B", "C"]
+    oauth_config: OAuthConfig | None = None
+    docs_url: str | None = None
+
+
+class APISchema(BaseModel):
+    source_url: str
+    service_name: str
+    discovery_method: Literal["mcp", "openapi", "graphql", "rest_heuristic", "browser"]
+    endpoints: list[Endpoint] = Field(default_factory=list)
+    auth: AuthRequirement
+    rate_limits: RateLimits | None = None
+    discovered_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+
+class SchemaDiff(BaseModel):
+    """Structured diff between two APISchema versions."""
+
+    added_endpoints: list[Endpoint] = Field(default_factory=list)
+    removed_endpoints: list[Endpoint] = Field(default_factory=list)
+    unchanged_endpoints: list[Endpoint] = Field(default_factory=list)
+    added_fields: list[str] = Field(default_factory=list)
+    removed_fields: list[str] = Field(default_factory=list)
+    unchanged_fields: list[str] = Field(default_factory=list)
+    has_breaking_changes: bool = False

liquid/models/sync.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from datetime import datetime  # noqa: TC003
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class SyncErrorType(StrEnum):
+    FIELD_NOT_FOUND = "field_not_found"
+    AUTH_ERROR = "auth_error"
+    RATE_LIMIT = "rate_limit"
+    SERVICE_DOWN = "service_down"
+    ENDPOINT_GONE = "endpoint_gone"
+    TRANSFORM_ERROR = "transform_error"
+    DELIVERY_ERROR = "delivery_error"
+
+
+class SyncError(BaseModel):
+    type: SyncErrorType
+    message: str
+    endpoint: str | None = None
+    details: dict[str, Any] | None = None
+
+
+class SyncResult(BaseModel):
+    adapter_id: str
+    started_at: datetime
+    finished_at: datetime
+    records_fetched: int = 0
+    records_mapped: int = 0
+    records_delivered: int = 0
+    errors: list[SyncError] = Field(default_factory=list)
+    next_cursor: str | None = None

liquid/protocols.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+    from liquid.models import DeliveryResult, FieldMapping, LLMResponse, MappedRecord, Message, Tool
+
+
+@runtime_checkable
+class Vault(Protocol):
+    async def store(self, key: str, value: str) -> None: ...
+    async def get(self, key: str) -> str: ...
+    async def delete(self, key: str) -> None: ...
+
+
+@runtime_checkable
+class LLMBackend(Protocol):
+    async def chat(self, messages: list[Message], tools: list[Tool] | None = None) -> LLMResponse: ...
+
+
+@runtime_checkable
+class DataSink(Protocol):
+    async def deliver(self, records: list[MappedRecord]) -> DeliveryResult: ...
+
+
+@runtime_checkable
+class KnowledgeStore(Protocol):
+    async def find_mapping(self, service: str, target_model: str) -> list[FieldMapping] | None: ...
+    async def store_mapping(self, service: str, target_model: str, mappings: list[FieldMapping]) -> None: ...

liquid/sync/__init__.py

@@ -0,0 +1,29 @@
+from liquid.sync.auto_repair import AutoRepairHandler
+from liquid.sync.engine import SyncEngine
+from liquid.sync.fetcher import Fetcher
+from liquid.sync.mapper import RecordMapper
+from liquid.sync.pagination import (
+    CursorPagination,
+    LinkHeaderPagination,
+    NoPagination,
+    OffsetPagination,
+    PageNumberPagination,
+    PaginationStrategy,
+)
+from liquid.sync.retry import RetryPolicy
+from liquid.sync.selector import RecordSelector
+
+__all__ = [
+    "AutoRepairHandler",
+    "CursorPagination",
+    "Fetcher",
+    "LinkHeaderPagination",
+    "NoPagination",
+    "OffsetPagination",
+    "PageNumberPagination",
+    "PaginationStrategy",
+    "RecordMapper",
+    "RecordSelector",
+    "RetryPolicy",
+    "SyncEngine",
+]

liquid/sync/auto_repair.py

@@ -0,0 +1,64 @@
+"""Auto-repair handler that triggers adapter repair on persistent failures."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Awaitable, Callable  # noqa: TC003
+from typing import TYPE_CHECKING, Any
+
+from liquid.events import Event, ReDiscoveryNeeded
+
+if TYPE_CHECKING:
+    from liquid.client import Liquid
+    from liquid.mapping.reviewer import MappingReview
+    from liquid.models.adapter import AdapterConfig
+
+logger = logging.getLogger(__name__)
+
+
+class AutoRepairHandler:
+    """Event handler that automatically repairs adapters on ReDiscoveryNeeded.
+
+    Usage:
+        handler = AutoRepairHandler(
+            liquid=liquid_client,
+            target_model={"amount": "float", "date": "datetime"},
+            config_provider=lambda: current_config,
+            on_repair=my_repair_callback,
+        )
+    """
+
+    def __init__(
+        self,
+        liquid: Liquid,
+        target_model: dict[str, Any],
+        config_provider: Callable[[], AdapterConfig],
+        on_repair: Callable[[AdapterConfig | MappingReview], Awaitable[None]],
+        auto_approve: bool = False,
+        confidence_threshold: float = 0.8,
+    ) -> None:
+        self.liquid = liquid
+        self.target_model = target_model
+        self.config_provider = config_provider
+        self.on_repair = on_repair
+        self.auto_approve = auto_approve
+        self.confidence_threshold = confidence_threshold
+
+    async def handle(self, event: Event) -> None:
+        if not isinstance(event, ReDiscoveryNeeded):
+            return
+
+        config = self.config_provider()
+        logger.info("Auto-repair triggered for adapter %s: %s", config.config_id, event.reason)
+
+        try:
+            result = await self.liquid.repair_adapter(
+                config=config,
+                target_model=self.target_model,
+                auto_approve=self.auto_approve,
+                confidence_threshold=self.confidence_threshold,
+            )
+            await self.on_repair(result)
+            logger.info("Auto-repair completed for adapter %s", config.config_id)
+        except Exception:
+            logger.exception("Auto-repair failed for adapter %s", config.config_id)

liquid/sync/engine.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+from liquid.events import ReDiscoveryNeeded, SyncCompleted, SyncFailed
+from liquid.exceptions import SyncRuntimeError
+from liquid.models import SyncResult
+from liquid.models.sync import SyncError, SyncErrorType
+from liquid.sync.fetcher import Fetcher  # noqa: TC001
+from liquid.sync.mapper import RecordMapper  # noqa: TC001
+from liquid.sync.retry import RetryPolicy, with_retry
+
+if TYPE_CHECKING:
+    from liquid.events import EventHandler
+    from liquid.models.adapter import AdapterConfig
+    from liquid.models.schema import Endpoint
+    from liquid.protocols import DataSink
+
+logger = logging.getLogger(__name__)
+
+
+class SyncEngine:
+    def __init__(
+        self,
+        fetcher: Fetcher,
+        mapper: RecordMapper,
+        sink: DataSink,
+        event_handler: EventHandler | None = None,
+        retry_policy: RetryPolicy | None = None,
+        failure_threshold: int = 5,
+    ) -> None:
+        self.fetcher = fetcher
+        self.mapper = mapper
+        self.sink = sink
+        self.event_handler = event_handler
+        self.retry_policy = retry_policy or RetryPolicy()
+        self.failure_threshold = failure_threshold
+        self._consecutive_failures = 0
+
+    async def run(self, config: AdapterConfig, cursor: str | None = None) -> SyncResult:
+        started_at = datetime.now(UTC)
+        total_fetched = 0
+        total_mapped = 0
+        total_delivered = 0
+        errors: list[SyncError] = []
+        current_cursor = cursor
+
+        for ep_path in config.sync.endpoints:
+            endpoint = self._find_endpoint(config, ep_path)
+            if endpoint is None:
+                errors.append(
+                    SyncError(
+                        type=SyncErrorType.ENDPOINT_GONE,
+                        message=f"Endpoint {ep_path} not found in schema",
+                        endpoint=ep_path,
+                    )
+                )
+                continue
+
+            try:
+                page_cursor = current_cursor
+                while True:
+                    fetch_result = await with_retry(
+                        lambda ep=endpoint, c=page_cursor: self.fetcher.fetch(
+                            endpoint=ep,
+                            base_url=config.schema_.source_url,
+                            auth_ref=config.auth_ref,
+                            cursor=c,
+                        ),
+                        self.retry_policy,
+                    )
+
+                    total_fetched += len(fetch_result.records)
+
+                    mapped_records = self.mapper.map_batch(fetch_result.records, ep_path)
+                    total_mapped += len(mapped_records)
+
+                    delivery = await self.sink.deliver(mapped_records)
+                    total_delivered += delivery.delivered
+
+                    if delivery.errors:
+                        for err_msg in delivery.errors:
+                            errors.append(
+                                SyncError(
+                                    type=SyncErrorType.DELIVERY_ERROR,
+                                    message=err_msg,
+                                    endpoint=ep_path,
+                                )
+                            )
+
+                    page_cursor = fetch_result.next_cursor
+                    current_cursor = page_cursor
+                    if page_cursor is None:
+                        break
+
+            except SyncRuntimeError as e:
+                error_type = _classify_error(e)
+                errors.append(
+                    SyncError(
+                        type=error_type,
+                        message=str(e),
+                        endpoint=ep_path,
+                    )
+                )
+
+        result = SyncResult(
+            adapter_id=config.config_id,
+            started_at=started_at,
+            finished_at=datetime.now(UTC),
+            records_fetched=total_fetched,
+            records_mapped=total_mapped,
+            records_delivered=total_delivered,
+            errors=errors,
+            next_cursor=current_cursor,
+        )
+
+        await self._handle_result(result, config)
+        return result
+
+    async def _handle_result(self, result: SyncResult, config: AdapterConfig) -> None:
+        if not self.event_handler:
+            return
+
+        if result.errors:
+            self._consecutive_failures += 1
+            await self.event_handler.handle(
+                SyncFailed(
+                    adapter_id=config.config_id,
+                    error=result.errors[-1],
+                    consecutive_failures=self._consecutive_failures,
+                )
+            )
+            if self._consecutive_failures >= self.failure_threshold:
+                failures = self._consecutive_failures
+                threshold = self.failure_threshold
+                await self.event_handler.handle(
+                    ReDiscoveryNeeded(
+                        adapter_id=config.config_id,
+                        reason=f"Consecutive failures ({failures}) exceeded threshold ({threshold})",
+                    )
+                )
+        else:
+            self._consecutive_failures = 0
+            await self.event_handler.handle(
+                SyncCompleted(
+                    adapter_id=config.config_id,
+                    result=result,
+                )
+            )
+
+    @staticmethod
+    def _find_endpoint(config: AdapterConfig, path: str) -> Endpoint | None:
+        for ep in config.schema_.endpoints:
+            if ep.path == path:
+                return ep
+        return None
+
+
+def _classify_error(exc: SyncRuntimeError) -> SyncErrorType:
+    from liquid.exceptions import AuthError, EndpointGoneError, FieldNotFoundError, RateLimitError, ServiceDownError
+
+    match exc:
+        case FieldNotFoundError():
+            return SyncErrorType.FIELD_NOT_FOUND
+        case AuthError():
+            return SyncErrorType.AUTH_ERROR
+        case RateLimitError():
+            return SyncErrorType.RATE_LIMIT
+        case ServiceDownError():
+            return SyncErrorType.SERVICE_DOWN
+        case EndpointGoneError():
+            return SyncErrorType.ENDPOINT_GONE
+        case _:
+            return SyncErrorType.FIELD_NOT_FOUND