offline-sync-sdk 0.2.0__py3-none-any.whl

sync_sdk/adapters/sqlalchemy_config.py

@@ -0,0 +1,145 @@
+"""Application configuration for SQL schemas and their sync registrations."""
+
+from collections.abc import Callable, Mapping
+from typing import Any, cast
+
+from pydantic import BaseModel, ConfigDict, create_model
+from sqlalchemy import JSON, Connection, MetaData, Table
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession
+
+from ..gateway import SyncGateway
+from ..registry import ConflictPolicy, Entity, EntityRegistry, optimistic_version
+from ..telemetry import PipelineTelemetry
+from .sqlalchemy import (
+    SQLAlchemyChangeCapture,
+    SQLAlchemyStore,
+    TableBinding,
+    initialize_sync_metadata,
+    sync_metadata,
+)
+
+
+class SQLAlchemyConfig:
+    """Declare participating tables once; create them only when explicitly asked.
+
+    Native SQLAlchemy tables retain application-defined columns, indexes, foreign
+    keys, constraints, and defaults. Registration and build() perform no I/O.
+    """
+
+    def __init__(self) -> None:
+        self._bindings: dict[str, TableBinding] = {}
+        self._entities: dict[str, Entity] = {}
+
+    def register_table(
+        self,
+        name: str,
+        table: Table,
+        *,
+        fields: Mapping[str, str] | None = None,
+        schema: type[BaseModel] | None = None,
+        key_parsers: Mapping[str, Callable[[str], Any]] | None = None,
+        value_decoders: Mapping[str, Callable[[Any], Any]] | None = None,
+        conflict_policy: ConflictPolicy = optimistic_version,
+    ) -> TableBinding:
+        """Register a new or existing table and return its key encoder.
+
+        Omit fields to sync every non-primary-key column, or explicitly select a
+        subset. schema is application-owned validation (including defaults).
+        """
+        if not name or len(name) > 191 or name in self._bindings:
+            raise ValueError(
+                "Entity names must be unique and contain 1 to 191 characters"
+            )
+        if table.name in sync_metadata.tables:
+            raise ValueError(
+                "Sync metadata tables cannot be registered as business data"
+            )
+        if any(
+            binding.table.fullname == table.fullname
+            for binding in self._bindings.values()
+        ):
+            raise ValueError(
+                "Register a physical table once to preserve one version history per row"
+            )
+        if fields is None:
+            fields = {
+                column.name: column.name
+                for column in table.columns
+                if not column.primary_key
+            }
+        binding = TableBinding(
+            table, fields=fields, key_parsers=key_parsers, value_decoders=value_decoders
+        )
+        if schema is None:
+            schema = self._payload_schema(name, table, fields)
+        self._bindings[name] = binding
+        self._entities[name] = Entity(name, schema, conflict_policy)
+        return binding
+
+    @staticmethod
+    def _payload_schema(
+        name: str, table: Table, fields: Mapping[str, str]
+    ) -> type[BaseModel]:
+        definitions: dict[str, Any] = {}
+        for field, column_name in fields.items():
+            column = table.c[column_name]
+            annotation: Any = (
+                Any if isinstance(column.type, JSON) else column.type.python_type
+            )
+            default: Any = ...
+            if column.nullable:
+                annotation = annotation | None
+            if column.default is not None and column.default.is_scalar:
+                default = cast(Any, column.default).arg
+            definitions[field] = (annotation, default)
+        return create_model(
+            f"{name}Payload", __config__=ConfigDict(extra="forbid"), **definitions
+        )
+
+    def build(
+        self,
+        session_factory: Callable[[], AsyncSession],
+        *,
+        telemetry: PipelineTelemetry | None = None,
+    ) -> SyncGateway:
+        """Snapshot this configuration into an embeddable gateway."""
+        return SyncGateway(
+            SQLAlchemyStore(session_factory, self._bindings, telemetry=telemetry),
+            EntityRegistry(self._entities.values()),
+            telemetry=telemetry,
+        )
+
+    def build_change_capture(
+        self, session_factory: Callable[[], AsyncSession]
+    ) -> SQLAlchemyChangeCapture:
+        """Build a polling worker for writes made outside the sync gateway."""
+        normalizers = {
+            name: (
+                lambda payload, schema=entity.schema: schema.model_validate(
+                    payload
+                ).model_dump(mode="json")
+            )
+            for name, entity in self._entities.items()
+            if entity.schema is not None
+        }
+        return SQLAlchemyChangeCapture(session_factory, self._bindings, normalizers)
+
+    async def create_tables(self, engine: AsyncEngine) -> None:
+        """Explicit setup: create registered business tables and sync metadata.
+
+        Existing tables are left intact. This is not schema migration, ALTER,
+        backfill, or change capture. Use application migrations for schema changes.
+        Foreign-key prerequisites must be registered or already exist. Run setup
+        once before serving traffic, rather than from concurrent workers.
+        """
+        tables = [binding.table for binding in self._bindings.values()]
+        async with engine.begin() as connection:
+            await connection.run_sync(self._create_tables, tables)
+        await initialize_sync_metadata(engine)
+
+    @staticmethod
+    def _create_tables(connection: Connection, tables: list[Table]) -> None:
+        # Native create_all handles dependency ordering and deferred ALTER FKs.
+        # Its explicit tables argument prevents creation of unregistered tables.
+        metadata = tables[0].metadata if tables else MetaData()
+        metadata.create_all(connection, tables=tables, checkfirst=True)

sync_sdk/client.py

@@ -0,0 +1,198 @@
+"""Bounded sync cycles and an optional reconnect loop."""
+
+import asyncio
+import random
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Literal
+from uuid import uuid4
+
+from .models import Mutation
+from .ports import LocalStore, RetryableError, SyncTransport
+from .telemetry import PipelineTelemetry, safe_telemetry
+
+if TYPE_CHECKING:
+    import httpx
+
+
+@dataclass(frozen=True)
+class SyncReport:
+    pushed: int = 0
+    pulled: int = 0
+    retry_at: float | None = None
+
+
+class SyncClient:
+    def __init__(
+        self,
+        store: LocalStore,
+        http: "httpx.AsyncClient | None" = None,
+        *,
+        transport: SyncTransport | None = None,
+        batch_size: int = 100,
+        max_batches: int = 10,
+        backoff_base: float = 1,
+        backoff_cap: float = 60,
+        clock: Callable[[], float] = time.time,
+        jitter: Callable[[], float] = random.random,
+        telemetry: PipelineTelemetry | None = None,
+    ):
+        if not 1 <= batch_size <= 1000 or max_batches < 1:
+            raise ValueError("Invalid batch limits")
+        if backoff_base <= 0 or backoff_cap < backoff_base:
+            raise ValueError("Invalid backoff limits")
+        if (http is None) == (transport is None):
+            raise ValueError("Provide exactly one of transport or http")
+        if http is not None:
+            from .adapters.http import HTTPTransport
+
+            transport = HTTPTransport(http, clock=clock)
+        assert transport is not None
+        self.store, self.transport = store, transport
+        self.batch_size, self.max_batches = batch_size, max_batches
+        self.backoff_base, self.backoff_cap = backoff_base, backoff_cap
+        self.clock, self.jitter = clock, jitter
+        self.telemetry = safe_telemetry(telemetry)
+        self._lock = asyncio.Lock()
+
+    def enqueue(
+        self,
+        *,
+        entity: str,
+        entity_id: str,
+        operation: Literal["create", "update", "delete"],
+        payload: dict | None = None,
+        base_version: int | None = None,
+    ) -> Mutation:
+        mutation = Mutation(
+            mutation_id=uuid4(),
+            client_id=self.store.client_id,
+            entity=entity,
+            entity_id=entity_id,
+            operation=operation,
+            payload=payload or {},
+            base_version=base_version,
+            created_at=datetime.now(UTC),
+        )
+        self.store.enqueue(mutation)
+        self.telemetry.gauge("sync.client.queue.depth", self.store.pending_count())
+        return mutation
+
+    async def sync_once(self) -> SyncReport:
+        """Push and pull bounded batches; save retry time after connection failures."""
+        async with self._lock:
+            started = time.perf_counter()
+            queue_depth = self.store.pending_count()
+            self.telemetry.gauge("sync.client.queue.depth", queue_depth)
+            with self.telemetry.span(
+                "sync.client.sync", {"sync.queue.depth": queue_depth}
+            ) as sync_span:
+                attempts, retry_at = self.store.retry_state
+                pushed = pulled = 0
+                outcome = "ok"
+                try:
+                    if retry_at > self.clock():
+                        outcome = "waiting"
+                        sync_span.set_attribute("sync.retry.at", retry_at)
+                        return SyncReport(retry_at=retry_at)
+
+                    for _ in range(self.max_batches):
+                        batch = self.store.pending(self.batch_size)
+                        if not batch:
+                            break
+                        with self.telemetry.span(
+                            "sync.client.push", {"sync.batch.size": len(batch)}
+                        ):
+                            results = await self.transport.push(batch)
+                        if len(results) != len(batch) or {
+                            result.mutation_id for result in results
+                        } != {mutation.mutation_id for mutation in batch}:
+                            raise ValueError(
+                                "Mutation acknowledgements do not match submitted batch"
+                            )
+                        self.store.acknowledge(results)
+                        pushed += len(results)
+                        for result in results:
+                            self.telemetry.count(
+                                "sync.client.mutations",
+                                attributes={"sync.mutation.status": result.status},
+                            )
+
+                    for _ in range(self.max_batches):
+                        with self.telemetry.span(
+                            "sync.client.pull", {"sync.cursor": self.store.cursor}
+                        ):
+                            page = await self.transport.pull(
+                                self.store.cursor, self.batch_size
+                            )
+                        self.store.apply_page(page)
+                        pulled += len(page.changes)
+                        self.telemetry.count(
+                            "sync.client.changes.pulled", len(page.changes)
+                        )
+                        if not page.has_more:
+                            break
+                except RetryableError as error:
+                    outcome = "retry"
+                    ceiling = min(
+                        self.backoff_cap, self.backoff_base * 2 ** min(attempts, 30)
+                    )
+                    delay = max(
+                        error.retry_after, ceiling * (0.5 + 0.5 * self.jitter())
+                    )
+                    retry_at = self.clock() + delay
+                    self.store.schedule_retry(attempts + 1, retry_at)
+                    failure_attributes = {"sync.failure.type": error.reason}
+                    self.telemetry.count(
+                        "sync.client.transport.failures",
+                        attributes=failure_attributes,
+                    )
+                    if error.reason == "connection":
+                        self.telemetry.count("sync.client.connection.failures")
+                    self.telemetry.count(
+                        "sync.client.retries", attributes=failure_attributes
+                    )
+                    self.telemetry.record("sync.client.retry.delay", delay)
+                    sync_span.add_event(
+                        "sync.retry.scheduled",
+                        {
+                            "sync.retry.attempt": attempts + 1,
+                            "sync.retry.delay": delay,
+                            "sync.failure.type": error.reason,
+                        },
+                    )
+                    return SyncReport(pushed, pulled, retry_at)
+                except Exception:
+                    outcome = "error"
+                    self.telemetry.count("sync.client.failures")
+                    raise
+                else:
+                    self.store.schedule_retry(0, 0)
+                    return SyncReport(pushed, pulled)
+                finally:
+                    depth = self.store.pending_count()
+                    self.telemetry.gauge("sync.client.queue.depth", depth)
+                    self.telemetry.record(
+                        "sync.client.sync.duration",
+                        (time.perf_counter() - started) * 1000,
+                        {"sync.outcome": outcome},
+                    )
+                    sync_span.set_attribute("sync.outcome", outcome)
+                    sync_span.set_attribute("sync.mutations.pushed", pushed)
+                    sync_span.set_attribute("sync.changes.pulled", pulled)
+                    sync_span.set_attribute("sync.queue.depth", depth)
+
+    async def run(self, *, poll_interval: float = 1) -> None:
+        """Reconnect until cancelled; the caller owns HTTP and SQLite lifetimes."""
+        if poll_interval <= 0:
+            raise ValueError("poll_interval must be positive")
+        while True:
+            report = await self.sync_once()
+            delay = (
+                max(0, report.retry_at - self.clock())
+                if report.retry_at is not None
+                else poll_interval
+            )
+            await asyncio.sleep(delay)

sync_sdk/gateway.py

@@ -0,0 +1,163 @@
+"""Reusable mutation pipeline with no database or web framework imports."""
+
+import time
+
+from pydantic import ValidationError
+
+from .models import ChangeBatch, Mutation, MutationResult
+from .ports import GatewayStore, GatewayTransaction, MutationRejected
+from .registry import EntityRegistry
+from .telemetry import PipelineTelemetry, safe_telemetry
+
+
+class SyncGateway:
+    def __init__(
+        self,
+        store: GatewayStore,
+        entities: EntityRegistry,
+        *,
+        telemetry: PipelineTelemetry | None = None,
+    ):
+        self.store, self.entities = store, entities
+        self.telemetry = safe_telemetry(telemetry)
+
+    async def apply_mutations(self, mutations: list[Mutation]) -> list[MutationResult]:
+        started = time.perf_counter()
+        results: list[MutationResult] = []
+        outcome = "ok"
+        with self.telemetry.span(
+            "sync.gateway.apply", {"sync.batch.size": len(mutations)}
+        ) as span:
+            try:
+                async with self.store.transaction() as transaction:
+                    for mutation in mutations:
+                        result, is_new = await self._apply_mutation(
+                            transaction, mutation
+                        )
+                        if is_new:
+                            await transaction.save_result(mutation, result)
+                        results.append(result)
+            except Exception:
+                outcome = "error"
+                self.telemetry.count("sync.gateway.batch.failures")
+                raise
+            finally:
+                self.telemetry.record(
+                    "sync.gateway.batch.duration",
+                    (time.perf_counter() - started) * 1000,
+                    {"sync.outcome": outcome},
+                )
+                span.set_attribute("sync.outcome", outcome)
+
+            for mutation, result in zip(mutations, results, strict=True):
+                attributes = {
+                    "sync.entity": mutation.entity,
+                    "sync.operation": mutation.operation,
+                    "sync.mutation.status": result.status,
+                }
+                self.telemetry.count("sync.gateway.mutations", attributes=attributes)
+                span.add_event("sync.mutation.result", attributes)
+                if result.status == "conflict":
+                    self.telemetry.count(
+                        "sync.gateway.conflicts",
+                        attributes={"sync.entity": mutation.entity},
+                    )
+                elif result.status == "rejected":
+                    self.telemetry.count(
+                        "sync.gateway.failed_mutations",
+                        attributes={
+                            "sync.entity": mutation.entity,
+                            "sync.mutation.status": result.status,
+                        },
+                    )
+            return results
+
+    async def _apply_mutation(
+        self, transaction: GatewayTransaction, mutation: Mutation
+    ) -> tuple[MutationResult, bool]:
+        existing = await transaction.get_result(mutation.mutation_id)
+        if existing is not None:
+            return existing, False
+
+        entity = self.entities.get(mutation.entity)
+        rejection = None
+        if entity is None:
+            rejection = "Entity is not registered"
+        else:
+            try:
+                mutation = entity.validate(mutation)
+            except ValidationError:
+                rejection = "Payload failed entity validation"
+
+        if rejection:
+            return (
+                MutationResult(
+                    mutation_id=mutation.mutation_id,
+                    status="rejected",
+                    message=rejection,
+                ),
+                True,
+            )
+
+        assert entity is not None
+        version = await transaction.get_version(mutation.entity, mutation.entity_id)
+        conflict = entity.conflict_policy(mutation, version)
+        if conflict:
+            return (
+                MutationResult(
+                    mutation_id=mutation.mutation_id,
+                    status="conflict",
+                    server_version=version,
+                    message=conflict,
+                ),
+                True,
+            )
+
+        version += 1
+        try:
+            await transaction.write(mutation, version)
+        except MutationRejected as error:
+            return (
+                MutationResult(
+                    mutation_id=mutation.mutation_id,
+                    status="rejected",
+                    message=str(error),
+                ),
+                True,
+            )
+
+        await transaction.append_change(mutation, version)
+        return (
+            MutationResult(
+                mutation_id=mutation.mutation_id,
+                status="applied",
+                server_version=version,
+            ),
+            True,
+        )
+
+    async def get_changes(self, cursor: int, limit: int = 100) -> ChangeBatch:
+        if cursor < 0 or not 1 <= limit <= 1000:
+            raise ValueError("Invalid cursor or page limit")
+
+        started = time.perf_counter()
+        with self.telemetry.span(
+            "sync.gateway.pull", {"sync.cursor": cursor, "sync.page.limit": limit}
+        ) as span:
+            try:
+                page = await self.store.get_changes(cursor, limit)
+            except Exception:
+                self.telemetry.count("sync.gateway.pull.failures")
+                raise
+            finally:
+                self.telemetry.record(
+                    "sync.gateway.pull.duration",
+                    (time.perf_counter() - started) * 1000,
+                )
+            span.set_attribute("sync.changes.returned", len(page.changes))
+            span.set_attribute("sync.has_more", page.has_more)
+            self.telemetry.count("sync.gateway.changes.returned", len(page.changes))
+            return page
+
+    async def health_check(self) -> bool:
+        return await self.store.health_check()

sync_sdk/integrations/__init__.py

@@ -0,0 +1 @@
+"""Optional application framework integrations."""

sync_sdk/integrations/fastapi.py

@@ -0,0 +1,32 @@
+"""Mount sync routes in a developer's existing FastAPI application."""
+
+from collections.abc import Callable
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Query
+
+from ..models import ChangeBatch, Mutation, MutationResult
+from ..ports import RemoteStore
+
+
+def create_sync_router(
+    get_gateway: Callable[..., RemoteStore], *, prefix: str = "/v1/sync"
+) -> APIRouter:
+    """Dependency providers may apply application authentication/tenant scoping."""
+    router = APIRouter(prefix=prefix, tags=["sync"])
+
+    @router.post("/mutations", response_model=list[MutationResult])
+    async def push(
+        mutations: list[Mutation], gateway: Annotated[RemoteStore, Depends(get_gateway)]
+    ):
+        return await gateway.apply_mutations(mutations)
+
+    @router.get("/changes", response_model=ChangeBatch)
+    async def pull(
+        gateway: Annotated[RemoteStore, Depends(get_gateway)],
+        cursor: int = Query(0, ge=0),
+        limit: int = Query(100, ge=1, le=1000),
+    ):
+        return await gateway.get_changes(cursor, limit)
+
+    return router

sync_sdk/models.py

@@ -0,0 +1,41 @@
+"""Wire contracts shared by the SDK and gateway."""
+
+from datetime import datetime
+from typing import Any, Literal
+from uuid import UUID
+
+from pydantic import BaseModel, Field
+
+
+class Mutation(BaseModel):
+    mutation_id: UUID
+    client_id: str
+    entity: str
+    entity_id: str
+    operation: Literal["create", "update", "delete"]
+    payload: dict[str, Any] = Field(default_factory=dict)
+    base_version: int | None = None
+    created_at: datetime
+
+
+class MutationResult(BaseModel):
+    mutation_id: UUID
+    status: Literal["applied", "conflict", "rejected"]
+    server_version: int | None = None
+    message: str | None = None
+
+
+class Change(BaseModel):
+    sequence: int
+    entity: str
+    entity_id: str
+    operation: Literal["create", "update", "delete"]
+    payload: dict[str, Any]
+    version: int
+    created_at: datetime
+
+
+class ChangeBatch(BaseModel):
+    cursor: int
+    has_more: bool
+    changes: list[Change]

sync_sdk/ports.py

@@ -0,0 +1,94 @@
+"""Database and framework independent contracts for sync integrations."""
+
+from contextlib import AbstractAsyncContextManager
+from typing import Protocol
+from uuid import UUID
+
+from .models import ChangeBatch, Mutation, MutationResult
+
+
+class MutationRejected(Exception):
+    """Adapter validation failed BEFORE any writes for this mutation.
+
+    A binding may use this for invalid keys/payloads. It must never raise this
+    after changing data; operational failures must abort the transaction instead.
+    """
+
+
+class RetryableError(Exception):
+    """A transport can be retried without changing mutation IDs."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        retry_after: float = 0,
+        reason: str = "transient",
+    ):
+        super().__init__(message)
+        self.retry_after = retry_after
+        self.reason = reason
+
+
+class SyncTransport(Protocol):
+    async def push(self, mutations: list[Mutation]) -> list[MutationResult]: ...
+    async def pull(self, cursor: int, limit: int) -> ChangeBatch: ...
+
+
+class LocalStore(Protocol):
+    """Single-worker durable store; all mutating methods commit before returning.
+
+    apply_page MUST atomically apply records/deletes and advance the cursor.
+    acknowledge MUST atomically persist the whole batch of results. Pending work
+    survives process interruption and retains immutable mutation IDs and bodies.
+    """
+
+    client_id: str
+
+    @property
+    def cursor(self) -> int: ...
+    @property
+    def retry_state(self) -> tuple[int, float]: ...
+    def enqueue(self, mutation: Mutation) -> None: ...
+    def pending(self, limit: int) -> list[Mutation]: ...
+    def pending_count(self) -> int: ...
+    def acknowledge(self, results: list[MutationResult]) -> None: ...
+    def apply_page(self, page: ChangeBatch) -> None: ...
+    def schedule_retry(self, attempts: int, retry_at: float) -> None: ...
+
+
+class GatewayTransaction(Protocol):
+    """One atomic, serialized gateway batch, including developer-owned data.
+
+    The adapter must prevent concurrent duplicate IDs/version races. Versions
+    survive deletion. Changes have strictly increasing, commit-ordered sequences:
+    a later cursor must never hide a change that commits subsequently. Aborting
+    or cancellation rolls back entity writes, versions, changes AND results.
+    """
+
+    async def get_result(self, mutation_id: UUID) -> MutationResult | None: ...
+    async def get_version(self, entity: str, entity_id: str) -> int: ...
+    async def write(self, mutation: Mutation, version: int) -> None:
+        """Write business data and its sync version in this transaction."""
+        ...
+
+    async def append_change(self, mutation: Mutation, version: int) -> None: ...
+    async def save_result(self, mutation: Mutation, result: MutationResult) -> None: ...
+
+
+class GatewayStore(Protocol):
+    """SQL/NoSQL adapter. No connection creation or schema creation in the engine."""
+
+    def transaction(self) -> AbstractAsyncContextManager[GatewayTransaction]: ...
+    async def get_changes(self, cursor: int, limit: int) -> ChangeBatch: ...
+    async def health_check(self) -> bool: ...
+
+
+class RemoteStore(Protocol):
+    """Service boundary used by HTTP, direct in-process, or other integrations."""
+
+    async def apply_mutations(
+        self, mutations: list[Mutation]
+    ) -> list[MutationResult]: ...
+    async def get_changes(self, cursor: int, limit: int = 100) -> ChangeBatch: ...
+    async def health_check(self) -> bool: ...

sync_sdk/py.typed

@@ -0,0 +1 @@
+# Marker file for PEP 561.