offline-sync-sdk 0.2.0__py3-none-any.whl

sync_sdk/adapters/sqlalchemy.py

@@ -0,0 +1,691 @@
+"""SQLAlchemy adapter for developer-owned tables and SDK-owned sync metadata.
+
+All participating tables must use the same transactional database. A singleton
+metadata row serializes gateway writes, including sequence allocation, so a pull
+cursor can never skip a late-committing change. This prioritizes correctness over
+write throughput. Schema initialization is an explicit application/setup action.
+"""
+
+import asyncio
+import json
+import time
+from collections.abc import Callable, Mapping
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any, Protocol
+from uuid import UUID
+
+from pydantic import TypeAdapter, ValidationError
+from sqlalchemy import (
+    JSON,
+    Column,
+    DateTime,
+    Integer,
+    MetaData,
+    String,
+    Table,
+    and_,
+    delete,
+    insert,
+    select,
+    text,
+    update,
+)
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession
+
+from ..models import Change, ChangeBatch, Mutation, MutationResult
+from ..ports import MutationRejected
+from ..telemetry import PipelineTelemetry, safe_telemetry
+
+sync_metadata = MetaData()
+clock = Table(
+    "sync_clock",
+    sync_metadata,
+    Column("id", Integer, primary_key=True),
+    Column("sequence", Integer, nullable=False),
+)
+versions = Table(
+    "sync_versions",
+    sync_metadata,
+    Column("entity", String(191), primary_key=True),
+    Column("entity_id", String(191), primary_key=True),
+    Column("version", Integer, nullable=False),
+)
+processed = Table(
+    "sync_processed_mutations",
+    sync_metadata,
+    Column("mutation_id", String(36), primary_key=True),
+    Column("client_id", String(191), nullable=False),
+    Column("result", JSON, nullable=False),
+    Column("processed_at", DateTime(timezone=True), nullable=False),
+)
+changes = Table(
+    "sync_change_log",
+    sync_metadata,
+    Column("sequence", Integer, primary_key=True, autoincrement=False),
+    Column("entity", String(191), nullable=False),
+    Column("entity_id", String(191), nullable=False),
+    Column("operation", String(16), nullable=False),
+    Column("payload", JSON, nullable=False),
+    Column("version", Integer, nullable=False),
+    Column("created_at", DateTime(timezone=True), nullable=False),
+)
+capture_checkpoints = Table(
+    "sync_capture_checkpoints",
+    sync_metadata,
+    Column("entity", String(191), primary_key=True),
+    Column("scanned_at", DateTime(timezone=True), nullable=False),
+)
+captured_rows = Table(
+    "sync_captured_rows",
+    sync_metadata,
+    Column("entity", String(191), primary_key=True),
+    Column("entity_id", String(191), primary_key=True),
+    Column("payload", JSON, nullable=False),
+)
+
+
+async def initialize_sync_metadata(engine: AsyncEngine) -> None:
+    """Create only SDK tables and the lock row. Run once before serving traffic.
+
+    Production applications may include sync_metadata in their own migrations.
+    This function never creates or alters business tables.
+    """
+    async with engine.begin() as connection:
+        await connection.run_sync(sync_metadata.create_all)
+        if (await connection.scalar(select(clock.c.id).where(clock.c.id == 1))) is None:
+            await connection.execute(insert(clock).values(id=1, sequence=0))
+
+
+class SQLAlchemyBinding(Protocol):
+    """Custom entity mapping; MUST use this session and MUST NOT commit."""
+
+    async def write(
+        self, session: AsyncSession, mutation: Mutation, version: int
+    ) -> None: ...
+
+
+class TableBinding:
+    """Map explicitly selected payload fields to application-owned columns.
+
+    Scalar keys use canonical strings. Composite keys use a canonical JSON object
+    of primary-key column names and string values, generated by encode_key().
+    Complete mapped payloads are required; nonmapped columns remain untouched.
+    """
+
+    def __init__(
+        self,
+        table: Table,
+        *,
+        fields: Mapping[str, str],
+        parse_key: Callable[[str], Any] | None = None,
+        key_parsers: Mapping[str, Callable[[str], Any]] | None = None,
+        value_decoders: Mapping[str, Callable[[Any], Any]] | None = None,
+    ):
+        self.keys = list(table.primary_key.columns)
+        if not self.keys:
+            raise ValueError("A synced table needs a primary key")
+        if parse_key is not None and (len(self.keys) != 1 or key_parsers is not None):
+            raise ValueError(
+                "parse_key is for scalar keys; use key_parsers for composite keys"
+            )
+        self.table = table
+        self.fields = dict(fields)
+        names = {column.name for column in self.keys}
+        if key_parsers is not None and set(key_parsers) != names:
+            raise ValueError("key_parsers must cover exactly the primary-key columns")
+        self.key_parsers = (
+            dict(key_parsers)
+            if key_parsers is not None
+            else {
+                column.name: parse_key or column.type.python_type
+                for column in self.keys
+            }
+        )
+        if len(set(self.fields.values())) != len(self.fields):
+            raise ValueError("Payload fields must map to distinct columns")
+        if any(
+            column not in table.c or column in names for column in self.fields.values()
+        ):
+            raise ValueError(
+                "Mapped columns must exist and cannot include the primary key"
+            )
+
+        overrides = dict(value_decoders or {})
+        if not set(overrides).issubset(self.fields):
+            raise ValueError("value_decoders must refer to registered payload fields")
+        self.value_decoders = {}
+        for field, column_name in self.fields.items():
+            column = table.c[column_name]
+            if field in overrides:
+                self.value_decoders[field] = overrides[field]
+            elif isinstance(column.type, JSON):
+                self.value_decoders[field] = lambda value: value
+            else:
+                self.value_decoders[field] = TypeAdapter(
+                    column.type.python_type
+                ).validate_python
+
+    def encode_key(self, value: Any) -> str:
+        """Generate a stable wire ID from a scalar or a composite-key dictionary."""
+        if len(self.keys) == 1:
+            encoded = str(value)
+        else:
+            if not isinstance(value, Mapping) or set(value) != set(self.key_parsers):
+                raise ValueError("Supply exactly the composite-key columns")
+            encoded = json.dumps(
+                {name: str(value[name]) for name in self.key_parsers},
+                sort_keys=True,
+                separators=(",", ":"),
+            )
+        self._decode_key(encoded)
+        return encoded
+
+    def _decode_key(self, entity_id: str) -> dict[str, Any]:
+        try:
+            if len(self.keys) == 1:
+                raw = {self.keys[0].name: entity_id}
+            else:
+                raw = json.loads(entity_id)
+                if not isinstance(raw, dict) or set(raw) != set(self.key_parsers):
+                    raise ValueError("Invalid composite key")
+                if json.dumps(raw, sort_keys=True, separators=(",", ":")) != entity_id:
+                    raise ValueError("Noncanonical composite key")
+            values = {}
+            for name, parser in self.key_parsers.items():
+                if not isinstance(raw[name], str):
+                    raise TypeError("Key parts must be strings")
+                value = parser(raw[name])
+                if str(value) != raw[name]:
+                    raise ValueError("Noncanonical key part")
+                values[name] = value
+            if len(entity_id) > 191:
+                raise ValueError(
+                    "Encoded key exceeds SQL metadata limit of 191 characters"
+                )
+            return values
+        except (ValueError, TypeError, OverflowError) as exc:
+            raise MutationRejected(
+                "Invalid entity ID; use the binding's encode_key() for canonical keys"
+            ) from exc
+
+    async def write(
+        self, session: AsyncSession, mutation: Mutation, version: int
+    ) -> None:
+        keys = self._decode_key(mutation.entity_id)
+        predicate = and_(*(self.table.c[name] == value for name, value in keys.items()))
+        if mutation.operation == "delete":
+            await session.execute(delete(self.table).where(predicate))
+            return
+        if set(mutation.payload) != set(self.fields):
+            raise MutationRejected(
+                "Payload must contain exactly the registered table fields; configure an Entity schema"
+            )
+        values: dict[str, Any] = {}
+        try:
+            for field, value in mutation.payload.items():
+                column = self.table.c[self.fields[field]]
+                if value is None:
+                    if not column.nullable:
+                        raise ValueError("Null is not allowed")
+                    values[column.name] = None
+                else:
+                    values[column.name] = self.value_decoders[field](value)
+        except (ValueError, TypeError, OverflowError, ValidationError) as exc:
+            raise MutationRejected(
+                "Payload value is incompatible with the registered column"
+            ) from exc
+        exists = await session.scalar(select(self.keys[0]).where(predicate))
+        if exists is None:
+            await session.execute(insert(self.table).values({**keys, **values}))
+        elif values:
+            await session.execute(update(self.table).where(predicate).values(values))
+
+    async def read_rows(
+        self,
+        session: AsyncSession,
+        normalize: Callable[[dict[str, Any]], dict[str, Any]],
+    ) -> dict[str, dict[str, Any]]:
+        """Read the sync-visible projection used by change capture."""
+        selected = [*self.keys]
+        selected.extend(self.table.c[name] for name in self.fields.values())
+        rows = (
+            (await session.execute(select(*selected).order_by(*self.keys)))
+            .mappings()
+            .all()
+        )
+        result: dict[str, dict[str, Any]] = {}
+        for row in rows:
+            if len(self.keys) == 1:
+                key: Any = row[self.keys[0].name]
+            else:
+                key = {column.name: row[column.name] for column in self.keys}
+            entity_id = self.encode_key(key)
+            payload = {
+                field: row[column_name] for field, column_name in self.fields.items()
+            }
+            result[entity_id] = normalize(payload)
+        return result
+
+
+@dataclass(frozen=True)
+class CaptureReport:
+    """Work completed by one database change-capture scan."""
+
+    scanned_rows: int = 0
+    captured_changes: int = 0
+    initialized_entities: int = 0
+
+
+class SQLAlchemyChangeCapture:
+    """Poll registered SQL tables and append outside writes to the sync feed.
+
+    An entity's first scan stores a baseline without publishing its existing rows.
+    Later scans publish creates, updates, and deletes. The durable snapshot and feed
+    entries are committed together under the same clock lock as gateway writes.
+    """
+
+    def __init__(
+        self,
+        session_factory: Callable[[], AsyncSession],
+        bindings: Mapping[str, TableBinding],
+        normalizers: Mapping[str, Callable[[dict[str, Any]], dict[str, Any]]],
+    ):
+        self.session_factory = session_factory
+        self.bindings = dict(bindings)
+        self.normalizers = dict(normalizers)
+
+    async def capture_once(self) -> CaptureReport:
+        scanned_rows = captured_count = initialized_count = 0
+        async with self.session_factory() as session, session.begin():
+            sequence = await _lock_clock(session)
+            transaction = _Transaction(session, self.bindings, sequence)
+            for entity, binding in self.bindings.items():
+                current = await binding.read_rows(session, self.normalizers[entity])
+                scanned_rows += len(current)
+                initialized = await session.scalar(
+                    select(capture_checkpoints.c.entity).where(
+                        capture_checkpoints.c.entity == entity
+                    )
+                )
+                if initialized is None:
+                    await session.execute(
+                        delete(captured_rows).where(captured_rows.c.entity == entity)
+                    )
+                    await self._save_rows(session, entity, current)
+                    await session.execute(
+                        insert(capture_checkpoints).values(
+                            entity=entity, scanned_at=datetime.now(UTC)
+                        )
+                    )
+                    initialized_count += 1
+                    continue
+
+                previous_rows = (
+                    (
+                        await session.execute(
+                            select(
+                                captured_rows.c.entity_id,
+                                captured_rows.c.payload,
+                            ).where(captured_rows.c.entity == entity)
+                        )
+                    )
+                    .mappings()
+                    .all()
+                )
+                previous = {row["entity_id"]: row["payload"] for row in previous_rows}
+                for entity_id in sorted(current.keys() | previous.keys()):
+                    payload = current.get(entity_id)
+                    old_payload = previous.get(entity_id)
+                    if payload is None:
+                        await transaction.append_captured_change(
+                            entity, entity_id, "delete", {}
+                        )
+                        await session.execute(
+                            delete(captured_rows).where(
+                                captured_rows.c.entity == entity,
+                                captured_rows.c.entity_id == entity_id,
+                            )
+                        )
+                    elif old_payload is None:
+                        await transaction.append_captured_change(
+                            entity, entity_id, "create", payload
+                        )
+                        await self._save_row(session, entity, entity_id, payload)
+                    elif payload != old_payload:
+                        await transaction.append_captured_change(
+                            entity, entity_id, "update", payload
+                        )
+                        await session.execute(
+                            update(captured_rows)
+                            .where(
+                                captured_rows.c.entity == entity,
+                                captured_rows.c.entity_id == entity_id,
+                            )
+                            .values(payload=payload)
+                        )
+                    else:
+                        continue
+                    captured_count += 1
+                await session.execute(
+                    update(capture_checkpoints)
+                    .where(capture_checkpoints.c.entity == entity)
+                    .values(scanned_at=datetime.now(UTC))
+                )
+        return CaptureReport(scanned_rows, captured_count, initialized_count)
+
+    async def run(self, *, interval: float = 1.0) -> None:
+        """Continuously capture changes until the task is cancelled."""
+        if interval <= 0:
+            raise ValueError("Capture interval must be greater than zero")
+        while True:
+            await self.capture_once()
+            await asyncio.sleep(interval)
+
+    @staticmethod
+    async def _save_row(
+        session: AsyncSession, entity: str, entity_id: str, payload: dict[str, Any]
+    ) -> None:
+        await session.execute(
+            insert(captured_rows).values(
+                entity=entity, entity_id=entity_id, payload=payload
+            )
+        )
+
+    @classmethod
+    async def _save_rows(
+        cls,
+        session: AsyncSession,
+        entity: str,
+        rows: Mapping[str, dict[str, Any]],
+    ) -> None:
+        for entity_id, payload in rows.items():
+            await cls._save_row(session, entity, entity_id, payload)
+
+
+async def _lock_clock(session: AsyncSession) -> int:
+    backend = session.get_bind().dialect.name
+    if backend == "sqlite":
+        await session.execute(text("BEGIN IMMEDIATE"))
+    sequence = await session.scalar(
+        select(clock.c.sequence).where(clock.c.id == 1).with_for_update()
+    )
+    if sequence is None:
+        raise RuntimeError("Sync metadata is not initialized")
+    return sequence
+
+
+class SQLAlchemyStore:
+    def __init__(
+        self,
+        session_factory: Callable[[], AsyncSession],
+        bindings: Mapping[str, SQLAlchemyBinding],
+        *,
+        telemetry: PipelineTelemetry | None = None,
+    ):
+        self.session_factory, self.bindings = session_factory, dict(bindings)
+        self.telemetry = safe_telemetry(telemetry)
+
+    @staticmethod
+    def _metric_attributes(
+        backend: str, *, operation: str, outcome: str
+    ) -> dict[str, str]:
+        return {
+            "db.system.name": backend,
+            "sync.storage.operation": operation,
+            "sync.outcome": outcome,
+        }
+
+    @asynccontextmanager
+    async def transaction(self):
+        started = time.perf_counter()
+        async with self.session_factory() as session:
+            backend = session.get_bind().dialect.name
+            span_attributes = {"db.system.name": backend}
+            outcome = "ok"
+            with self.telemetry.span(
+                "sync.storage.transaction", span_attributes
+            ) as span:
+                try:
+                    async with session.begin():
+                        sequence = await _lock_clock(session)
+                        yield _Transaction(session, self.bindings, sequence)
+                except BaseException:
+                    outcome = "error"
+                    self.telemetry.count(
+                        "sync.storage.failures",
+                        attributes=self._metric_attributes(
+                            backend, operation="transaction", outcome=outcome
+                        ),
+                    )
+                    raise
+                finally:
+                    attributes = self._metric_attributes(
+                        backend, operation="transaction", outcome=outcome
+                    )
+                    self.telemetry.count(
+                        "sync.storage.transactions", attributes=attributes
+                    )
+                    self.telemetry.record(
+                        "sync.storage.transaction.duration",
+                        (time.perf_counter() - started) * 1000,
+                        attributes,
+                    )
+                    span.set_attribute("sync.outcome", outcome)
+
+    async def get_changes(self, cursor: int, limit: int) -> ChangeBatch:
+        started = time.perf_counter()
+        async with self.session_factory() as session:
+            backend = session.get_bind().dialect.name
+            outcome = "ok"
+            try:
+                rows = (
+                    (
+                        await session.execute(
+                            select(changes)
+                            .where(changes.c.sequence > cursor)
+                            .order_by(changes.c.sequence)
+                            .limit(limit + 1)
+                        )
+                    )
+                    .mappings()
+                    .all()
+                )
+                batch = [Change.model_validate(dict(row)) for row in rows[:limit]]
+                return ChangeBatch(
+                    cursor=batch[-1].sequence if batch else cursor,
+                    has_more=len(rows) > limit,
+                    changes=batch,
+                )
+            except BaseException:
+                outcome = "error"
+                self.telemetry.count(
+                    "sync.storage.failures",
+                    attributes=self._metric_attributes(
+                        backend, operation="pull", outcome=outcome
+                    ),
+                )
+                raise
+            finally:
+                self.telemetry.record(
+                    "sync.storage.pull.duration",
+                    (time.perf_counter() - started) * 1000,
+                    self._metric_attributes(backend, operation="pull", outcome=outcome),
+                )
+
+    async def health_check(self) -> bool:
+        started = time.perf_counter()
+        async with self.session_factory() as session:
+            backend = session.get_bind().dialect.name
+            outcome = "ok"
+            try:
+                healthy = (
+                    await session.scalar(select(clock.c.id).where(clock.c.id == 1)) == 1
+                )
+                if not healthy:
+                    outcome = "error"
+                return healthy
+            except (SQLAlchemyError, OSError):
+                outcome = "error"
+                return False
+            finally:
+                self.telemetry.record(
+                    "sync.storage.health.duration",
+                    (time.perf_counter() - started) * 1000,
+                    self._metric_attributes(
+                        backend, operation="health", outcome=outcome
+                    ),
+                )
+
+
+class _Transaction:
+    def __init__(
+        self,
+        session: AsyncSession,
+        bindings: Mapping[str, SQLAlchemyBinding],
+        sequence: int,
+    ):
+        self.session, self.bindings, self.sequence = session, bindings, sequence
+
+    async def get_result(self, mutation_id: UUID) -> MutationResult | None:
+        result = await self.session.scalar(
+            select(processed.c.result).where(
+                processed.c.mutation_id == str(mutation_id)
+            )
+        )
+        return MutationResult.model_validate(result) if result is not None else None
+
+    async def get_version(self, entity: str, entity_id: str) -> int:
+        return (
+            await self.session.scalar(
+                select(versions.c.version).where(
+                    versions.c.entity == entity, versions.c.entity_id == entity_id
+                )
+            )
+            or 0
+        )
+
+    async def write(self, mutation: Mutation, version: int) -> None:
+        binding = self.bindings.get(mutation.entity)
+        if binding is None:
+            raise ValueError(
+                f"No storage binding for registered entity: {mutation.entity}"
+            )
+        await binding.write(self.session, mutation, version)
+        predicate = (versions.c.entity == mutation.entity) & (
+            versions.c.entity_id == mutation.entity_id
+        )
+        existing = await self.session.scalar(
+            select(versions.c.version).where(predicate)
+        )
+        if existing is None:
+            await self.session.execute(
+                insert(versions).values(
+                    entity=mutation.entity,
+                    entity_id=mutation.entity_id,
+                    version=version,
+                )
+            )
+        else:
+            await self.session.execute(
+                update(versions).where(predicate).values(version=version)
+            )
+        if isinstance(binding, TableBinding):
+            await self._sync_capture_snapshot(mutation)
+
+    async def _sync_capture_snapshot(self, mutation: Mutation) -> None:
+        initialized = await self.session.scalar(
+            select(capture_checkpoints.c.entity).where(
+                capture_checkpoints.c.entity == mutation.entity
+            )
+        )
+        if initialized is None:
+            return
+        predicate = (captured_rows.c.entity == mutation.entity) & (
+            captured_rows.c.entity_id == mutation.entity_id
+        )
+        existing = await self.session.scalar(
+            select(captured_rows.c.entity_id).where(predicate)
+        )
+        if mutation.operation == "delete":
+            if existing is not None:
+                await self.session.execute(delete(captured_rows).where(predicate))
+        elif existing is None:
+            await self.session.execute(
+                insert(captured_rows).values(
+                    entity=mutation.entity,
+                    entity_id=mutation.entity_id,
+                    payload=mutation.payload,
+                )
+            )
+        else:
+            await self.session.execute(
+                update(captured_rows).where(predicate).values(payload=mutation.payload)
+            )
+
+    async def append_change(self, mutation: Mutation, version: int) -> None:
+        self.sequence += 1
+        await self.session.execute(
+            insert(changes).values(
+                sequence=self.sequence,
+                entity=mutation.entity,
+                entity_id=mutation.entity_id,
+                operation=mutation.operation,
+                payload=mutation.payload,
+                version=version,
+                created_at=datetime.now(UTC),
+            )
+        )
+        await self.session.execute(
+            update(clock).where(clock.c.id == 1).values(sequence=self.sequence)
+        )
+
+    async def append_captured_change(
+        self,
+        entity: str,
+        entity_id: str,
+        operation: str,
+        payload: dict[str, Any],
+    ) -> None:
+        version = await self.get_version(entity, entity_id) + 1
+        predicate = (versions.c.entity == entity) & (versions.c.entity_id == entity_id)
+        existing = await self.session.scalar(
+            select(versions.c.version).where(predicate)
+        )
+        if existing is None:
+            await self.session.execute(
+                insert(versions).values(
+                    entity=entity, entity_id=entity_id, version=version
+                )
+            )
+        else:
+            await self.session.execute(
+                update(versions).where(predicate).values(version=version)
+            )
+        self.sequence += 1
+        await self.session.execute(
+            insert(changes).values(
+                sequence=self.sequence,
+                entity=entity,
+                entity_id=entity_id,
+                operation=operation,
+                payload=payload,
+                version=version,
+                created_at=datetime.now(UTC),
+            )
+        )
+        await self.session.execute(
+            update(clock).where(clock.c.id == 1).values(sequence=self.sequence)
+        )
+
+    async def save_result(self, mutation: Mutation, result: MutationResult) -> None:
+        await self.session.execute(
+            insert(processed).values(
+                mutation_id=str(mutation.mutation_id),
+                client_id=mutation.client_id,
+                result=result.model_dump(mode="json"),
+                processed_at=datetime.now(UTC),
+            )
+        )