edgesync 0.2.0__py3-none-any.whl

edgesync/__init__.py

@@ -0,0 +1,61 @@
+"""EdgeSync: reliable data delivery for unreliable networks.
+
+    from edgesync import EdgeSync
+
+    async with EdgeSync(
+        database="edgesync.db", endpoint="https://api.example.com/telemetry"
+    ) as sync:
+        receipt = await sync.publish({"temperature": 28.5})
+
+EdgeSync provides **at-least-once** delivery, not exactly-once. See
+docs/reliability.md for the full guarantee.
+"""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _pkg_version
+from pathlib import Path
+
+from edgesync.client import DeadLetterQueue, EdgeSync
+from edgesync.config import OverflowPolicy
+from edgesync.exceptions import (
+    ConfigurationError,
+    EdgeSyncError,
+    EdgeSyncNotStartedError,
+    MessageNotFoundError,
+    QueueFullError,
+    SerializationError,
+    StorageError,
+    TransportError,
+)
+from edgesync.models.delivery import DeliveryOutcome, DeliveryResult
+from edgesync.models.message import Message, MessageStatus
+from edgesync.models.receipt import PublishReceipt
+from edgesync.models.stats import QueueStats
+from edgesync.retry.policy import RetryPolicy
+
+try:
+    __version__ = _pkg_version("edgesync")
+except PackageNotFoundError:  # pragma: no cover - only when run from an uninstalled checkout
+    __version__ = (Path(__file__).resolve().parent.parent / "VERSION").read_text().strip()
+
+__all__ = [
+    "__version__",
+    "EdgeSync",
+    "DeadLetterQueue",
+    "OverflowPolicy",
+    "RetryPolicy",
+    "Message",
+    "MessageStatus",
+    "PublishReceipt",
+    "QueueStats",
+    "DeliveryOutcome",
+    "DeliveryResult",
+    "EdgeSyncError",
+    "ConfigurationError",
+    "StorageError",
+    "QueueFullError",
+    "TransportError",
+    "EdgeSyncNotStartedError",
+    "SerializationError",
+    "MessageNotFoundError",
+]

edgesync/client.py

@@ -0,0 +1,227 @@
+"""The public EdgeSync client -- the only class most users need.
+
+``EdgeSync`` wires together the queue manager, storage backend, transport
+registry, and background worker behind a small, stable surface:
+
+    async with EdgeSync(database=..., endpoint=...) as sync:
+        receipt = await sync.publish(data)
+
+See docs/getting-started.md for a full walkthrough and docs/reliability.md
+for the exact delivery guarantee this class provides.
+"""
+
+from __future__ import annotations
+
+from types import TracebackType
+
+from edgesync.config import EdgeSyncConfig, OverflowPolicy
+from edgesync.exceptions import ConfigurationError, EdgeSyncNotStartedError, MessageNotFoundError
+from edgesync.logging import logger
+from edgesync.models.message import Message, MessageStatus
+from edgesync.models.receipt import PublishReceipt
+from edgesync.models.stats import QueueStats
+from edgesync.queue.manager import QueueManager
+from edgesync.retry.policy import RetryPolicy
+from edgesync.storage.base import StorageBackend
+from edgesync.storage.sqlite import SQLiteStorage
+from edgesync.transports.base import Transport
+from edgesync.transports.http import HTTPTransport
+from edgesync.transports.registry import TransportRegistry
+from edgesync.utils.clock import Clock, SystemClock
+from edgesync.worker.lifecycle import LifecycleGuard
+from edgesync.worker.sync_worker import SyncWorker
+
+
+class DeadLetterQueue:
+    """Inspect and manage messages that exhausted retries or failed permanently.
+
+    Accessed via ``sync.dead_letters``. EdgeSync never deletes dead-lettered
+    messages on its own -- callers must explicitly ``retry`` or ``delete``
+    them.
+    """
+
+    def __init__(self, storage: StorageBackend) -> None:
+        self._storage = storage
+
+    async def list(self, limit: int = 100, offset: int = 0) -> list[Message]:
+        """List dead-lettered messages, most recently failed first."""
+        return await self._storage.list_dead_letters(limit=limit, offset=offset)
+
+    async def retry(self, message_id: str) -> None:
+        """Move a dead-lettered message back to PENDING for immediate retry."""
+        await self._require_dead_letter(message_id)
+        await self._storage.retry_dead_letter(message_id)
+        logger.info("dead-letter message %s requeued for retry", message_id)
+
+    async def delete(self, message_id: str) -> None:
+        """Permanently delete a dead-lettered message."""
+        await self._require_dead_letter(message_id)
+        await self._storage.delete_message(message_id)
+        logger.info("dead-letter message %s deleted", message_id)
+
+    async def _require_dead_letter(self, message_id: str) -> None:
+        message = await self._storage.get_message(message_id)
+        if message is None or message.status is not MessageStatus.DEAD_LETTER:
+            raise MessageNotFoundError(f"no dead-letter message with id {message_id!r}")
+
+
+class EdgeSync:
+    """Reliable, durable edge-to-cloud data delivery.
+
+    A message accepted by :meth:`publish` is durably persisted before the
+    call returns, and is only removed from the local queue once the
+    destination acknowledges it (or it is dead-lettered per the configured
+    retry policy). EdgeSync provides **at-least-once** delivery, not
+    exactly-once -- see docs/reliability.md.
+    """
+
+    def __init__(
+        self,
+        *,
+        database: str,
+        endpoint: str | None = None,
+        destinations: dict[str, Transport] | None = None,
+        default_destination: str = "default",
+        batch_size: int = 50,
+        worker_concurrency: int = 5,
+        poll_interval: float = 1.0,
+        lease_duration: float = 60.0,
+        max_messages: int | None = 1_000_000,
+        max_storage_bytes: int | None = None,
+        overflow_policy: OverflowPolicy = OverflowPolicy.REJECT_NEW,
+        retry_policy: RetryPolicy | None = None,
+        shutdown_grace_period: float = 30.0,
+        idempotency_header: str = "Idempotency-Key",
+        http_timeout: float = 30.0,
+        clock: Clock | None = None,
+    ) -> None:
+        if endpoint is None and not destinations:
+            raise ConfigurationError(
+                "EdgeSync requires endpoint=... and/or destinations={...} "
+                "so publish() has somewhere to deliver messages"
+            )
+
+        self._config = EdgeSyncConfig(
+            database=database,
+            endpoint=endpoint,
+            default_destination=default_destination,
+            batch_size=batch_size,
+            worker_concurrency=worker_concurrency,
+            poll_interval=poll_interval,
+            lease_duration=lease_duration,
+            max_messages=max_messages,
+            max_storage_bytes=max_storage_bytes,
+            overflow_policy=overflow_policy,
+            shutdown_grace_period=shutdown_grace_period,
+            idempotency_header=idempotency_header,
+            http_timeout=http_timeout,
+            retry_policy=retry_policy or RetryPolicy(),
+        )
+
+        self._clock = clock or SystemClock()
+        self._storage: StorageBackend = SQLiteStorage(database, clock=self._clock)
+
+        self._transports = TransportRegistry(default_destination)
+        if endpoint is not None:
+            self._transports.register(
+                default_destination,
+                HTTPTransport(
+                    endpoint,
+                    timeout=http_timeout,
+                    idempotency_header=idempotency_header,
+                ),
+            )
+        for name, transport in (destinations or {}).items():
+            self._transports.register(name, transport)
+
+        self._queue = QueueManager(self._storage, self._config, self._clock)
+        self._worker = SyncWorker(
+            self._storage,
+            self._transports,
+            self._config.retry_policy,
+            batch_size=batch_size,
+            concurrency=worker_concurrency,
+            poll_interval=poll_interval,
+            lease_duration=lease_duration,
+            clock=self._clock,
+        )
+        self._lifecycle = LifecycleGuard("EdgeSync")
+        self.dead_letters = DeadLetterQueue(self._storage)
+
+    async def start(self) -> None:
+        """Open durable storage, recover crash state, and start the worker.
+
+        Idempotent: calling ``start()`` while already started is a no-op.
+        """
+        if not await self._lifecycle.begin_start():
+            logger.debug("EdgeSync.start() called while already started; ignoring")
+            return
+        await self._storage.initialize()
+        await self._transports.start_all()
+        await self._worker.start()
+        self._lifecycle.mark_running()
+        logger.info("EdgeSync started (database=%r)", self._config.database)
+
+    async def publish(
+        self,
+        data: object,
+        *,
+        destination: str | None = None,
+        headers: dict[str, str] | None = None,
+        priority: int = 0,
+        expires_in: float | None = None,
+    ) -> PublishReceipt:
+        """Durably queue ``data`` for delivery and return a receipt.
+
+        The returned receipt confirms local durability, not delivery -- the
+        message will be delivered by the background worker, with retries,
+        independent of whether the caller stays connected.
+        """
+        self._require_started()
+        return await self._queue.publish(
+            data,
+            destination=destination,
+            headers=headers,
+            priority=priority,
+            expires_in=expires_in,
+        )
+
+    async def stats(self) -> QueueStats:
+        """Return a snapshot of the local queue's state."""
+        self._require_started()
+        return await self._storage.get_stats()
+
+    async def close(self, *, grace_period: float | None = None) -> None:
+        """Gracefully stop the worker and release all resources.
+
+        In-flight deliveries get up to ``grace_period`` seconds (defaults to
+        ``shutdown_grace_period``) to finish before being cancelled. No
+        queued message is ever discarded during shutdown.
+        """
+        if not await self._lifecycle.begin_stop():
+            return
+        grace = grace_period if grace_period is not None else self._config.shutdown_grace_period
+        await self._worker.stop(grace)
+        await self._transports.close_all()
+        await self._storage.close()
+        self._lifecycle.mark_stopped()
+        logger.info("EdgeSync stopped")
+
+    def _require_started(self) -> None:
+        if not self._lifecycle.is_running:
+            raise EdgeSyncNotStartedError(
+                "EdgeSync.start() must be called (or use `async with EdgeSync(...)`) "
+                "before this operation"
+            )
+
+    async def __aenter__(self) -> EdgeSync:
+        await self.start()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        await self.close()

edgesync/config.py

@@ -0,0 +1,70 @@
+"""EdgeSync configuration.
+
+``EdgeSyncConfig`` is validated eagerly at construction time so
+misconfiguration fails fast at startup rather than surfacing as a mysterious
+runtime error deep in the worker loop.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+
+from edgesync.exceptions import ConfigurationError
+from edgesync.retry.policy import RetryPolicy
+
+
+class OverflowPolicy(str, Enum):
+    """What to do when the queue is at capacity and a new message arrives.
+
+    The default is ``REJECT_NEW`` because EdgeSync must never silently drop
+    a message that was already accepted -- rejecting a *new* publish with a
+    clear exception is always safer than silently discarding queued data.
+    """
+
+    REJECT_NEW = "reject_new"
+    DROP_OLDEST = "drop_oldest"
+    DROP_LOWEST_PRIORITY = "drop_lowest_priority"
+    DEAD_LETTER = "dead_letter"
+
+
+@dataclass(frozen=True)
+class EdgeSyncConfig:
+    """Validated, immutable configuration for an ``EdgeSync`` instance."""
+
+    database: str
+    endpoint: str | None = None
+    default_destination: str = "default"
+    batch_size: int = 50
+    worker_concurrency: int = 5
+    poll_interval: float = 1.0
+    lease_duration: float = 60.0
+    max_messages: int | None = 1_000_000
+    max_storage_bytes: int | None = None
+    overflow_policy: OverflowPolicy = OverflowPolicy.REJECT_NEW
+    shutdown_grace_period: float = 30.0
+    idempotency_header: str = "Idempotency-Key"
+    http_timeout: float = 30.0
+    retry_policy: RetryPolicy = field(default_factory=RetryPolicy)
+
+    def __post_init__(self) -> None:
+        if not self.database:
+            raise ConfigurationError("database must be a non-empty path or URL")
+        if self.batch_size <= 0:
+            raise ConfigurationError("batch_size must be > 0")
+        if self.worker_concurrency <= 0:
+            raise ConfigurationError("worker_concurrency must be > 0")
+        if self.poll_interval <= 0:
+            raise ConfigurationError("poll_interval must be > 0")
+        if self.lease_duration <= 0:
+            raise ConfigurationError("lease_duration must be > 0")
+        if self.max_messages is not None and self.max_messages <= 0:
+            raise ConfigurationError("max_messages must be > 0 or None")
+        if self.max_storage_bytes is not None and self.max_storage_bytes <= 0:
+            raise ConfigurationError("max_storage_bytes must be > 0 or None")
+        if self.shutdown_grace_period < 0:
+            raise ConfigurationError("shutdown_grace_period must be >= 0")
+        if self.http_timeout <= 0:
+            raise ConfigurationError("http_timeout must be > 0")
+        if not self.idempotency_header:
+            raise ConfigurationError("idempotency_header must be a non-empty string")

edgesync/exceptions.py

@@ -0,0 +1,51 @@
+"""Exception hierarchy for EdgeSync.
+
+All exceptions raised by EdgeSync inherit from :class:`EdgeSyncError` so
+callers can catch the entire family with a single ``except`` clause while
+still being able to handle specific failure modes precisely.
+"""
+
+from __future__ import annotations
+
+
+class EdgeSyncError(Exception):
+    """Base class for all EdgeSync exceptions."""
+
+
+class ConfigurationError(EdgeSyncError):
+    """Raised when EdgeSync is configured with invalid or unsafe values."""
+
+
+class StorageError(EdgeSyncError):
+    """Raised when a durable storage operation fails."""
+
+
+class QueueFullError(EdgeSyncError):
+    """Raised when a message cannot be accepted because the queue is full.
+
+    EdgeSync never silently drops a message that could not be accepted --
+    callers must handle this exception explicitly.
+    """
+
+
+class TransportError(EdgeSyncError):
+    """Raised for unexpected transport-level failures.
+
+    Expected delivery failures (timeouts, 5xx responses, etc.) are reported
+    through :class:`~edgesync.models.delivery.DeliveryResult` and do not
+    raise. This exception is reserved for transport misconfiguration or
+    failures that occur outside of a single message delivery attempt (e.g.
+    ``start()``/``close()`` failures).
+    """
+
+
+class EdgeSyncNotStartedError(EdgeSyncError):
+    """Raised when an operation requires ``EdgeSync.start()`` to have run."""
+
+
+class SerializationError(EdgeSyncError):
+    """Raised when a message payload cannot be serialized to JSON."""
+
+
+class MessageNotFoundError(EdgeSyncError):
+    """Raised when an operation references a message ID that does not exist."""

edgesync/logging.py

@@ -0,0 +1,16 @@
+"""Logging setup.
+
+EdgeSync never configures global logging (no ``basicConfig``, no handlers).
+It only exposes a namespaced logger under ``edgesync`` and lets the host
+application decide how to handle it, per standard library best practice.
+
+Message payloads are never logged by default since they may contain
+sensitive data -- only message IDs, destinations, and error summaries are.
+"""
+
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger("edgesync")
+logger.addHandler(logging.NullHandler())

edgesync/models/__init__.py

@@ -0,0 +1,16 @@
+"""Data models shared across EdgeSync's storage, transport, and worker layers."""
+
+from edgesync.models.delivery import DeliveryOutcome, DeliveryResult
+from edgesync.models.message import JSONValue, Message, MessageStatus
+from edgesync.models.receipt import PublishReceipt
+from edgesync.models.stats import QueueStats
+
+__all__ = [
+    "DeliveryOutcome",
+    "DeliveryResult",
+    "JSONValue",
+    "Message",
+    "MessageStatus",
+    "PublishReceipt",
+    "QueueStats",
+]

edgesync/models/delivery.py

@@ -0,0 +1,69 @@
+"""Delivery outcome types returned by transports."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class DeliveryOutcome(str, Enum):
+    """The result classification of a single delivery attempt."""
+
+    SUCCESS = "success"
+    RETRYABLE_FAILURE = "retryable_failure"
+    PERMANENT_FAILURE = "permanent_failure"
+
+
+@dataclass(slots=True, frozen=True)
+class DeliveryResult:
+    """The outcome of a transport's attempt to deliver one message.
+
+    Transports are responsible only for attempting delivery and classifying
+    the outcome. They never touch storage or retry scheduling -- that is the
+    worker's job, driven by this result.
+    """
+
+    outcome: DeliveryOutcome
+    status_code: int | None = None
+    error: str | None = None
+    response_snippet: str | None = None
+
+    @property
+    def success(self) -> bool:
+        return self.outcome is DeliveryOutcome.SUCCESS
+
+    @property
+    def retryable(self) -> bool:
+        return self.outcome is DeliveryOutcome.RETRYABLE_FAILURE
+
+    @classmethod
+    def success_result(cls, status_code: int | None = None) -> DeliveryResult:
+        return cls(outcome=DeliveryOutcome.SUCCESS, status_code=status_code)
+
+    @classmethod
+    def retryable_failure(
+        cls,
+        error: str,
+        status_code: int | None = None,
+        response_snippet: str | None = None,
+    ) -> DeliveryResult:
+        return cls(
+            outcome=DeliveryOutcome.RETRYABLE_FAILURE,
+            status_code=status_code,
+            error=error,
+            response_snippet=response_snippet,
+        )
+
+    @classmethod
+    def permanent_failure(
+        cls,
+        error: str,
+        status_code: int | None = None,
+        response_snippet: str | None = None,
+    ) -> DeliveryResult:
+        return cls(
+            outcome=DeliveryOutcome.PERMANENT_FAILURE,
+            status_code=status_code,
+            error=error,
+            response_snippet=response_snippet,
+        )

edgesync/models/message.py

@@ -0,0 +1,62 @@
+"""The durable message model.
+
+A ``Message`` is the unit of work EdgeSync persists, claims, delivers, and
+retries. Every field maps directly to a column in the storage backend so the
+in-memory representation and the durable record never diverge.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Union
+
+# JSON-compatible payload type. EdgeSync only accepts JSON-serializable
+# data -- see edgesync.queue.manager for the validation boundary.
+JSONValue = Union["dict[str, JSONValue]", "list[JSONValue]", str, int, float, bool, None]
+
+
+class MessageStatus(str, Enum):
+    """The lifecycle state of a message in the durable queue.
+
+    PENDING     -- eligible for delivery once ``next_attempt_at`` has passed.
+    IN_FLIGHT   -- claimed by a worker and leased until ``lease_until``.
+    DELIVERED   -- acknowledged by the destination; removed from the active
+                   queue shortly after (see storage/sqlite.py).
+    DEAD_LETTER -- exhausted retries or failed permanently; retained until a
+                   caller retries or deletes it via ``sync.dead_letters``.
+
+    There is no separate "FAILED" state: a retryable failure returns a
+    message directly to PENDING with an updated ``next_attempt_at``, which
+    keeps the state machine small and avoids an unreachable transient state.
+    """
+
+    PENDING = "PENDING"
+    IN_FLIGHT = "IN_FLIGHT"
+    DELIVERED = "DELIVERED"
+    DEAD_LETTER = "DEAD_LETTER"
+
+
+@dataclass(slots=True)
+class Message:
+    """A single unit of data queued for delivery to a destination."""
+
+    id: str
+    destination: str
+    payload: JSONValue
+    headers: dict[str, str]
+    priority: int
+    status: MessageStatus
+    attempts: int
+    created_at: datetime
+    updated_at: datetime
+    next_attempt_at: datetime
+    size_bytes: int
+    last_attempt_at: datetime | None = None
+    delivered_at: datetime | None = None
+    last_error: str | None = None
+    lease_id: str | None = None
+    lease_until: datetime | None = None
+    expires_at: datetime | None = None
+    metadata: dict[str, str] = field(default_factory=dict)

edgesync/models/receipt.py

@@ -0,0 +1,21 @@
+"""The receipt returned to callers by ``EdgeSync.publish()``."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+
+
+@dataclass(slots=True, frozen=True)
+class PublishReceipt:
+    """Proof that a message was durably accepted into the local queue.
+
+    Receiving a receipt means the message has been committed to durable
+    storage -- it does **not** mean the message has reached the destination
+    yet. See docs/reliability.md for the full delivery guarantee.
+    """
+
+    message_id: str
+    destination: str
+    accepted_at: datetime
+    priority: int

edgesync/models/stats.py

@@ -0,0 +1,27 @@
+"""Queue statistics returned by ``EdgeSync.stats()``."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(slots=True, frozen=True)
+class QueueStats:
+    """A snapshot of the durable queue's state.
+
+    ``delivered`` is a cumulative, all-time counter -- delivered messages
+    are removed from the active table shortly after acknowledgement to keep
+    the on-disk queue bounded, so this number is not a row count.
+    """
+
+    pending: int
+    in_flight: int
+    delivered: int
+    dead_letter: int
+    retrying: int
+    storage_bytes: int | None = None
+
+    @property
+    def active(self) -> int:
+        """Messages not yet delivered or dead-lettered."""
+        return self.pending + self.in_flight

edgesync/queue/__init__.py

@@ -0,0 +1,5 @@
+"""Publish-side orchestration: validates, builds, and durably enqueues messages."""
+
+from edgesync.queue.manager import QueueManager
+
+__all__ = ["QueueManager"]