edgesync 0.2.0__py3-none-any.whl

edgesync/queue/manager.py

@@ -0,0 +1,92 @@
+"""Publish-side queue orchestration.
+
+``QueueManager`` is the only thing that turns raw caller data into a durable
+``Message``. It owns payload validation and ID/size/timestamp computation,
+then delegates persistence (including capacity enforcement) to the storage
+backend. ``publish()`` only returns once ``storage.enqueue`` has confirmed
+the message is durably committed -- never before.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import timedelta
+
+from edgesync.config import EdgeSyncConfig
+from edgesync.exceptions import SerializationError
+from edgesync.logging import logger
+from edgesync.models.message import Message, MessageStatus
+from edgesync.models.receipt import PublishReceipt
+from edgesync.storage.base import StorageBackend
+from edgesync.utils.clock import Clock
+from edgesync.utils.ids import generate_message_id
+
+
+class QueueManager:
+    """Validates and durably enqueues outgoing messages."""
+
+    def __init__(self, storage: StorageBackend, config: EdgeSyncConfig, clock: Clock) -> None:
+        self._storage = storage
+        self._config = config
+        self._clock = clock
+
+    async def publish(
+        self,
+        data: object,
+        *,
+        destination: str | None = None,
+        headers: dict[str, str] | None = None,
+        priority: int = 0,
+        expires_in: float | None = None,
+    ) -> PublishReceipt:
+        try:
+            serialized = json.dumps(data)
+        except (TypeError, ValueError) as exc:
+            raise SerializationError(f"publish() payload is not JSON-serializable: {exc}") from exc
+
+        payload = json.loads(serialized)
+        headers_dict = dict(headers or {})
+        size_bytes = len(serialized.encode("utf-8")) + sum(
+            len(k.encode("utf-8")) + len(v.encode("utf-8")) for k, v in headers_dict.items()
+        )
+
+        now = self._clock.now()
+        expires_at = now + timedelta(seconds=expires_in) if expires_in is not None else None
+        resolved_destination = destination or self._config.default_destination
+
+        message = Message(
+            id=generate_message_id(),
+            destination=resolved_destination,
+            payload=payload,
+            headers=headers_dict,
+            priority=priority,
+            status=MessageStatus.PENDING,
+            attempts=0,
+            created_at=now,
+            updated_at=now,
+            next_attempt_at=now,
+            size_bytes=size_bytes,
+            expires_at=expires_at,
+        )
+
+        await self._storage.enqueue(
+            message,
+            max_messages=self._config.max_messages,
+            max_storage_bytes=self._config.max_storage_bytes,
+            overflow_policy=self._config.overflow_policy,
+        )
+
+        logger.info(
+            "message %s queued (destination=%r, priority=%d, bytes=%d)",
+            message.id,
+            resolved_destination,
+            priority,
+            size_bytes,
+        )
+
+        return PublishReceipt(
+            message_id=message.id,
+            destination=resolved_destination,
+            accepted_at=now,
+            priority=priority,
+        )

edgesync/retry/__init__.py

@@ -0,0 +1,5 @@
+"""Retry policy and exponential backoff with jitter."""
+
+from edgesync.retry.policy import RetryPolicy
+
+__all__ = ["RetryPolicy"]

edgesync/retry/backoff.py

@@ -0,0 +1,42 @@
+"""Exponential backoff with full jitter.
+
+``attempts`` is the number of failed delivery attempts recorded for a
+message so far (i.e. the value *after* incrementing on the failure that is
+being scheduled for retry). This keeps the mapping intuitive:
+
+    attempts=1 -> initial_delay      (first retry)
+    attempts=2 -> initial_delay * multiplier
+    attempts=3 -> initial_delay * multiplier**2
+    ...
+    capped at max_delay
+
+Jitter uses the "full jitter" strategy (delay = uniform(0, computed_delay)),
+which spreads out retries and avoids synchronized retry storms across many
+edge devices reconnecting at once.
+"""
+
+from __future__ import annotations
+
+import random
+
+
+def compute_delay(
+    attempts: int,
+    *,
+    initial_delay: float,
+    max_delay: float,
+    multiplier: float,
+    jitter: bool,
+    rng: random.Random | None = None,
+) -> float:
+    if attempts < 1:
+        return 0.0
+
+    raw_delay = initial_delay * (multiplier ** (attempts - 1))
+    delay = min(raw_delay, max_delay)
+
+    if jitter:
+        generator = rng or random
+        delay = generator.uniform(0.0, delay)
+
+    return max(delay, 0.0)

edgesync/retry/policy.py

@@ -0,0 +1,56 @@
+"""Configurable retry policy."""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass
+
+from edgesync.exceptions import ConfigurationError
+from edgesync.retry.backoff import compute_delay
+
+
+@dataclass(frozen=True)
+class RetryPolicy:
+    """Controls delivery retry timing.
+
+    Args:
+        initial_delay: Delay, in seconds, before the first retry.
+        max_delay: Upper bound on any single retry delay.
+        multiplier: Exponential growth factor applied per attempt.
+        jitter: Whether to randomize each delay (full jitter) to avoid
+            synchronized retry storms.
+        max_attempts: Maximum number of delivery attempts before a message
+            is moved to the dead-letter queue. ``None`` means retry
+            indefinitely.
+    """
+
+    initial_delay: float = 1.0
+    max_delay: float = 300.0
+    multiplier: float = 2.0
+    jitter: bool = True
+    max_attempts: int | None = None
+
+    def __post_init__(self) -> None:
+        if self.initial_delay <= 0:
+            raise ConfigurationError("retry_policy.initial_delay must be > 0")
+        if self.max_delay < self.initial_delay:
+            raise ConfigurationError("retry_policy.max_delay must be >= initial_delay")
+        if self.multiplier < 1:
+            raise ConfigurationError("retry_policy.multiplier must be >= 1")
+        if self.max_attempts is not None and self.max_attempts < 1:
+            raise ConfigurationError("retry_policy.max_attempts must be >= 1 or None")
+
+    def next_delay(self, attempts: int, *, rng: random.Random | None = None) -> float:
+        """Return the delay, in seconds, before the next retry attempt."""
+        return compute_delay(
+            attempts,
+            initial_delay=self.initial_delay,
+            max_delay=self.max_delay,
+            multiplier=self.multiplier,
+            jitter=self.jitter,
+            rng=rng,
+        )
+
+    def is_exhausted(self, attempts: int) -> bool:
+        """Return True once ``attempts`` has reached ``max_attempts``."""
+        return self.max_attempts is not None and attempts >= self.max_attempts

edgesync/storage/__init__.py

@@ -0,0 +1,6 @@
+"""Durable storage backends for EdgeSync's queue."""
+
+from edgesync.storage.base import StorageBackend
+from edgesync.storage.sqlite import SQLiteStorage
+
+__all__ = ["StorageBackend", "SQLiteStorage"]

edgesync/storage/base.py

@@ -0,0 +1,109 @@
+"""The storage abstraction.
+
+The synchronization worker and public client depend only on this interface,
+never on SQLite directly, so alternative durable backends can be added later
+without touching worker or client code.
+
+Every mutating method that operates on a claimed message accepts the
+``lease_id`` returned by :meth:`StorageBackend.claim_batch` and uses it as a
+fencing token: an update only applies if the message is still owned by that
+lease. This is what makes storage-level ownership safe across concurrent
+workers, not just in-memory locks.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from datetime import datetime
+
+from edgesync.config import OverflowPolicy
+from edgesync.models.message import Message
+from edgesync.models.stats import QueueStats
+
+
+class StorageBackend(ABC):
+    """Durable storage contract for the EdgeSync queue."""
+
+    @abstractmethod
+    async def initialize(self) -> None:
+        """Open the backend, apply migrations, and recover expired leases."""
+
+    @abstractmethod
+    async def enqueue(
+        self,
+        message: Message,
+        *,
+        max_messages: int | None = None,
+        max_storage_bytes: int | None = None,
+        overflow_policy: OverflowPolicy = OverflowPolicy.REJECT_NEW,
+    ) -> None:
+        """Durably persist ``message``.
+
+        Raises:
+            QueueFullError: if the queue is at capacity and ``overflow_policy``
+                is ``REJECT_NEW`` (or eviction cannot free enough capacity).
+        """
+
+    @abstractmethod
+    async def claim_batch(self, limit: int, lease_duration: float) -> list[Message]:
+        """Atomically claim up to ``limit`` eligible PENDING messages.
+
+        Claimed messages transition to IN_FLIGHT with a lease that expires
+        after ``lease_duration`` seconds unless completed first.
+        """
+
+    @abstractmethod
+    async def mark_delivered(self, message_id: str, lease_id: str) -> None:
+        """Mark a claimed message as successfully delivered and remove it."""
+
+    @abstractmethod
+    async def schedule_retry(
+        self,
+        message_id: str,
+        lease_id: str,
+        error: str,
+        next_attempt_at: datetime,
+    ) -> None:
+        """Return a claimed message to PENDING for a future retry."""
+
+    @abstractmethod
+    async def move_to_dead_letter(self, message_id: str, lease_id: str, error: str) -> None:
+        """Move a claimed message to the dead-letter queue."""
+
+    @abstractmethod
+    async def recover_expired_leases(self) -> int:
+        """Return abandoned IN_FLIGHT messages (expired lease) to PENDING.
+
+        Returns the number of messages recovered.
+        """
+
+    @abstractmethod
+    async def expire_stale_messages(self) -> int:
+        """Move PENDING messages past their ``expires_at`` to DEAD_LETTER.
+
+        Returns the number of messages expired.
+        """
+
+    @abstractmethod
+    async def get_stats(self) -> QueueStats:
+        """Return a snapshot of queue statistics."""
+
+    @abstractmethod
+    async def get_message(self, message_id: str) -> Message | None:
+        """Fetch a single message by ID, regardless of status."""
+
+    @abstractmethod
+    async def list_dead_letters(self, limit: int = 100, offset: int = 0) -> list[Message]:
+        """List messages currently in the dead-letter queue."""
+
+    @abstractmethod
+    async def retry_dead_letter(self, message_id: str) -> None:
+        """Move a dead-lettered message back to PENDING for immediate retry."""
+
+    @abstractmethod
+    async def delete_message(self, message_id: str) -> None:
+        """Permanently delete a message (e.g. from the dead-letter queue)."""
+
+    @abstractmethod
+    async def close(self) -> None:
+        """Release all resources held by the backend."""

edgesync/storage/migrations.py

@@ -0,0 +1,70 @@
+"""Schema migrations for the SQLite backend.
+
+Migrations are applied sequentially against ``PRAGMA user_version``. Each
+migration's SQL is idempotent (``IF NOT EXISTS``) so re-running migrations
+against an already-migrated database is a safe no-op.
+"""
+
+from __future__ import annotations
+
+import aiosqlite
+
+_MIGRATION_V1 = """
+CREATE TABLE IF NOT EXISTS messages (
+    id              TEXT PRIMARY KEY,
+    destination     TEXT NOT NULL,
+    payload         TEXT NOT NULL,
+    headers         TEXT NOT NULL DEFAULT '{}',
+    metadata        TEXT NOT NULL DEFAULT '{}',
+    priority        INTEGER NOT NULL DEFAULT 0,
+    status          TEXT NOT NULL,
+    attempts        INTEGER NOT NULL DEFAULT 0,
+    created_at      TEXT NOT NULL,
+    updated_at      TEXT NOT NULL,
+    next_attempt_at TEXT NOT NULL,
+    last_attempt_at TEXT,
+    delivered_at    TEXT,
+    last_error      TEXT,
+    lease_id        TEXT,
+    lease_until     TEXT,
+    expires_at      TEXT,
+    size_bytes      INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE INDEX IF NOT EXISTS idx_messages_claim
+    ON messages (status, priority DESC, next_attempt_at ASC, created_at ASC);
+
+CREATE INDEX IF NOT EXISTS idx_messages_lease
+    ON messages (status, lease_until);
+
+CREATE INDEX IF NOT EXISTS idx_messages_status
+    ON messages (status);
+
+CREATE INDEX IF NOT EXISTS idx_messages_expires
+    ON messages (status, expires_at);
+
+CREATE TABLE IF NOT EXISTS counters (
+    key   TEXT PRIMARY KEY,
+    value INTEGER NOT NULL DEFAULT 0
+);
+
+INSERT OR IGNORE INTO counters (key, value) VALUES ('delivered_total', 0);
+"""
+
+# Ordered list of (version, sql) migrations. Append new migrations here --
+# never edit an already-released migration's SQL.
+MIGRATIONS: list[tuple[int, str]] = [
+    (1, _MIGRATION_V1),
+]
+
+
+async def apply_migrations(conn: aiosqlite.Connection) -> None:
+    """Bring the database schema up to the latest known version."""
+    cursor = await conn.execute("PRAGMA user_version")
+    row = await cursor.fetchone()
+    current_version = row[0] if row else 0
+
+    for version, sql in MIGRATIONS:
+        if version > current_version:
+            await conn.executescript(sql)
+            await conn.execute(f"PRAGMA user_version = {version}")