edgesync 0.2.0__py3-none-any.whl

edgesync/storage/sqlite.py

@@ -0,0 +1,529 @@
+"""SQLite durable storage backend.
+
+Design notes (see docs/storage.md for the full writeup):
+
+* A single ``aiosqlite`` connection is used per ``SQLiteStorage`` instance,
+  opened in autocommit mode (``isolation_level=None``). Single-statement
+  writes (claim, mark_delivered, schedule_retry, ...) commit atomically as
+  soon as they execute -- no explicit transaction needed. The one
+  compound read-then-write operation (``enqueue``'s capacity check) wraps
+  itself in an explicit ``BEGIN IMMEDIATE`` / ``COMMIT`` guarded by an
+  ``asyncio.Lock`` so concurrent ``enqueue`` calls on the same connection
+  never try to open two transactions at once.
+* ``PRAGMA journal_mode=WAL`` + ``PRAGMA synchronous=FULL`` by default:
+  WAL for concurrent readers during writes, and ``synchronous=FULL`` so a
+  commit is not reported successful until it is durable on disk -- this
+  library explicitly has to survive power loss, not just process crashes.
+  Callers that can tolerate a small durability/performance trade-off may
+  pass ``synchronous="NORMAL"``.
+* Message ownership across workers is enforced with a lease fencing token
+  (``lease_id``): every completion write is qualified with
+  ``WHERE id = ? AND lease_id = ? AND status = 'IN_FLIGHT'``, so a worker
+  whose lease already expired and was reclaimed by someone else can never
+  clobber the new owner's state.
+* Delivered messages are removed from the ``messages`` table immediately
+  (a cumulative counter in ``counters`` tracks the all-time total) to keep
+  the on-disk queue bounded for storage-constrained edge devices.
+  Dead-lettered messages are retained until a caller retries or deletes
+  them explicitly.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+from datetime import datetime, timedelta, timezone
+
+import aiosqlite
+
+from edgesync.config import OverflowPolicy
+from edgesync.exceptions import QueueFullError, StorageError
+from edgesync.logging import logger
+from edgesync.models.message import Message, MessageStatus
+from edgesync.models.stats import QueueStats
+from edgesync.storage.base import StorageBackend
+from edgesync.storage.migrations import apply_migrations
+from edgesync.utils.clock import Clock, SystemClock
+from edgesync.utils.ids import generate_lease_id
+
+_ACTIVE_STATUSES = (MessageStatus.PENDING.value, MessageStatus.IN_FLIGHT.value)
+
+
+def _resolve_database_path(database: str) -> str:
+    if database == ":memory:":
+        return database
+    if database.startswith("sqlite:///"):
+        return database[len("sqlite:///") :]
+    if database.startswith("sqlite://"):
+        return database[len("sqlite://") :]
+    return database
+
+
+def _dt_to_str(value: datetime | None) -> str | None:
+    if value is None:
+        return None
+    return value.astimezone(timezone.utc).isoformat()
+
+
+def _str_to_dt(value: str | None) -> datetime | None:
+    if value is None:
+        return None
+    return datetime.fromisoformat(value)
+
+
+class SQLiteStorage(StorageBackend):
+    """A durable, single-file SQLite-backed queue."""
+
+    def __init__(
+        self,
+        database: str,
+        *,
+        clock: Clock | None = None,
+        synchronous: str = "FULL",
+    ) -> None:
+        self._path = _resolve_database_path(database)
+        self._clock = clock or SystemClock()
+        self._synchronous = synchronous
+        self._conn: aiosqlite.Connection | None = None
+        self._write_lock = asyncio.Lock()
+
+    @property
+    def _connection(self) -> aiosqlite.Connection:
+        if self._conn is None:
+            raise StorageError("SQLiteStorage has not been initialized; call initialize() first")
+        return self._conn
+
+    async def initialize(self) -> None:
+        if self._conn is not None:
+            return
+        directory = os.path.dirname(self._path)
+        if directory:
+            os.makedirs(directory, exist_ok=True)
+
+        try:
+            conn = await aiosqlite.connect(self._path, isolation_level=None)
+        except Exception as exc:
+            raise StorageError(f"failed to open database at {self._path!r}: {exc}") from exc
+
+        conn.row_factory = aiosqlite.Row
+        await conn.execute("PRAGMA foreign_keys = ON")
+        await conn.execute("PRAGMA busy_timeout = 30000")
+        if self._path != ":memory:":
+            await conn.execute("PRAGMA journal_mode = WAL")
+        await conn.execute(f"PRAGMA synchronous = {self._synchronous}")
+
+        self._conn = conn
+        await apply_migrations(conn)
+
+        recovered = await self.recover_expired_leases()
+        if recovered:
+            logger.info("recovered %d message(s) with expired leases on startup", recovered)
+
+    async def close(self) -> None:
+        if self._conn is not None:
+            await self._conn.close()
+            self._conn = None
+
+    # -- enqueue ---------------------------------------------------------
+
+    async def enqueue(
+        self,
+        message: Message,
+        *,
+        max_messages: int | None = None,
+        max_storage_bytes: int | None = None,
+        overflow_policy: OverflowPolicy = OverflowPolicy.REJECT_NEW,
+    ) -> None:
+        conn = self._connection
+        async with self._write_lock:
+            await conn.execute("BEGIN IMMEDIATE")
+            try:
+                await self._enforce_capacity(
+                    message,
+                    max_messages=max_messages,
+                    max_storage_bytes=max_storage_bytes,
+                    overflow_policy=overflow_policy,
+                )
+                await conn.execute(
+                    """
+                    INSERT INTO messages (
+                        id, destination, payload, headers, metadata, priority,
+                        status, attempts, created_at, updated_at, next_attempt_at,
+                        last_attempt_at, delivered_at, last_error, lease_id,
+                        lease_until, expires_at, size_bytes
+                    ) VALUES (
+                        :id, :destination, :payload, :headers, :metadata, :priority,
+                        :status, :attempts, :created_at, :updated_at, :next_attempt_at,
+                        :last_attempt_at, :delivered_at, :last_error, :lease_id,
+                        :lease_until, :expires_at, :size_bytes
+                    )
+                    """,
+                    self._message_to_row(message),
+                )
+                await conn.execute("COMMIT")
+            except BaseException:
+                await conn.execute("ROLLBACK")
+                raise
+
+    async def _enforce_capacity(
+        self,
+        message: Message,
+        *,
+        max_messages: int | None,
+        max_storage_bytes: int | None,
+        overflow_policy: OverflowPolicy,
+    ) -> None:
+        conn = self._connection
+
+        if max_storage_bytes is not None and message.size_bytes > max_storage_bytes:
+            raise QueueFullError(
+                f"message ({message.size_bytes} bytes) exceeds max_storage_bytes="
+                f"{max_storage_bytes} and can never be enqueued"
+            )
+        if max_messages is None and max_storage_bytes is None:
+            return
+
+        max_iterations = 10_000  # safety bound against pathological loops
+        for _ in range(max_iterations):
+            cursor = await conn.execute(
+                "SELECT COUNT(*), COALESCE(SUM(size_bytes), 0) FROM messages "
+                f"WHERE status IN ('{_ACTIVE_STATUSES[0]}', '{_ACTIVE_STATUSES[1]}')"
+            )
+            row = await cursor.fetchone()
+            assert row is not None  # COUNT(*)/SUM always return exactly one row
+            count, total_bytes = row[0], row[1]
+
+            over_count = max_messages is not None and count >= max_messages
+            over_bytes = (
+                max_storage_bytes is not None
+                and (total_bytes + message.size_bytes) > max_storage_bytes
+            )
+            if not over_count and not over_bytes:
+                return
+
+            if overflow_policy is OverflowPolicy.REJECT_NEW:
+                raise QueueFullError(
+                    f"queue is full (messages={count}, bytes={total_bytes}); "
+                    "overflow_policy=reject_new"
+                )
+
+            victim_id = await self._select_eviction_victim(overflow_policy)
+            if victim_id is None:
+                raise QueueFullError(
+                    "queue is full and no evictable message is available "
+                    "(all active messages are currently in flight)"
+                )
+            await self._evict(victim_id, overflow_policy)
+
+        raise QueueFullError("queue is full and could not be reduced below capacity")
+
+    async def _select_eviction_victim(self, overflow_policy: OverflowPolicy) -> str | None:
+        conn = self._connection
+        if overflow_policy in (OverflowPolicy.DROP_OLDEST, OverflowPolicy.DEAD_LETTER):
+            order_by = "created_at ASC"
+        elif overflow_policy is OverflowPolicy.DROP_LOWEST_PRIORITY:
+            order_by = "priority ASC, created_at ASC"
+        else:
+            return None
+        cursor = await conn.execute(
+            f"SELECT id FROM messages WHERE status = 'PENDING' ORDER BY {order_by} LIMIT 1"
+        )
+        row = await cursor.fetchone()
+        return row[0] if row else None
+
+    async def _evict(self, message_id: str, overflow_policy: OverflowPolicy) -> None:
+        conn = self._connection
+        now = _dt_to_str(self._clock.now())
+        if overflow_policy is OverflowPolicy.DEAD_LETTER:
+            logger.warning("queue full: moving message %s to dead-letter to make room", message_id)
+            await conn.execute(
+                "UPDATE messages SET status = 'DEAD_LETTER', last_error = :error, "
+                "updated_at = :now WHERE id = :id",
+                {"error": "evicted: queue capacity exceeded", "now": now, "id": message_id},
+            )
+        else:
+            logger.warning(
+                "queue full: dropping message %s (overflow_policy=%s)",
+                message_id,
+                overflow_policy.value,
+            )
+            await conn.execute("DELETE FROM messages WHERE id = :id", {"id": message_id})
+
+    # -- claim / complete --------------------------------------------------
+
+    async def claim_batch(self, limit: int, lease_duration: float) -> list[Message]:
+        conn = self._connection
+        now = self._clock.now()
+        lease_id = generate_lease_id()
+        lease_until = now + timedelta(seconds=lease_duration)
+
+        async with self._write_lock:
+            await conn.execute("BEGIN IMMEDIATE")
+            try:
+                await conn.execute(
+                    """
+                    UPDATE messages
+                    SET status = 'IN_FLIGHT',
+                        lease_id = :lease_id,
+                        lease_until = :lease_until,
+                        updated_at = :now
+                    WHERE id IN (
+                        SELECT id FROM messages
+                        WHERE status = 'PENDING'
+                          AND next_attempt_at <= :now
+                        ORDER BY priority DESC, next_attempt_at ASC, created_at ASC
+                        LIMIT :limit
+                    )
+                    """,
+                    {
+                        "lease_id": lease_id,
+                        "lease_until": _dt_to_str(lease_until),
+                        "now": _dt_to_str(now),
+                        "limit": limit,
+                    },
+                )
+                cursor = await conn.execute(
+                    "SELECT * FROM messages WHERE lease_id = :lease_id", {"lease_id": lease_id}
+                )
+                rows = await cursor.fetchall()
+                await conn.execute("COMMIT")
+            except BaseException:
+                await conn.execute("ROLLBACK")
+                raise
+
+        messages = [self._row_to_message(row) for row in rows]
+        if messages:
+            logger.debug("claimed %d message(s) with lease %s", len(messages), lease_id)
+        return messages
+
+    async def mark_delivered(self, message_id: str, lease_id: str) -> None:
+        conn = self._connection
+        now = _dt_to_str(self._clock.now())
+        cursor = await conn.execute(
+            "UPDATE messages SET status = 'DELIVERED', delivered_at = :now, "
+            "updated_at = :now WHERE id = :id AND lease_id = :lease_id AND status = 'IN_FLIGHT'",
+            {"now": now, "id": message_id, "lease_id": lease_id},
+        )
+        if cursor.rowcount == 0:
+            logger.warning(
+                "mark_delivered: message %s no longer owned by lease %s (stale lease?)",
+                message_id,
+                lease_id,
+            )
+            return
+        await conn.execute("UPDATE counters SET value = value + 1 WHERE key = 'delivered_total'")
+        await conn.execute("DELETE FROM messages WHERE id = :id", {"id": message_id})
+
+    async def schedule_retry(
+        self,
+        message_id: str,
+        lease_id: str,
+        error: str,
+        next_attempt_at: datetime,
+    ) -> None:
+        conn = self._connection
+        now = _dt_to_str(self._clock.now())
+        cursor = await conn.execute(
+            """
+            UPDATE messages
+            SET status = 'PENDING',
+                attempts = attempts + 1,
+                last_error = :error,
+                last_attempt_at = :now,
+                next_attempt_at = :next_attempt_at,
+                lease_id = NULL,
+                lease_until = NULL,
+                updated_at = :now
+            WHERE id = :id AND lease_id = :lease_id AND status = 'IN_FLIGHT'
+            """,
+            {
+                "error": error[:2000],
+                "now": now,
+                "next_attempt_at": _dt_to_str(next_attempt_at),
+                "id": message_id,
+                "lease_id": lease_id,
+            },
+        )
+        if cursor.rowcount == 0:
+            logger.warning(
+                "schedule_retry: message %s no longer owned by lease %s (stale lease?)",
+                message_id,
+                lease_id,
+            )
+
+    async def move_to_dead_letter(self, message_id: str, lease_id: str, error: str) -> None:
+        conn = self._connection
+        now = _dt_to_str(self._clock.now())
+        cursor = await conn.execute(
+            """
+            UPDATE messages
+            SET status = 'DEAD_LETTER',
+                last_error = :error,
+                last_attempt_at = :now,
+                updated_at = :now,
+                lease_id = NULL,
+                lease_until = NULL
+            WHERE id = :id AND lease_id = :lease_id AND status = 'IN_FLIGHT'
+            """,
+            {"error": error[:2000], "now": now, "id": message_id, "lease_id": lease_id},
+        )
+        if cursor.rowcount == 0:
+            logger.warning(
+                "move_to_dead_letter: message %s no longer owned by lease %s (stale lease?)",
+                message_id,
+                lease_id,
+            )
+        else:
+            logger.warning("message %s moved to dead-letter: %s", message_id, error)
+
+    # -- recovery / expiry -------------------------------------------------
+
+    async def recover_expired_leases(self) -> int:
+        conn = self._connection
+        now = _dt_to_str(self._clock.now())
+        cursor = await conn.execute(
+            """
+            UPDATE messages
+            SET status = 'PENDING', lease_id = NULL, lease_until = NULL, updated_at = :now
+            WHERE status = 'IN_FLIGHT' AND lease_until <= :now
+            """,
+            {"now": now},
+        )
+        return cursor.rowcount
+
+    async def expire_stale_messages(self) -> int:
+        conn = self._connection
+        now = _dt_to_str(self._clock.now())
+        cursor = await conn.execute(
+            """
+            UPDATE messages
+            SET status = 'DEAD_LETTER', last_error = 'message expired', updated_at = :now
+            WHERE status = 'PENDING' AND expires_at IS NOT NULL AND expires_at <= :now
+            """,
+            {"now": now},
+        )
+        return cursor.rowcount
+
+    # -- stats / dead-letter management ------------------------------------
+
+    async def get_stats(self) -> QueueStats:
+        conn = self._connection
+        now = _dt_to_str(self._clock.now())
+        cursor = await conn.execute(
+            """
+            SELECT
+                SUM(CASE WHEN status = 'PENDING' AND next_attempt_at <= :now THEN 1 ELSE 0 END),
+                SUM(CASE WHEN status = 'PENDING' AND next_attempt_at > :now THEN 1 ELSE 0 END),
+                SUM(CASE WHEN status = 'IN_FLIGHT' THEN 1 ELSE 0 END),
+                SUM(CASE WHEN status = 'DEAD_LETTER' THEN 1 ELSE 0 END)
+            FROM messages
+            """,
+            {"now": now},
+        )
+        row = await cursor.fetchone()
+        assert row is not None  # aggregate SUM(...) always returns exactly one row
+        pending, retrying, in_flight, dead_letter = (value or 0 for value in row)
+
+        counter_cursor = await conn.execute(
+            "SELECT value FROM counters WHERE key = 'delivered_total'"
+        )
+        counter_row = await counter_cursor.fetchone()
+        delivered_total = counter_row[0] if counter_row else 0
+
+        storage_bytes = None
+        if self._path != ":memory:" and os.path.exists(self._path):
+            storage_bytes = os.path.getsize(self._path)
+
+        return QueueStats(
+            pending=pending,
+            in_flight=in_flight,
+            delivered=delivered_total,
+            dead_letter=dead_letter,
+            retrying=retrying,
+            storage_bytes=storage_bytes,
+        )
+
+    async def get_message(self, message_id: str) -> Message | None:
+        conn = self._connection
+        cursor = await conn.execute("SELECT * FROM messages WHERE id = :id", {"id": message_id})
+        row = await cursor.fetchone()
+        return self._row_to_message(row) if row else None
+
+    async def list_dead_letters(self, limit: int = 100, offset: int = 0) -> list[Message]:
+        conn = self._connection
+        cursor = await conn.execute(
+            "SELECT * FROM messages WHERE status = 'DEAD_LETTER' "
+            "ORDER BY updated_at DESC LIMIT :limit OFFSET :offset",
+            {"limit": limit, "offset": offset},
+        )
+        rows = await cursor.fetchall()
+        return [self._row_to_message(row) for row in rows]
+
+    async def retry_dead_letter(self, message_id: str) -> None:
+        conn = self._connection
+        now = _dt_to_str(self._clock.now())
+        await conn.execute(
+            """
+            UPDATE messages
+            SET status = 'PENDING', attempts = 0, last_error = NULL,
+                next_attempt_at = :now, updated_at = :now
+            WHERE id = :id AND status = 'DEAD_LETTER'
+            """,
+            {"now": now, "id": message_id},
+        )
+
+    async def delete_message(self, message_id: str) -> None:
+        conn = self._connection
+        await conn.execute("DELETE FROM messages WHERE id = :id", {"id": message_id})
+
+    # -- row <-> model mapping ----------------------------------------------
+
+    @staticmethod
+    def _message_to_row(message: Message) -> dict[str, object]:
+        return {
+            "id": message.id,
+            "destination": message.destination,
+            "payload": json.dumps(message.payload),
+            "headers": json.dumps(message.headers),
+            "metadata": json.dumps(message.metadata),
+            "priority": message.priority,
+            "status": message.status.value,
+            "attempts": message.attempts,
+            "created_at": _dt_to_str(message.created_at),
+            "updated_at": _dt_to_str(message.updated_at),
+            "next_attempt_at": _dt_to_str(message.next_attempt_at),
+            "last_attempt_at": _dt_to_str(message.last_attempt_at),
+            "delivered_at": _dt_to_str(message.delivered_at),
+            "last_error": message.last_error,
+            "lease_id": message.lease_id,
+            "lease_until": _dt_to_str(message.lease_until),
+            "expires_at": _dt_to_str(message.expires_at),
+            "size_bytes": message.size_bytes,
+        }
+
+    @staticmethod
+    def _row_to_message(row: aiosqlite.Row) -> Message:
+        assert row["next_attempt_at"] is not None
+        assert row["created_at"] is not None
+        assert row["updated_at"] is not None
+        return Message(
+            id=row["id"],
+            destination=row["destination"],
+            payload=json.loads(row["payload"]),
+            headers=json.loads(row["headers"]),
+            metadata=json.loads(row["metadata"]),
+            priority=row["priority"],
+            status=MessageStatus(row["status"]),
+            attempts=row["attempts"],
+            created_at=_str_to_dt(row["created_at"]),  # type: ignore[arg-type]
+            updated_at=_str_to_dt(row["updated_at"]),  # type: ignore[arg-type]
+            next_attempt_at=_str_to_dt(row["next_attempt_at"]),  # type: ignore[arg-type]
+            last_attempt_at=_str_to_dt(row["last_attempt_at"]),
+            delivered_at=_str_to_dt(row["delivered_at"]),
+            last_error=row["last_error"],
+            lease_id=row["lease_id"],
+            lease_until=_str_to_dt(row["lease_until"]),
+            expires_at=_str_to_dt(row["expires_at"]),
+            size_bytes=row["size_bytes"],
+        )

edgesync/transports/__init__.py

@@ -0,0 +1,7 @@
+"""Delivery transports."""
+
+from edgesync.transports.base import Transport
+from edgesync.transports.http import HTTPTransport
+from edgesync.transports.registry import TransportRegistry
+
+__all__ = ["Transport", "HTTPTransport", "TransportRegistry"]

edgesync/transports/base.py

@@ -0,0 +1,30 @@
+"""The transport abstraction.
+
+Transports are responsible only for attempting delivery of a single message
+and classifying the outcome. They must never touch storage or retry
+scheduling -- that is the worker's responsibility, driven by the
+``DeliveryResult`` a transport returns.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from edgesync.models.delivery import DeliveryResult
+from edgesync.models.message import Message
+
+
+class Transport(ABC):
+    """A destination EdgeSync can deliver messages to."""
+
+    @abstractmethod
+    async def start(self) -> None:
+        """Acquire any resources needed for delivery (e.g. open a client)."""
+
+    @abstractmethod
+    async def deliver(self, message: Message) -> DeliveryResult:
+        """Attempt to deliver a single message and classify the outcome."""
+
+    @abstractmethod
+    async def close(self) -> None:
+        """Release resources acquired in :meth:`start`."""

edgesync/transports/http.py

@@ -0,0 +1,112 @@
+"""Production HTTP transport backed by ``httpx``.
+
+A single ``httpx.AsyncClient`` is created in :meth:`start` and reused for
+every delivery -- connections are pooled, not reopened per message. The
+message ID is always sent via a configurable idempotency header (default
+``Idempotency-Key``) so destinations can de-duplicate retried deliveries,
+which is required for at-least-once delivery to be safe to build on.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+import httpx
+
+from edgesync.models.delivery import DeliveryResult
+from edgesync.models.message import Message
+from edgesync.transports.base import Transport
+
+DEFAULT_RETRYABLE_STATUS_CODES = frozenset({408, 425, 429, *range(500, 600)})
+_RESPONSE_SNIPPET_LIMIT = 500
+
+
+class HTTPTransport(Transport):
+    """Deliver messages as JSON over HTTP."""
+
+    def __init__(
+        self,
+        url: str,
+        *,
+        method: str = "POST",
+        timeout: float = 30.0,
+        headers: dict[str, str] | None = None,
+        idempotency_header: str = "Idempotency-Key",
+        success_status_codes: Iterable[int] | None = None,
+        retryable_status_codes: Iterable[int] = DEFAULT_RETRYABLE_STATUS_CODES,
+        verify: bool = True,
+        client: httpx.AsyncClient | None = None,
+    ) -> None:
+        self._url = url
+        self._method = method
+        self._timeout = timeout
+        self._default_headers = dict(headers or {})
+        self._idempotency_header = idempotency_header
+        self._success_status_codes = (
+            frozenset(success_status_codes)
+            if success_status_codes is not None
+            else None  # None means "any 2xx"
+        )
+        self._retryable_status_codes = frozenset(retryable_status_codes)
+        self._verify = verify
+        self._external_client = client
+        self._client: httpx.AsyncClient | None = None
+
+    async def start(self) -> None:
+        if self._client is not None:
+            return
+        if self._external_client is not None:
+            self._client = self._external_client
+        else:
+            self._client = httpx.AsyncClient(
+                timeout=self._timeout,
+                verify=self._verify,
+            )
+
+    async def close(self) -> None:
+        if self._client is not None and self._external_client is None:
+            await self._client.aclose()
+        self._client = None
+
+    async def deliver(self, message: Message) -> DeliveryResult:
+        if self._client is None:
+            raise RuntimeError("HTTPTransport.start() must be called before deliver()")
+
+        headers = {**self._default_headers, **message.headers}
+        headers[self._idempotency_header] = message.id
+
+        try:
+            response = await self._client.request(
+                self._method,
+                self._url,
+                json=message.payload,
+                headers=headers,
+                timeout=self._timeout,
+            )
+        except httpx.TimeoutException as exc:
+            return DeliveryResult.retryable_failure(f"timeout: {exc}")
+        except httpx.ConnectError as exc:
+            return DeliveryResult.retryable_failure(f"connection failed: {exc}")
+        except httpx.TransportError as exc:
+            # Covers DNS failures, network errors, protocol errors, etc.
+            return DeliveryResult.retryable_failure(f"transport error: {exc}")
+
+        return self._classify(response)
+
+    def _classify(self, response: httpx.Response) -> DeliveryResult:
+        status = response.status_code
+        is_success = (
+            status in self._success_status_codes
+            if self._success_status_codes is not None
+            else 200 <= status < 300
+        )
+        if is_success:
+            return DeliveryResult.success_result(status_code=status)
+
+        snippet = response.text[:_RESPONSE_SNIPPET_LIMIT]
+        error = f"HTTP {status}"
+        if status in self._retryable_status_codes:
+            return DeliveryResult.retryable_failure(
+                error, status_code=status, response_snippet=snippet
+            )
+        return DeliveryResult.permanent_failure(error, status_code=status, response_snippet=snippet)