xtr-orm-messenger 6.0.0__py3-none-any.whl

xtr_orm_messenger/transport/orm_options.py

@@ -0,0 +1,145 @@
+"""The settings an ``orm://`` transport reads from its configuration."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Final
+
+from xtr_messenger import InvalidTransportOptionError
+from xtr_messenger.transport.transport_options import (
+    as_bool,
+    as_float,
+    as_int,
+    reject_unknown_options,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+__all__ = ["DEFAULT_QUEUE", "DEFAULT_TABLE_NAME", "ORM_OPTIONS", "ORM_SCHEME", "OrmOptions"]
+
+ORM_SCHEME: Final = "orm"
+"""The DSN scheme the transport serves."""
+
+DEFAULT_TABLE_NAME: Final = "messenger_messages"
+"""The table messages are kept in when the configuration names none."""
+
+DEFAULT_QUEUE: Final = "default"
+"""The queue a transport reads and writes when the configuration names none."""
+
+ORM_OPTIONS: Final = (
+    "auto_setup",
+    "check_delayed_interval",
+    "get_notify_timeout",
+    "poll_interval",
+    "queue",
+    "redeliver_timeout",
+    "table_name",
+    "transactional",
+    "use_notify",
+)
+"""Every setting an ``orm://`` transport accepts; any other is refused."""
+
+_QUEUE_LENGTH: Final = 190
+"""The width of the ``queue_name`` column, which indexes on every database at that size."""
+
+_IDENTIFIER: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+"""A table name: a plain identifier, so it names one table and one notification channel."""
+
+
+@dataclass(frozen=True, slots=True)
+class OrmOptions:
+    """How an ``orm://`` transport keeps, hands out and hands back its messages.
+
+    Attributes:
+        table_name: The table messages are kept in.
+        queue: The queue this transport reads and writes; transports sharing a
+            table but naming different queues never see each other's messages.
+        redeliver_timeout: Seconds after which a message handed to a worker
+            and never settled is handed out again — the worker is taken to have
+            died with it.
+        auto_setup: Create the table the first time an operation finds it
+            missing.
+        poll_interval: Seconds a worker waits before asking again when nothing
+            was due.
+        transactional: Write a message in the session of the unit of work
+            under way, so it commits or rolls back with the handler's own
+            changes; outside a unit it is written in a transaction of its own.
+        use_notify: On PostgreSQL, wake a waiting worker the moment a message
+            is sent instead of when its poll interval runs out.
+        check_delayed_interval: Milliseconds a worker waiting for a
+            notification waits at most before asking anyway — a delayed message
+            falling due sends none.
+        get_notify_timeout: Milliseconds one wait for a notification lasts at
+            most.
+    """
+
+    table_name: str = DEFAULT_TABLE_NAME
+    queue: str = DEFAULT_QUEUE
+    redeliver_timeout: int = 3600
+    auto_setup: bool = True
+    poll_interval: float = 1.0
+    transactional: bool = False
+    use_notify: bool = True
+    check_delayed_interval: int = 60_000
+    get_notify_timeout: int = 60_000
+
+    def __post_init__(self) -> None:
+        """Refuse values the transport would misread.
+
+        Raises:
+            InvalidTransportOptionError: When a name is not a plain identifier,
+                a queue is empty or too long for its column, or a duration is
+                not positive.
+        """
+        if _IDENTIFIER.fullmatch(self.table_name) is None:
+            raise InvalidTransportOptionError(
+                "table_name", self.table_name, "a plain identifier: letters, digits and '_'"
+            )
+        if not 0 < len(self.queue) <= _QUEUE_LENGTH:
+            raise InvalidTransportOptionError(
+                "queue", self.queue, f"between 1 and {_QUEUE_LENGTH} characters"
+            )
+        if self.redeliver_timeout <= 0:
+            raise InvalidTransportOptionError(
+                "redeliver_timeout", str(self.redeliver_timeout), "a positive number of seconds"
+            )
+        if self.poll_interval <= 0:
+            raise InvalidTransportOptionError(
+                "poll_interval", str(self.poll_interval), "a positive number of seconds"
+            )
+        for name, value in (
+            ("check_delayed_interval", self.check_delayed_interval),
+            ("get_notify_timeout", self.get_notify_timeout),
+        ):
+            if value <= 0:
+                raise InvalidTransportOptionError(
+                    name, str(value), "a positive number of milliseconds"
+                )
+
+    @classmethod
+    def from_settings(cls, settings: Mapping[str, str]) -> OrmOptions:
+        """Read the options out of a transport's settings, the defaults filling the rest.
+
+        Raises:
+            UnknownTransportOptionError: If a setting is not one this transport
+                accepts.
+            InvalidTransportOptionError: If a value is not what its setting
+                takes.
+        """
+        reject_unknown_options(ORM_SCHEME, settings, ORM_OPTIONS)
+        base = cls()
+        return cls(
+            table_name=settings.get("table_name", base.table_name),
+            queue=settings.get("queue", base.queue),
+            redeliver_timeout=as_int(settings, "redeliver_timeout", base.redeliver_timeout),
+            auto_setup=as_bool(settings, "auto_setup", base.auto_setup),
+            poll_interval=as_float(settings, "poll_interval", base.poll_interval),
+            transactional=as_bool(settings, "transactional", base.transactional),
+            use_notify=as_bool(settings, "use_notify", base.use_notify),
+            check_delayed_interval=as_int(
+                settings, "check_delayed_interval", base.check_delayed_interval
+            ),
+            get_notify_timeout=as_int(settings, "get_notify_timeout", base.get_notify_timeout),
+        )

xtr_orm_messenger/transport/orm_received_stamp.py

@@ -0,0 +1,22 @@
+"""Which row of the messages table an envelope was received from."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import final
+
+from xtr_messenger import NonSendableStampInterface
+
+__all__ = ["OrmReceivedStamp"]
+
+
+@final
+@dataclass(frozen=True, slots=True)
+class OrmReceivedStamp(NonSendableStampInterface):
+    """The row an envelope came from, which settling it deletes.
+
+    Never sent on: it names a row of this table, which means nothing to a
+    transport the envelope is dispatched to next.
+    """
+
+    id: str

xtr_orm_messenger/transport/orm_receiver.py

@@ -0,0 +1,214 @@
+"""The receive half of an ``orm://`` transport."""
+
+from __future__ import annotations
+
+import logging
+import secrets
+from typing import TYPE_CHECKING, Final, final
+
+from sqlalchemy.exc import DBAPIError
+from typing_extensions import override
+from xtr_clock import SystemClock
+from xtr_messenger import (
+    EncodedEnvelope,
+    MessageDecodingFailedError,
+    ReceivedStamp,
+    ReceiverInterface,
+    TransportMessageIdStamp,
+)
+
+from xtr_orm_messenger.exception import TransportError
+
+from ._database_errors import is_retryable
+from .orm_received_stamp import OrmReceivedStamp
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Awaitable, Callable
+
+    from xtr_clock import ClockInterface
+    from xtr_messenger import Envelope, SerializerInterface
+
+    from .connection import Connection
+    from .stored_message import StoredMessage
+
+__all__ = ["MAX_RETRIES", "RECONNECT_TIMEOUT", "OrmReceiver"]
+
+MAX_RETRIES: Final = 3
+"""How many conflicts in a row a poll, or a settle, sits through before it gives up."""
+
+RECONNECT_TIMEOUT: Final = 5
+"""Seconds a settle waits for a lost database to come back before it gives up.
+
+Shorter than a process supervisor usually gives a worker to stop.
+"""
+
+_FIRST_BACKOFF: Final = 0.1
+"""Seconds before a settle that ran into another transaction tries again; doubled each time."""
+
+#: Warnings go through the messenger channel, so an application capturing the
+#: standard library's logging sees them beside the rest of the bus's output.
+_LOGGER: Final = logging.getLogger("messenger")
+
+
+@final
+class OrmReceiver(ReceiverInterface):
+    """Claims the messages of one queue, and deletes each once it is settled.
+
+    Workers polling one table run into each other's transactions; a conflict
+    says nothing about the message, so a poll sits through
+    :data:`MAX_RETRIES` of them in a row before it gives up, and a settle
+    tries again with a growing pause. A settle whose database went away waits
+    up to :data:`RECONNECT_TIMEOUT` seconds for it and tries once more: losing
+    an acknowledgement to a database restart would have the message handed
+    out, and handled, again.
+    """
+
+    __slots__ = ("_clock", "_conflicts", "_connection", "_name", "_serializer")
+
+    def __init__(
+        self,
+        connection: Connection,
+        serializer: SerializerInterface,
+        name: str,
+        clock: ClockInterface | None = None,
+    ) -> None:
+        """Receive from ``connection`` as the transport ``name``, decoding with ``serializer``."""
+        self._connection = connection
+        self._serializer = serializer
+        self._name = name
+        self._clock: ClockInterface = clock if clock is not None else SystemClock()
+        self._conflicts = 0
+
+    @override
+    async def get(self) -> AsyncIterator[Envelope]:
+        """Yield each message as it falls due, waiting between polls that find none.
+
+        Never runs dry: a worker stops it by cancelling. A message is yielded
+        the moment it is claimed, with nothing awaited in between, so a
+        cancellation never loses one; and one lost anyway — the process
+        killed — is handed out again after ``redeliver_timeout``.
+
+        A message that does not decode can never be handled, so it is
+        rejected at once, its reason logged — never its body, which may carry
+        anything a producer put in a message.
+
+        Raises:
+            TransportError: When the database refuses a poll for any reason
+                but a conflict, or for a conflict :data:`MAX_RETRIES` times in
+                a row.
+        """
+        while True:
+            stored = await self._claim()
+            if stored is None:
+                await self._connection.wait()
+                continue
+            try:
+                envelope = self.decode(stored)
+            except MessageDecodingFailedError as error:
+                _LOGGER.warning(
+                    "rejecting message %s of the %r transport: it does not decode (%s)",
+                    stored.id,
+                    self._name,
+                    error.reason,
+                )
+                await self._settle(stored.id, self._connection.reject, "reject")
+                continue
+            yield envelope.with_stamps(ReceivedStamp(self._name))
+
+    @override
+    async def ack(self, envelope: Envelope) -> None:
+        """Delete the message ``envelope`` was received as: it was handled.
+
+        Raises:
+            TransportError: When ``envelope`` was not received from this
+                transport, or the database refuses the delete.
+        """
+        await self._settle(self._message_id(envelope), self._connection.ack, "acknowledge")
+
+    @override
+    async def reject(self, envelope: Envelope) -> None:
+        """Delete the message ``envelope`` was received as: it failed for good.
+
+        Raises:
+            TransportError: When ``envelope`` was not received from this
+                transport, or the database refuses the delete.
+        """
+        await self._settle(self._message_id(envelope), self._connection.reject, "reject")
+
+    def decode(self, stored: StoredMessage) -> Envelope:
+        """Rebuild the envelope ``stored`` keeps, stamped with the row it came from.
+
+        A transport message id the envelope carried from an earlier transport
+        is replaced by this row's, so the id it carries is the one that
+        settles it here.
+
+        Raises:
+            MessageDecodingFailedError: When the row does not decode.
+        """
+        envelope = self._serializer.decode(
+            EncodedEnvelope(body=stored.body, headers=dict(stored.headers))
+        )
+        return envelope.without_stamps(TransportMessageIdStamp).with_stamps(
+            OrmReceivedStamp(stored.id), TransportMessageIdStamp(stored.id)
+        )
+
+    async def _claim(self) -> StoredMessage | None:
+        """Claim the next message, or return ``None`` for nothing due or a conflict to sit out."""
+        try:
+            stored = await self._connection.get()
+        except DBAPIError as error:
+            if not is_retryable(error):
+                raise TransportError(self._name, "get messages", str(error.orig)) from error
+            self._conflicts += 1
+            if self._conflicts >= MAX_RETRIES:
+                self._conflicts = 0
+                raise TransportError(self._name, "get messages", str(error.orig)) from error
+            return None
+        self._conflicts = 0
+        return stored
+
+    async def _settle(
+        self, message_id: str, operation: Callable[[str], Awaitable[None]], action: str
+    ) -> None:
+        delay = _FIRST_BACKOFF
+        retries = 0
+        reconnected = False
+        while True:
+            try:
+                await operation(message_id)
+            except DBAPIError as error:
+                if is_retryable(error) and retries < MAX_RETRIES:
+                    retries += 1
+                    await self._clock.sleep_async(delay * _jitter())
+                    delay *= 2
+                    continue
+                if not reconnected and error.connection_invalidated and await self._reconnected():
+                    reconnected = True
+                    continue
+                raise TransportError(
+                    self._name, f"{action} message {message_id}", str(error.orig)
+                ) from error
+            return
+
+    async def _reconnected(self) -> bool:
+        """Wait up to :data:`RECONNECT_TIMEOUT` seconds for the database to answer again."""
+        for _ in range(RECONNECT_TIMEOUT):
+            await self._clock.sleep_async(1)
+            if await self._connection.ping():
+                return True
+        return False
+
+    def _message_id(self, envelope: Envelope) -> str:
+        stamp = envelope.last(OrmReceivedStamp)
+        if stamp is None:
+            raise TransportError(
+                self._name,
+                "settle a message",
+                "it carries no OrmReceivedStamp, so it was not received from this transport",
+            )
+        return stamp.id
+
+
+def _jitter() -> float:
+    """Return a factor within 10% of one, so workers that collided do not collide again."""
+    return 1 + (secrets.randbelow(21) - 10) / 100

xtr_orm_messenger/transport/orm_sender.py

@@ -0,0 +1,51 @@
+"""The send half of an ``orm://`` transport."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, final
+
+from sqlalchemy.exc import DBAPIError
+from typing_extensions import override
+from xtr_messenger import DelayStamp, SenderInterface, TransportMessageIdStamp
+
+from xtr_orm_messenger.exception import TransportError
+
+if TYPE_CHECKING:
+    from xtr_messenger import Envelope, SerializerInterface
+
+    from .connection import Connection
+
+__all__ = ["OrmSender"]
+
+
+@final
+class OrmSender(SenderInterface):
+    """Writes each envelope it is handed into the table, due after its delay."""
+
+    __slots__ = ("_connection", "_name", "_serializer")
+
+    def __init__(self, connection: Connection, serializer: SerializerInterface, name: str) -> None:
+        """Send through ``connection`` as the transport ``name``, encoding with ``serializer``."""
+        self._connection = connection
+        self._serializer = serializer
+        self._name = name
+
+    @override
+    async def send(self, envelope: Envelope) -> Envelope:
+        """Keep ``envelope``, due after its :class:`~xtr_messenger.DelayStamp`, if any.
+
+        Returns it stamped with the id the row was given.
+
+        Raises:
+            MessageEncodingFailedError: When the envelope does not encode.
+            TransportError: When the database refuses the insert.
+        """
+        encoded = self._serializer.encode(envelope)
+        delay = envelope.last(DelayStamp)
+        try:
+            message_id = await self._connection.send(
+                encoded.body, encoded.headers, delay.delay_ms if delay is not None else 0
+            )
+        except DBAPIError as error:
+            raise TransportError(self._name, "send a message", str(error.orig)) from error
+        return envelope.with_stamps(TransportMessageIdStamp(message_id))

xtr_orm_messenger/transport/orm_transport.py

@@ -0,0 +1,170 @@
+"""Both halves of an ``orm://`` transport, and what can be asked of its table."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, final
+
+from sqlalchemy.exc import DBAPIError
+from typing_extensions import override
+from xtr_messenger import TransportInterface
+
+from xtr_orm_messenger.exception import TransportError
+
+from .orm_received_stamp import OrmReceivedStamp
+from .orm_receiver import OrmReceiver
+from .orm_sender import OrmSender
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+    from xtr_clock import ClockInterface
+    from xtr_messenger import Envelope, SerializerInterface
+
+    from .connection import Connection
+
+__all__ = ["OrmTransport"]
+
+
+@final
+class OrmTransport(TransportInterface):
+    """Sends to, receives from and lists one queue of a messages table.
+
+    Beyond sending and receiving, the queue can be looked into — counted,
+    listed, one message found by id — and a message a long handler holds can
+    be kept from being handed out again.
+    """
+
+    __slots__ = ("_connection", "_name", "_receiver", "_sender")
+
+    def __init__(
+        self,
+        connection: Connection,
+        serializer: SerializerInterface,
+        *,
+        name: str = "orm",
+        clock: ClockInterface | None = None,
+    ) -> None:
+        """Use ``connection`` as the transport ``name``.
+
+        Both halves encode and decode with ``serializer``, so a producer and a
+        worker built from the same configuration always agree on the wire
+        format.
+        """
+        self._connection = connection
+        self._name = name
+        self._receiver = OrmReceiver(connection, serializer, name, clock)
+        self._sender = OrmSender(connection, serializer, name)
+
+    @property
+    def name(self) -> str:
+        """The transport's name, which every envelope it receives is stamped with."""
+        return self._name
+
+    @property
+    def connection(self) -> Connection:
+        """What runs this transport's statements."""
+        return self._connection
+
+    @override
+    async def send(self, envelope: Envelope) -> Envelope:
+        """Keep ``envelope`` until a worker claims it; see :meth:`OrmSender.send`."""
+        return await self._sender.send(envelope)
+
+    @override
+    def get(self) -> AsyncIterator[Envelope]:
+        """Yield each message as it falls due; see :meth:`OrmReceiver.get`."""
+        return self._receiver.get()
+
+    @override
+    async def ack(self, envelope: Envelope) -> None:
+        """Delete the message ``envelope`` was received as: it was handled."""
+        await self._receiver.ack(envelope)
+
+    @override
+    async def reject(self, envelope: Envelope) -> None:
+        """Delete the message ``envelope`` was received as: it failed for good."""
+        await self._receiver.reject(envelope)
+
+    async def keepalive(self, envelope: Envelope, seconds: int | None = None) -> None:
+        """Keep the message ``envelope`` was received as from being handed out again.
+
+        Restarts its redelivery clock: call it while a handler runs longer
+        than ``redeliver_timeout``. ``seconds`` is how long the caller means to
+        wait before calling again, refused when longer than the timeout, which
+        would hand the message out in between.
+
+        Raises:
+            TransportError: When ``seconds`` is longer than
+                ``redeliver_timeout``, ``envelope`` was not received from this
+                transport, or the database refuses the update.
+        """
+        timeout = self._connection.options.redeliver_timeout
+        if seconds is not None and seconds > timeout:
+            raise TransportError(
+                self._name,
+                "keep a message alive",
+                f"{seconds} seconds is longer than its redeliver_timeout of {timeout}",
+            )
+        stamp = envelope.last(OrmReceivedStamp)
+        if stamp is None:
+            raise TransportError(
+                self._name,
+                "keep a message alive",
+                "it carries no OrmReceivedStamp, so it was not received from this transport",
+            )
+        try:
+            await self._connection.keepalive(stamp.id)
+        except DBAPIError as error:
+            raise TransportError(self._name, "keep a message alive", str(error.orig)) from error
+
+    async def find(self, message_id: str) -> Envelope | None:
+        """Return the envelope kept as ``message_id`` in this queue, or ``None``.
+
+        Raises:
+            MessageDecodingFailedError: When the row does not decode.
+            TransportError: When the database refuses the query.
+        """
+        try:
+            stored = await self._connection.find(message_id)
+        except DBAPIError as error:
+            raise TransportError(self._name, "find a message", str(error.orig)) from error
+        return None if stored is None else self._receiver.decode(stored)
+
+    async def all(self, limit: int | None = None) -> AsyncIterator[Envelope]:
+        """Yield the envelopes a worker could claim now, oldest first, ``limit`` at most.
+
+        Raises:
+            MessageDecodingFailedError: When a row does not decode.
+            TransportError: When the database refuses the query.
+        """
+        try:
+            async for stored in self._connection.all(limit):
+                yield self._receiver.decode(stored)
+        except DBAPIError as error:
+            raise TransportError(self._name, "list messages", str(error.orig)) from error
+
+    async def count(self) -> int:
+        """Return how many messages a worker could claim now.
+
+        Raises:
+            TransportError: When the database refuses the query.
+        """
+        try:
+            return await self._connection.count()
+        except DBAPIError as error:
+            raise TransportError(self._name, "count messages", str(error.orig)) from error
+
+    async def setup(self) -> None:
+        """Create the messages table if it does not exist.
+
+        Raises:
+            TransportError: When the database refuses to create it.
+        """
+        try:
+            await self._connection.setup()
+        except DBAPIError as error:
+            raise TransportError(self._name, "create its table", str(error.orig)) from error
+
+    async def close(self) -> None:
+        """Release the connection a worker listens for notifications on, if any."""
+        await self._connection.close()