talus 1.0.0__py3-none-any.whl

talus/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '1.0.0'
+__version_tuple__ = version_tuple = (1, 0, 0)

talus/base.py

@@ -0,0 +1,117 @@
+"""
+Base wrapper definition shared by both consumer and producer wrappers
+"""
+import logging
+from typing import Callable
+
+import pika
+from tenacity import Retrying
+
+from talus.models.connection_parameters import ConnectionParameterFactory
+from talus.models.queue import Queue
+from talus.models.retryer import ConnectionRetryerFactory
+
+logger = logging.getLogger(__name__)
+
+
+class DurableConnection:
+    """
+    RabbitMQ connector that establishes a blocking connection and channel
+    """
+
+    def __init__(
+        self,
+        connection_parameters: pika.ConnectionParameters
+        | Callable[[], pika.ConnectionParameters] = ConnectionParameterFactory(),
+        connection_retryer: Retrying | Callable[[], Retrying] = ConnectionRetryerFactory(),
+    ):
+        """
+        Constructor for a durable connection to RabbitMQ.
+        :param connection_parameters: A pika.ConnectionParameters object or one resulting
+            from a callable that returns a ConnectionParameters object.
+        :param connection_retryer: A tenacity.Retrying object or one resulting from a callable that returns
+            a Retrying object.
+        """
+        if isinstance(connection_parameters, pika.ConnectionParameters):
+            self.connection_parameters = connection_parameters
+        else:  # factory
+            self.connection_parameters = connection_parameters()
+        if isinstance(connection_retryer, Retrying):
+            self.connection_retryer = connection_retryer
+        else:  # factory
+            self.connection_retryer = connection_retryer()
+        self.connection: pika.BlockingConnection | None = None
+        self.channel: pika.adapters.blocking_connection.BlockingChannel | None = None
+
+    def connect(self):
+        """
+        Connect to the RabbitMQ server retrying errors configured in the connection_retryer.
+        """
+        if not self.is_connected:
+            self.connection_retryer(self._connect)
+
+    def _connect(self):
+        """
+        Configures and initiates connection to the RabbitMQ server.
+        """
+        logger.debug(
+            f"Attempt to connect to RabbitMQ: connection_params={self.connection_parameters}"
+        )
+        self.connection = pika.BlockingConnection(self.connection_parameters)
+        logger.info(f"Connection Created")
+        self.channel = self.connection.channel()
+        logger.info("Channel Created")
+        logger.info(f"Connected to RabbitMQ: connection={self.connection_parameters}")
+
+    @property
+    def is_connected(self):
+        """
+        Current state of the connection.  Only updated when the connection is used.
+
+        :return: Latest connection state
+        """
+        if self.connection is not None:
+            return self.connection.is_open
+        return False
+
+    def disconnect(self):
+        """
+        Closes connection and related channels to the RabbitMQ Server.
+        """
+
+        if self.is_connected:
+            self.connection.close()
+        logger.info(f"Disconnected from RabbitMQ: " f"connection={self.connection_parameters}")
+
+    def create_queue(self, queue: Queue):
+        """
+        Create a queue on an already opened connection.
+        """
+        self.channel.queue_declare(
+            queue=queue.name,
+            durable=queue.durable,
+            passive=queue.passive,
+            auto_delete=queue.auto_delete,
+            exclusive=queue.exclusive,
+            arguments=queue.arguments,
+        )
+        logger.info(f"Queue Created: queue={queue.name}")
+
+    def __enter__(self):
+        """
+        Entry for context manager.
+
+        :return: connected instance of self
+        """
+        self.connect()
+        return self
+
+    def __exit__(self, exc_type, value, traceback):
+        """
+        Exit for context manager which disconnects from rabbitmq
+        """
+        self.disconnect()
+
+    def __repr__(self):
+        """Representation of the DurableConnection object"""
+        return f"{self.__class__.__name__}(connection_parameters={self.connection_parameters!r}), connection_retryer={self.connection_retryer!r})"

talus/consumer.py

@@ -0,0 +1,179 @@
+"""
+Consumer implementation of the base connection wrapper
+"""
+import logging
+from typing import Callable
+from typing import Generator
+
+import pika.adapters.blocking_connection
+from tenacity import Retrying
+
+from talus.base import DurableConnection
+from talus.models.connection_parameters import ConsumerConnectionParameterFactory
+from talus.models.queue import Queue
+from talus.models.retryer import ConnectionRetryerFactory
+
+logger = logging.getLogger(__name__)
+
+
+LISTEN_CALLBACK_TYPE = Callable[
+    [
+        pika.adapters.blocking_connection.BlockingChannel,
+        pika.spec.Basic.Deliver,
+        pika.spec.BasicProperties,
+        bytes,
+    ],
+    None,
+]
+
+
+class DurableConsumer(DurableConnection):
+    """
+    RabbitMQ connector for consuming from a single queue in RabbitMQ.
+    >>> from talus.models.queue import Queue
+    >>> from talus.models.connection_parameters import ConsumerConnectionParameterFactory
+    >>> from talus.models.retryer import ConnectionRetryerFactory
+    >>> from talus.consumer import DurableConsumer
+    >>> from talus.models.processor import MessageProcessorBase
+    >>> from talus.models.message import ConsumeMessageBase
+    >>> consume_queue = Queue(name="test_queue")
+    >>> class MessageProcessor(MessageProcessorBase):
+    >>>     def process_message(message: ConsumeMessageBase):
+    >>>         print(message)
+    >>> with DurableConsumer(consume_queue=consume_queue, prefetch_count=1, connection_parameters=ConsumerConnectionParameterFactory(), connection_retryer=ConnectionRetryerFactory()) as consumer:
+    >>>     with MessageProcessor() as message_processor:
+    >>>         consumer.listen(message_processor)
+    """
+
+    def __init__(
+        self,
+        consume_queue: Queue,
+        prefetch_count: int = 1,
+        connection_parameters: pika.ConnectionParameters
+        | Callable[[], pika.ConnectionParameters] = ConsumerConnectionParameterFactory(),
+        connection_retryer: Retrying | Callable[[], Retrying] = ConnectionRetryerFactory(),
+    ):
+        """
+        Constructor for the consumer connector
+
+        :param consume_queue: Queue to consume messages from
+
+        :param prefetch_count: Number of un-Acked message delivered at a time
+
+        :param connection_parameters: A pika.ConnectionParameters object or one resulting
+            from a callable that returns a ConnectionParameters object.
+
+        :param connection_retryer: A tenacity.Retrying object or one resulting from a callable that returns
+            a Retrying object.
+        """
+        super().__init__(
+            connection_parameters=connection_parameters,
+            connection_retryer=connection_retryer,
+        )
+        self.consume_queue = consume_queue
+        self.prefetch_count = prefetch_count
+
+    def _connect(self) -> None:
+        """
+        Configures and initiates consumer connection to the RabbitMQ server which includes
+        setting up the queue to consume from and the prefetch count.
+        """
+        super()._connect()
+        self.channel.basic_qos(prefetch_count=self.prefetch_count)
+        self.create_queue(queue=self.consume_queue)
+
+    def _listen(self, callback: LISTEN_CALLBACK_TYPE) -> None:
+        """
+        Listens for messages on the channel configured on the consumer instance
+
+        :param callback: Function to execute when a message is received. with the signature
+        (ch, method, properties, body).
+        ch: Copy of the channel used to acknowledge receipt (pika.Channel)
+        method: Management keys for the delivered message e.g. delivery mode (pika.spec.Basic.Deliver)
+        properties: Message properties (pika.spec.BasicProperties)
+        body: Message body for a transfer message (bytes)
+        """
+        self.connect()
+        logger.info(f"Starting Listener on Queue: consumer_queue={self.consume_queue}")
+        self.channel.basic_consume(queue=self.consume_queue.name, on_message_callback=callback)
+        self.channel.start_consuming()
+
+    def listen(self, message_processor: LISTEN_CALLBACK_TYPE) -> None:
+        """
+        Retries calls to _listen and executes the message_processor callback when a message is received.
+        This method is blocking and will not return until the connection is closed.
+        :param message_processor: Callable to execute when a message is received. with the signature
+        (channel, method, properties, body).
+        channel: Copy of the channel used to acknowledge receipt (pika.Channel)
+        method: Management keys for the delivered message e.g. delivery mode (pika.spec.Basic.Deliver)
+        properties: Message properties (pika.spec.BasicProperties)
+        body: Message body for a transfer message (bytes)
+        and returns None.
+        """
+        self.connection_retryer(self._listen, callback=message_processor)
+
+    def consume_generator(self, auto_ack=False, inactivity_timeout: float = 0.1) -> Generator:
+        """
+        Creates a generator for messages that are on the instance consumer_queue.
+        Retry logic is not applied to prevent the resetting of the generator cursor
+
+        :param auto_ack: Automatically acknowledge messages
+        :param inactivity_timeout: Number of seconds to wait for a message before returning None
+
+        :return: Generator of (method, properties, body)
+        """
+        self.connect()
+        logger.info(f"Creating consumer generator on Queue: consumer_queue={self.consume_queue}")
+        return self.channel.consume(
+            queue=self.consume_queue.name, auto_ack=auto_ack, inactivity_timeout=inactivity_timeout
+        )
+
+    def cancel_consume_generator(self) -> None:
+        """
+        Resets the active consume generator
+        :return: None
+        """
+        logger.info(f"Cancelling consumer generator on Queue: consumer_queue={self.consume_queue}")
+        self.channel.cancel()
+
+    def acknowledge_message(self, delivery_tag, multiple=False) -> None:
+        """
+        Record a message as acknowledged.
+        Retry logic is not applied since creating a new channel would be unable
+        to acknowledge the message received on the now dead channel
+
+        :param delivery_tag: method.delivery_tag
+
+        :param multiple: Acknowledge multiple messages by setting to True and acknowledging the last message
+
+        :return: None
+        """
+        self.channel.basic_ack(delivery_tag, multiple)
+
+    def reject_message(self, delivery_tag) -> None:
+        """
+        Record a message as rejected.  Will go to dead letter exchange if configured on the server.
+        Retry logic is not applied since creating a new channel would be unable
+        to use the delivery tag received on the now dead channel
+
+        :param delivery_tag: method.delivery_tag
+
+        :return: None
+        """
+        self.channel.basic_reject(delivery_tag=delivery_tag, requeue=False)
+
+    def requeue_message(self, delivery_tag) -> None:
+        """
+        Return message back to the queue.
+        Retry logic is not applied since creating a new channel would be unable
+        to use the delivery tag received on the now dead channel
+
+        :param delivery_tag: method.delivery_tag
+
+        :return: None
+        """
+        self.channel.basic_nack(delivery_tag=delivery_tag, requeue=True)
+
+    def __repr__(self):
+        """Representation of the DurableConsumer instance."""
+        return f"{self.__class__.__name__}(consume_queue={self.consume_queue !r}, prefetch_count={self.prefetch_count}, connection_parameters={self.connection_parameters!r}, connection_retryer={self.connection_retryer!r})"

talus/models/binding.py

@@ -0,0 +1,46 @@
+"""
+Bindings contain the many-to-many relationship between a message routing key and a queue on an exchange.
+"""
+from typing import Type
+
+from pydantic import AfterValidator
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from typing_extensions import Annotated
+
+from talus.models.message import PublishMessageBase
+from talus.models.queue import Queue
+
+
+def validate_routing_key(message: Type[PublishMessageBase]):
+    """Custom pydantic validator to ensure that the routing key is not empty."""
+    if not message.default_routing_key:
+        raise ValueError("default_routing_key must not be empty")
+    return message
+
+
+class Binding(BaseModel):
+    """
+    Binding configuration.
+
+    >>> from talus.models.binding import Binding
+    >>> from talus.models.message import PublishMessageBase
+    >>> from talus.models.queue import Queue
+    >>> queue = Queue(name="my.queue")
+    >>> class PublishMessage(PublishMessageBase):
+    >>>     default_routing_key = "my.routing.key"
+    >>> binding = Binding(message=PublishMessage, queue=queue)
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    message: Annotated[Type[PublishMessageBase], AfterValidator(validate_routing_key)]
+    queue: Queue
+
+    @property
+    def routing_key(self) -> str:
+        return self.message.default_routing_key
+
+    @property
+    def queue_name(self) -> str:
+        return self.queue.name

talus/models/connection_parameters.py

@@ -0,0 +1,53 @@
+"""Connection Parameter Factory Model"""
+import pika
+from pydantic import BaseModel
+from pydantic import Field
+
+
+MINUTES_1 = 60
+HOUR_1 = 3600
+
+
+class ConnectionParameterFactory(BaseModel):
+    """
+    Translator for the connection configuration to a pika.ConnectionParameters object.
+    >>> from talus.models.connection_parameters import ConnectionParameterFactory
+    >>> factory = ConnectionParameterFactory()
+    >>> connection_parameters = factory()
+    """
+
+    rabbitmq_host: str = Field(
+        default="127.0.0.1"
+    )  # Host name or IP of the rabbitMQ server. e.g. 127.0.0.1
+    rabbitmq_port: int = 5672  # Port the rabbitmq server listens on e.g. 5672
+    rabbitmq_user: str = Field(default="guest")  # Username for the rabbitMQ server e.g. guest
+    rabbitmq_pass: str = Field(default="guest")  # Password for the rabbitMQ server e.g. guest
+    connection_name: str = Field(
+        default="default_connection"
+    )  # Name of the connection that will be visible in the rabbitmq admin console
+    # Controls AMQP heartbeat timeout negotiation
+    # during connection tuning. An integer value always overrides the value
+    # proposed by broker. Use 0 to deactivate heartbeats and None to always
+    # accept the broker's proposal.
+    heartbeat: int | None = None
+
+    def __call__(self) -> pika.ConnectionParameters:
+        return pika.ConnectionParameters(
+            host=self.rabbitmq_host,
+            port=self.rabbitmq_port,
+            credentials=pika.credentials.PlainCredentials(self.rabbitmq_user, self.rabbitmq_pass),
+            heartbeat=self.heartbeat,
+            client_properties={"connection_name": self.connection_name},
+        )
+
+
+class ConsumerConnectionParameterFactory(ConnectionParameterFactory):
+    """Consumer Connection Parameter Factory Model which defaults to a 1 minute heartbeat."""
+
+    heartbeat: int = MINUTES_1
+
+
+class ProducerConnectionParameterFactory(ConnectionParameterFactory):
+    """Producer Connection Parameter Factory Model which defaults to a 1 hour heartbeat."""
+
+    heartbeat: int = HOUR_1

talus/models/exchange.py

@@ -0,0 +1,23 @@
+"""Exchange models."""
+from pika.exchange_type import ExchangeType
+from pydantic import BaseModel
+
+
+class Exchange(BaseModel):
+    """
+    Exchange model.
+
+    >>> from talus.models.exchange import Exchange
+    >>> exchange = Exchange(name="my.exchange")
+    """
+
+    name: str = "default.x"
+    type: ExchangeType = ExchangeType.direct
+    passive: bool = False
+    durable: bool = True
+    auto_delete: bool = False
+    internal: bool = False
+    arguments: dict | None = None
+
+    def __str__(self):
+        return self.name

talus/models/message.py

@@ -0,0 +1,125 @@
+"""
+Module encapsulating message structure to facilitate use with the Consumer and Producer wrappers
+"""
+import uuid
+from typing import Type
+
+import pika.spec
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+
+
+class MessageBodyBase(BaseModel):
+    """
+    Base class for message schemas which can be used to validate message bodies.
+    This is a pydantic.BaseModel so each message body schema can be validated.  Extend this class
+    to add fields to a new message body schema.  For Example:
+    >>> class ObjectBucketBody(MessageBodyBase):
+    >>>     objectName: str
+    >>>     bucket: str
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    conversationId: str = Field(default_factory=lambda: uuid.uuid4().hex)
+
+
+class _MessageBase:
+    """
+    Base class for messages establishing a common interface for use by DurableConnections.
+    Not intended for direct use.  Use ConsumeMessageBase or PublishMessageBase instead.
+    """
+
+    message_body_cls: Type[MessageBodyBase] = MessageBodyBase
+
+    def __init__(self, routing_key: str, body: bytes | dict | str | MessageBodyBase):
+        self.routing_key = routing_key
+        if isinstance(body, dict | BaseModel):
+            self.body: MessageBodyBase = self.message_body_cls.model_validate(body)
+        else:  # str | bytes
+            self.body: MessageBodyBase = self.message_body_cls.model_validate_json(body)
+
+
+class ConsumeMessageBase(_MessageBase):
+    """
+    Base class for messages consumed from a DurableConsumer MessageProcessor.  Each instance
+    correlates with a single consumed message.  The body of the message is validated against the
+    message_body_cls class attribute.
+    >>> class ObjectBucketBody(MessageBodyBase):
+    >>>    objectName: str
+    >>>    bucket: str
+    >>>
+    >>> class InboundMessage(ConsumeMessageBase):
+    >>>     message_body_cls: Type[ObjectBucketBody] = ObjectBucketBody  # Validate the body with ObjectBucketBody
+    >>>
+    >>> queue = Queue("my_queue")
+    >>> with DurableConsumer(consume_queue=queue) as consumer:
+    >>>     consumer.consume(lambda ch, method, properties, body: InboundMessage(method, properties, body))
+    """
+
+    def __init__(
+        self,
+        method: pika.spec.Basic.Deliver,
+        properties: pika.spec.BasicProperties,
+        body: bytes | dict | str | MessageBodyBase,
+    ):
+        super().__init__(method.routing_key, body)
+        self.method = method
+        self.properties = properties
+
+    @property
+    def delivery_tag(self):
+        return self.method.delivery_tag
+
+    @property
+    def headers(self) -> dict | None:
+        return self.properties.headers
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(method={self.method!r}, properties={self.properties!r}, body={self.body!r})"
+
+
+class PublishMessageBase(_MessageBase):
+    """
+    Base class for messages published to with a DurableProducer.
+    Each instance correlates with a single message to publish.  The body of the message is validated
+    against the class defined in message_body_cls class attribute.  Additional class attributes for
+    routing_key and headers are provided for customizing the message at the class level e.g. all
+    instances share a routing_key.
+
+    >>> class OutboundMessage(PublishMessageBase):
+    >>>     message_body_cls: Type[ObjectBucketBody] = ObjectBucketBody
+    >>>     default_routing_key: str = "outbound.m"
+    >>>     headers: dict[str, str] = Field(default={"header1": "value1"})
+    >>>
+    >>>
+    >>> queue = Queue(name="test_queue")
+    >>> exchange = Exchange(name="test_exchange")
+    >>> queue_bindings = Binding(queue=queue, message=OutboundMessage)
+    >>> with DurableProducer(queue_bindings=queue_bindings, publish_exchange=exchange) as producer:
+    >>>     body = {"objectName": "object", "bucket": "bucket"}
+    >>>     producer.publish(OutboundMessage(body))
+    """
+
+    default_routing_key: str = "default.m"
+    headers: dict[str, str] | None = None
+
+    def __init__(
+        self,
+        body: bytes | dict | str | MessageBodyBase,
+    ):
+        super().__init__(self.default_routing_key, body)
+
+    @property
+    def properties(self) -> pika.spec.BasicProperties:
+        return pika.BasicProperties(
+            content_type="text/plain",
+            priority=0,
+            delivery_mode=pika.DeliveryMode.Persistent,
+            content_encoding="UTF-8",
+            headers=self.headers,
+        )
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(body={self.body!r}) # {self.routing_key = }"