DefStream-RabbitMQ 0.1.28__123-py3-none-any.whl

DefStream/Broker/Implements/RabbitMQ/Consumer.py

@@ -0,0 +1,420 @@
+# -*- coding: utf-8 -*-
+
+from DefStream.Broker import (
+	Consumer as BaseConsumer,
+	Recoverable,
+	EventHandler,
+	Event as BaseEvent,
+)
+
+from .Serializer import (
+	Decoder,
+	Encoder,
+	JavaScriptObjectNotationEncoder,
+)
+from logging import getLogger
+
+from .Event import (
+	Event,
+	Requeue,
+)
+from .Subscription import Subscription
+from DefStream.Broker.Event import EventErrorPolicy
+from typing import (
+	Any,
+	Callable,
+	Optional,
+)
+
+from pika import (
+	BasicProperties,
+	BlockingConnection,
+)
+from pika.exceptions import (
+	AMQPConnectionError,
+	AuthenticationError,
+	ChannelClosedByBroker,
+	ConnectionClosedByBroker,
+	IncompatibleProtocolError,
+	ProbableAccessDeniedError,
+	ProbableAuthenticationError,
+	UnroutableError,
+)
+
+__all__ = (
+	'Consumer',
+)
+
+
+class ConsumptionInterrupted(RuntimeError):
+	pass
+
+
+class Consumer(
+	BaseConsumer,
+	Recoverable,
+):
+	"""Consumer of Event Broker for RabbitMQ"""
+	logger = getLogger('Broker.RabbitMQ.Consumer')
+
+	def __init__(
+		self,
+		connection: BlockingConnection,
+		decode: Decoder,
+		handler: Optional[EventHandler] = None,
+		qos: int = 1,
+		offset: Optional[int] = None,
+		encode: Optional[Encoder] = None,
+	):
+		self.connection = connection
+		self.decode = decode
+		self.qos = qos
+		try:
+			self.channel = self.connection.channel()
+			self.channel.basic_qos(
+				prefetch_count=qos,
+				global_qos=False,
+			)
+		except Exception as error:
+			self.logger.error('Consumption channel initialization failed: qos=%s error_type=%s', qos, type(error).__name__)
+			raise
+		self.logger.debug('Consumption channel created: qos=%s offset=%s', qos, offset)
+		self.offset = offset
+		self.handler = handler
+		self.encode = encode if encode is not None else JavaScriptObjectNotationEncoder()
+		self.response_channel = None
+		self.queues = set()
+		self.stopped = False
+		self.callback_error: Optional[BaseException] = None
+		return
+
+	def __del__(self):
+		self.close()
+		return
+
+	def subs(
+		self,
+		queue: Any,
+		handler: Optional[EventHandler] = None,
+	) -> None:
+		if handler is None:
+			handler = self.handler
+		if handler is None:
+			raise ValueError('subs requires a handler or a default handler')
+		if isinstance(
+			queue,
+			str,
+		):
+			subscription = Subscription(
+				queue=queue,
+				offset=self.offset,
+			)
+		else:
+			subscription = Subscription.parse(queue)
+		if subscription.queue in self.queues:
+			raise ValueError('A Run is already subscribed to queue {}'.format(subscription.queue))
+		qos = self.qos if subscription.qos is None else subscription.qos
+		self.logger.debug('Applying subscription: queue=%s qos=%s offset=%s', subscription.queue, qos, subscription.offset)
+		try:
+			self.channel.basic_qos(
+				prefetch_count=qos,
+				global_qos=False,
+			)
+			tag = self.channel.basic_consume(
+				subscription.queue,
+				Callback(
+					self.decode,
+					handler,
+					subscription.queue,
+					logger=self.logger,
+					owner=self,
+				),
+				auto_ack=False,
+				exclusive=False,
+				arguments={'x-stream-offset': subscription.offset} if subscription.offset is not None else None,
+			)
+		except Exception as error:
+			self.logger.error('Subscription failed: queue=%s error_type=%s', subscription.queue, type(error).__name__)
+			raise
+		self.queues.add(subscription.queue)
+		self.logger.debug('Subscription applied: queue=%s consumer_tag=%s', subscription.queue, tag)
+		return
+
+	def call(self, callback: Callable[[], None]) -> None:
+		self.logger.debug('Scheduling callback on connection owner thread')
+		try:
+			self.connection.add_callback_threadsafe(callback)
+		except Exception as error:
+			self.logger.error('Callback scheduling failed: error_type=%s', type(error).__name__)
+			raise
+		self.logger.debug('Callback scheduled on connection owner thread')
+		return
+
+	def fail(
+		self,
+		event: BaseEvent,
+		policy: Optional[EventErrorPolicy] = None,
+	) -> None:
+		self.logger.debug('Task failure policy selected: policy=%s', type(policy).__name__ if policy is not None else 'Requeue')
+		if isinstance(
+			event,
+			Event,
+		) and event.properties.reply_to:
+			event = FailedRequest(
+				self,
+				event,
+			)
+		(policy if policy is not None else Requeue())(event)
+		return
+
+	def complete(
+		self,
+		event: BaseEvent,
+		result: Any,
+	) -> None:
+		self.logger.debug('Task completion policy selected: ack')
+		if isinstance(
+			event,
+			Event,
+		) and event.properties.reply_to:
+			self.logger.debug('Building RPC success response: queue=%s correlation_id=%s', event.queue, event.properties.correlation_id)
+			self.respond(
+				event,
+				{
+					'ok': True,
+					'result': result,
+				},
+			)
+		event.ack()
+		return
+
+	def respond(
+		self,
+		event: Event,
+		response: Any,
+	) -> None:
+		if not event.properties.correlation_id:
+			self.logger.error('RPC response validation failed: missing correlation_id')
+			raise ValueError('RPC request requires correlation_id')
+		channel = self.response_channel
+		self.logger.debug('Attempting RPC response: correlation_id=%s reply_to=%s', event.properties.correlation_id, event.properties.reply_to)
+		if channel is None:
+			try:
+				channel = self.connection.channel()
+				channel.confirm_delivery()
+				self.response_channel = channel
+				self.logger.debug('RPC response channel created: publisher_confirm=True')
+			except Exception as error:
+				self.logger.error('RPC response channel creation failed: correlation_id=%s error_type=%s', event.properties.correlation_id, type(error).__name__)
+				raise
+		else:
+			self.logger.debug('Reusing RPC response channel')
+		try:
+			channel.basic_publish(
+				exchange='',
+				routing_key=event.properties.reply_to,
+				body=self.encode(response),
+				properties=BasicProperties(
+					correlation_id=event.properties.correlation_id,
+					content_type=self.encode.format,
+					content_encoding=self.encode.charset,
+				),
+				mandatory=True,
+			)
+		except UnroutableError:
+			self.logger.warning(
+				'RPC reply is unroutable; settling request without a response: correlation_id=%s reply_to=%s',
+				event.properties.correlation_id,
+				event.properties.reply_to,
+			)
+		except Exception as error:
+			self.logger.error('RPC response publication failed: correlation_id=%s error_type=%s', event.properties.correlation_id, type(error).__name__)
+			raise
+		else:
+			self.logger.debug('RPC response published: correlation_id=%s', event.properties.correlation_id)
+		return
+
+	def close(self) -> None:
+		response_channel = getattr(self, 'response_channel', None)
+		try:
+			if response_channel is not None and response_channel.is_open:
+				response_channel.close()
+				self.logger.debug('RPC response channel closed')
+			channel = getattr(self, 'channel', None)
+			if channel is not None and channel.is_open:
+				channel.close()
+				self.logger.debug('Consumption channel closed')
+		except Exception as error:
+			self.logger.error('Channel close failed: error_type=%s', type(error).__name__)
+			raise
+		return
+
+	def run(self):
+		self.logger.debug('Starting consumption loop')
+		self.callback_error = None
+		self.timer = self.connection.call_later(0.2, self.check_channel)
+		try:
+			self.channel.start_consuming()
+			if not self.stopped:
+				raise ConsumptionInterrupted('Consumption loop ended without a stop request')
+		except Exception as error:
+			if isinstance(error, ChannelClosedByBroker) and self.ack_timeout(error):
+				self.logger.warning(
+					'Delivery acknowledgement timed out: queues=%s reason=%s',
+					sorted(self.queues),
+					error.reply_text,
+				)
+			owned_boundaries = (
+				Event.ack.__code__,
+				Event.nack.__code__,
+				Event.reject.__code__,
+				Consumer.respond.__code__,
+				Consumer.stop.__code__,
+			)
+			trace = error.__traceback__
+			observed = error is self.callback_error
+			while trace is not None:
+				if trace.tb_frame.f_code in owned_boundaries:
+					observed = True
+					self.callback_error = None
+					break
+				trace = trace.tb_next
+			if not observed:
+				self.logger.error('Consumption loop failed: error_type=%s', type(error).__name__)
+			raise
+		finally:
+			self.connection.remove_timeout(self.timer)
+			self.logger.debug('Consumption loop ended')
+		return
+
+	def check_channel(self) -> None:
+		if not self.stopped and self.channel.is_open:
+			self.timer = self.connection.call_later(0.2, self.check_channel)
+		return
+
+	@staticmethod
+	def ack_timeout(error: ChannelClosedByBroker) -> bool:
+		return (
+			error.reply_code == 406
+			and 'acknowledgement' in error.reply_text.lower()
+			and 'timed out' in error.reply_text.lower()
+		)
+
+	def is_recoverable(self, error: BaseException) -> bool:
+		if error is self.callback_error:
+			return False
+		if isinstance(error, (
+			AuthenticationError,
+			ProbableAuthenticationError,
+			ProbableAccessDeniedError,
+			IncompatibleProtocolError,
+		)):
+			return False
+		if isinstance(error, ChannelClosedByBroker):
+			return self.ack_timeout(error) or error.reply_code in (320, 541)
+		if isinstance(error, ConnectionClosedByBroker):
+			return error.reply_code in (320, 541)
+		return isinstance(error, (
+			AMQPConnectionError,
+			ConsumptionInterrupted,
+		))
+
+	def stop(self):
+		if not self.stopped:
+			self.stopped = True
+			self.logger.debug('Consumption loop stop requested')
+			try:
+				self.channel.stop_consuming()
+			except Exception as error:
+				self.logger.error('Failed to stop consumption loop: error_type=%s', type(error).__name__)
+				raise
+			self.logger.debug('Consumption loop stopped')
+		return
+
+
+class FailedRequest(BaseEvent):
+	def __init__(
+		self,
+		consumer: Consumer,
+		event: Event,
+	):
+		self.consumer = consumer
+		self.event = event
+
+	@property
+	def body(self) -> Any:
+		return self.event.body
+
+	def ack(self) -> None:
+		self.event.ack()
+
+	def nack(self) -> None:
+		self.event.nack()
+
+	def reject(self) -> None:
+		self.consumer.logger.debug('Building RPC failure response: queue=%s correlation_id=%s', self.event.queue, self.event.properties.correlation_id)
+		self.consumer.respond(
+			self.event,
+			{
+				'ok': False,
+				'error': {
+					'code': 'ProcessingError',
+					'message': 'Request processing failed',
+				},
+			},
+		)
+		self.event.reject()
+		return
+
+
+class Callback(object):
+	def __init__(
+		self,
+		decode: Decoder,
+		handler: EventHandler,
+		queue: str,
+		logger=None,
+		owner=None,
+	):
+		self.decode = decode
+		self.handler = handler
+		self.queue = queue
+		self.logger = logger if logger is not None else Consumer.logger
+		self.owner = owner
+		return
+	def __call__(
+		self,
+		channel,
+		method,
+		properties,
+		body,
+	):
+		try:
+			return self.deliver(channel, method, properties, body)
+		except Exception as error:
+			if self.owner is not None:
+				self.owner.callback_error = error
+			raise
+
+	def deliver(
+		self,
+		channel,
+		method,
+		properties,
+		body,
+	):
+		self.logger.debug('Message received: queue=%s consumer_tag=%s delivery_tag=%s message_id=%s', self.queue, method.consumer_tag, method.delivery_tag, properties.message_id)
+		try:
+			decoded = self.decode(body, properties.content_type, properties.content_encoding)
+		except Exception as error:
+			self.logger.error('Message decoding failed: queue=%s delivery_tag=%s error_type=%s', self.queue, method.delivery_tag, type(error).__name__)
+			raise
+		self.logger.debug('Message decoded; dispatching to handler: queue=%s delivery_tag=%s', self.queue, method.delivery_tag)
+		return self.handler(Event(
+			channel,
+			self.queue,
+			method.consumer_tag,
+			method.delivery_tag,
+			properties,
+			decoded,
+		))

DefStream/Broker/Implements/RabbitMQ/Event.py

@@ -0,0 +1,146 @@
+# -*- coding: utf-8 -*-
+
+from DefStream.Broker import (
+	Event as BaseEvent,
+	EventHandler as BaseEventHandler,
+)
+from DefStream.Broker.Event import EventErrorPolicy
+
+from abc import abstractmethod
+from logging import getLogger
+
+__all__ = (
+	'Event',
+	'EventHandler',
+	'Requeue',
+	'Reject',
+)
+
+
+class Event(BaseEvent):
+	"""Event of Event Broker for RabbitMQ"""
+	logger = getLogger('Broker.RabbitMQ.Event')
+
+	def __init__(
+		self,
+		channel,
+		queue,
+		consumer,
+		transaction,
+		properties,
+		body,
+	):
+		self.channel = channel
+		self.queue = queue
+		self.consumer = consumer
+		self.transaction = transaction
+		self.properties = properties
+		self.props = properties.headers or {}
+		self.payload = body
+		return
+
+	def __repr__(self):
+		return 'Event(id={}, body={}, type={}, headers={})'.format(
+			self.properties.message_id,
+			self.payload,
+			self.properties.type,
+			self.props,
+		)
+
+	def ack(self):
+		self.logger.debug('ack requested: queue=%s delivery_tag=%s message_id=%s', self.queue, self.transaction, self.id)
+		try:
+			self.channel.basic_ack(self.transaction)
+		except Exception as error:
+			self.logger.error('ack failed: queue=%s delivery_tag=%s error_type=%s', self.queue, self.transaction, type(error).__name__)
+			raise
+		self.logger.debug('ack broker API call completed: queue=%s delivery_tag=%s', self.queue, self.transaction)
+		return
+
+	def nack(self):
+		self.logger.debug('nack requested: queue=%s delivery_tag=%s message_id=%s requeue=True', self.queue, self.transaction, self.id)
+		try:
+			self.channel.basic_nack(
+				self.transaction,
+				requeue=True,
+			)
+		except Exception as error:
+			self.logger.error('nack failed: queue=%s delivery_tag=%s error_type=%s', self.queue, self.transaction, type(error).__name__)
+			raise
+		self.logger.debug('nack broker API call completed: queue=%s delivery_tag=%s', self.queue, self.transaction)
+		return
+
+	def reject(self):
+		self.logger.debug('reject requested: queue=%s delivery_tag=%s message_id=%s requeue=False', self.queue, self.transaction, self.id)
+		try:
+			self.channel.basic_reject(
+				self.transaction,
+				requeue=False,
+			)
+		except Exception as error:
+			self.logger.error('reject failed: queue=%s delivery_tag=%s error_type=%s', self.queue, self.transaction, type(error).__name__)
+			raise
+		self.logger.debug('reject broker API call completed: queue=%s delivery_tag=%s', self.queue, self.transaction)
+		return
+
+	def headers(self):
+		return self.props
+
+	def header(
+		self,
+		key,
+	):
+		if key not in self.props:
+			return None
+		return self.props[key]
+
+	@property
+	def src(self):
+		return self.queue
+
+	@property
+	def id(self):
+		return self.properties.message_id
+
+	@property
+	def type(self):
+		return self.properties.type
+
+	@property
+	def body(self):
+		return self.payload
+	
+	@property
+	def priority(self):
+		if not hasattr(
+			self.properties,
+			'priority',
+		):
+			return None
+		return self.properties.priority
+
+
+class EventHandler(BaseEventHandler):
+	"""Event Handler Interface of Event Broker for RabbitMQ"""
+	@abstractmethod
+	def __call__(
+		self,
+		event: Event,
+	):
+		raise NotImplementedError('{} must be implemented __call__'.format(self.__class__.__name__))
+
+
+class Requeue(EventErrorPolicy):
+	def __call__(
+		self,
+		event: BaseEvent,
+	) -> None:
+		event.nack()
+
+
+class Reject(EventErrorPolicy):
+	def __call__(
+		self,
+		event: BaseEvent,
+	) -> None:
+		event.reject()