DefStream-RabbitMQ 0.1.5__16-py3-none-any.whl

DefStream/Broker/Implements/RabbitMQ/Exchange.py

@@ -0,0 +1,278 @@
+# -*- coding: utf-8 -*-
+
+from DefStream.Broker import (
+	Sendable,
+	Callable,
+	RemoteError,
+	validate_priority,
+)
+
+from .Serializer import (
+	Encoder,
+	Decoder,
+)
+from copy import deepcopy
+from logging import getLogger
+from math import (
+	ceil,
+	isfinite,
+)
+
+from pika import (
+	BlockingConnection,
+	BasicProperties,
+	ConnectionParameters,
+)
+
+from time import (
+	time,
+	monotonic,
+)
+from enum import Enum
+from uuid import uuid4
+
+from typing import (
+	Any,
+	Dict,
+	Optional,
+)
+
+__all__ = (
+	'Exchange',
+)
+
+
+class Exchange(
+	Sendable,
+	Callable,
+):
+	"""Exchange of Event Broker for RabbitMQ"""
+	logger = getLogger('DefStream.Broker.Implements.RabbitMQ.Exchange')
+
+	def __init__(
+		self,
+		connection: BlockingConnection,
+		encode: Encoder,
+		name: str = None,
+		decode: Optional[Decoder] = None,
+		rpc_parameters: Optional[ConnectionParameters] = None,
+	):
+		self.connection = connection
+		self.encode = encode
+		try:
+			self.channel = self.connection.channel()
+		except Exception as error:
+			self.logger.error('Exchange channel creation failed: exchange=%s error_type=%s', name, type(error).__name__)
+			raise
+		self.channel.auto_decode = False
+		self.exchange = name
+		self.decode = decode
+		self.rpc_parameters = rpc_parameters
+		return
+
+	def __del__(self):
+		channel = getattr(self, 'channel', None)
+		try:
+			if channel and channel.is_open:
+				channel.close()
+				self.logger.debug('Exchange channel cleanup completed: exchange=%s', self.exchange)
+		except Exception as error:
+			self.logger.error('Exchange channel cleanup failed: error_type=%s', type(error).__name__)
+			raise
+		return
+	
+	def __str__(self): return self.exchange
+
+	def send(
+		self,
+		body,
+		event: str = None,
+		headers: Dict = {},
+		priority: Optional[int] = None,
+		expiration: int = None,
+		timestamp: int = None,
+		persistent: bool = True,
+		id: str = None,
+	):
+		validate_priority(
+			priority,
+			255,
+		)
+		if not timestamp: timestamp = int(time())
+		if not id: id = uuid4().hex
+		properties = BasicProperties(
+			type=event if event else '',
+			headers=headers,
+			content_type=self.encode.format,
+			content_encoding=self.encode.charset,
+			priority=priority,
+			timestamp=timestamp,
+			expiration=expiration,
+			message_id=id,
+			delivery_mode=2 if persistent else 1,
+		)
+		self.logger.debug('Publishing message: exchange=%s routing_key=%s message_id=%s priority=%s persistent=%s publisher_confirm=False', self.exchange, event, id, priority, persistent)
+		try:
+			self.channel.basic_publish(
+				exchange=self.exchange,
+				routing_key=event if event else '',
+				properties=properties,
+				body=self.encode(body),
+			)
+		except Exception as error:
+			self.logger.error('Message publication failed: exchange=%s routing_key=%s message_id=%s error_type=%s', self.exchange, event, id, type(error).__name__)
+			raise
+		self.logger.debug('Publish API call completed: exchange=%s message_id=%s does not imply Consumer processing completion', self.exchange, id)
+		return id
+
+	def call(
+		self,
+		body: Any,
+		timeout: float = 5,
+		*,
+		event: Optional[str] = None,
+	) -> Any:
+		if self.rpc_parameters is None or self.decode is None:
+			raise RuntimeError('RPC requires decoder and connection parameters; obtain the exchange through Connection.exchange')
+		if isinstance(
+			timeout,
+			bool,
+		) or not isinstance(
+			timeout,
+			(
+				int,
+				float,
+			),
+		) or not isfinite(timeout) or timeout <= 0:
+			raise ValueError('RPC timeout must be a finite positive number of seconds')
+		deadline = monotonic() + timeout
+		settings = deepcopy(self.rpc_parameters)
+		settings.connection_attempts = 1
+		settings.socket_timeout = min(
+			settings.socket_timeout,
+			timeout,
+		)
+		settings.stack_timeout = timeout
+		settings.blocked_connection_timeout = timeout
+		timed_out = False
+		self.logger.debug('Opening dedicated RPC connection: exchange=%s routing_key=%s timeout=%s', self.exchange, event, timeout)
+		try:
+			connection = BlockingConnection(settings)
+		except Exception as error:
+			self.logger.error('Dedicated RPC connection failed: exchange=%s error_type=%s', self.exchange, type(error).__name__)
+			raise
+		try:
+			channel = connection.channel()
+			reply_queue = channel.queue_declare(
+				queue='',
+				exclusive=True,
+				auto_delete=True,
+			).method.queue
+			channel.confirm_delivery()
+			correlation_id = uuid4().hex
+			self.logger.debug('RPC reply queue created: exchange=%s reply_to=%s correlation_id=%s publisher_confirm=True', self.exchange, reply_queue, correlation_id)
+			responses = []
+
+			def receive(
+				channel,
+				method,
+				properties,
+				payload,
+			):
+				if properties.correlation_id == correlation_id:
+					responses.append((
+						properties,
+						payload,
+					))
+					self.logger.debug('RPC response received: exchange=%s correlation_id=%s', self.exchange, correlation_id)
+				else:
+					self.logger.debug('Ignoring RPC response with mismatched correlation_id: exchange=%s correlation_id=%s', self.exchange, properties.correlation_id)
+
+			channel.basic_consume(
+				reply_queue,
+				on_message_callback=receive,
+				auto_ack=True,
+			)
+			remaining = deadline - monotonic()
+			if remaining <= 0:
+				timed_out = True
+				self.logger.warning('RPC timed out before publication: exchange=%s routing_key=%s correlation_id=%s', self.exchange, event, correlation_id)
+				raise TimeoutError('RPC timed out before request publication')
+			self.logger.debug('Publishing RPC request: exchange=%s routing_key=%s correlation_id=%s', self.exchange, event, correlation_id)
+			channel.basic_publish(
+				exchange=self.exchange,
+				routing_key=event if event is not None else '',
+				body=self.encode(body),
+				properties=BasicProperties(
+					reply_to=reply_queue,
+					correlation_id=correlation_id,
+					content_type=self.encode.format,
+					content_encoding=self.encode.charset,
+					expiration=str(max(
+						1,
+						ceil(remaining * 1000),
+					)),
+				),
+				mandatory=True,
+			)
+			self.logger.debug('RPC request published: exchange=%s routing_key=%s correlation_id=%s', self.exchange, event, correlation_id)
+			while not responses:
+				remaining = deadline - monotonic()
+				if remaining <= 0:
+					timed_out = True
+					self.logger.warning('RPC response timed out after publication; delivered work is not cancelled: exchange=%s routing_key=%s correlation_id=%s', self.exchange, event, correlation_id)
+					raise TimeoutError(f'RPC response timed out for exchange {self.exchange!r} routing_key={event!r}')
+				connection.process_data_events(time_limit=min(
+					remaining,
+					0.1,
+				))
+			properties, payload = responses[0]
+			response = self.decode(
+				payload,
+				properties.content_type,
+				properties.content_encoding,
+			)
+			if not isinstance(
+				response,
+				dict,
+			) or type(response.get('ok')) is not bool:
+				raise ValueError('Invalid RPC response envelope')
+			if response['ok']:
+				if 'result' not in response:
+					raise ValueError('RPC response has no result')
+				self.logger.debug('RPC response validated: exchange=%s routing_key=%s correlation_id=%s elapsed=%s', self.exchange, event, correlation_id, timeout - (deadline - monotonic()))
+				return response['result']
+			error = response.get('error')
+			if not isinstance(
+				error,
+				dict,
+			) or not isinstance(
+				error.get('code'),
+				str,
+			) or not isinstance(
+				error.get('message'),
+				str,
+			):
+				raise ValueError('Invalid RPC error response')
+			self.logger.warning('RPC remote task error: exchange=%s routing_key=%s correlation_id=%s', self.exchange, event, correlation_id)
+			raise RemoteError(
+				error['code'],
+				error['message'],
+			)
+		except TimeoutError:
+			if not timed_out:
+				self.logger.error('RPC internal timeout error: exchange=%s routing_key=%s', self.exchange, event)
+			raise
+		except RemoteError:
+			raise
+		except Exception as error:
+			self.logger.error('RPC transmission or response handling failed: exchange=%s routing_key=%s error_type=%s', self.exchange, event, type(error).__name__)
+			raise
+		finally:
+			try:
+				if connection.is_open:
+					connection.close()
+					self.logger.debug('Dedicated RPC connection and reply queue cleanup completed: exchange=%s', self.exchange)
+			except Exception as error:
+				self.logger.error('Dedicated RPC connection cleanup failed: exchange=%s error_type=%s', self.exchange, type(error).__name__)
+				raise

DefStream/Broker/Implements/RabbitMQ/Policies.py

@@ -0,0 +1,9 @@
+from DefStream.Broker import (
+	Requeue,
+	Reject,
+)
+
+__all__ = (
+	'Requeue',
+	'Reject',
+)

DefStream/Broker/Implements/RabbitMQ/Queue.py

@@ -0,0 +1,319 @@
+# -*- coding: utf-8 -*-
+
+from DefStream.Broker import (
+	Sendable,
+	Callable,
+	Poppable,
+	RemoteError,
+	validate_priority,
+)
+
+from .Serializer import (
+	Encoder,
+	Decoder,
+)
+from .Event import Event
+from copy import deepcopy
+from logging import getLogger
+from math import (
+	ceil,
+	isfinite,
+)
+
+from pika import (
+	BlockingConnection,
+	BasicProperties,
+	ConnectionParameters,
+)
+
+from time import (
+	time,
+	sleep,
+	monotonic,
+)
+from uuid import uuid4
+
+from typing import (
+	Any,
+	Optional,
+	Dict,
+)
+
+__all__ = (
+	'Queue'
+)
+
+
+class Queue(
+	Sendable,
+	Callable,
+	Poppable,
+):
+	"""Queue of Event Broker for RabbitMQ"""
+	logger = getLogger('DefStream.Broker.Implements.RabbitMQ.Queue')
+
+	def __init__(
+		self,
+		connection: BlockingConnection,
+		encode: Encoder,
+		decode: Decoder,
+		name: str = None,
+		rpc_parameters: Optional[ConnectionParameters] = None,
+	):
+		self.connection = connection
+		self.encode = encode
+		self.decode = decode
+		try:
+			self.channel = self.connection.channel()
+		except Exception as error:
+			self.logger.error('Queue channel creation failed: queue=%s error_type=%s', name, type(error).__name__)
+			raise
+		self.channel.auto_decode = False
+		self.queue = name
+		self.event = None
+		self.rpc_parameters = rpc_parameters
+		return
+
+	def __del__(self):
+		channel = getattr(self, 'channel', None)
+		try:
+			if channel and channel.is_open:
+				channel.close()
+				self.logger.debug('Queue channel cleanup completed: queue=%s', self.queue)
+		except Exception as error:
+			self.logger.error('Queue channel cleanup failed: error_type=%s', type(error).__name__)
+			raise
+		return
+	
+	def __str__(self): return self.queue
+
+	def send(
+		self,
+		body,
+		event: str = None,
+		headers: Dict = {},
+		priority: Optional[int] = None,
+		expiration: int = None,
+		timestamp: int = None,
+		persistent: bool = True,
+		id: str = None,
+		reply_to: str = None,
+		correlation_id: str = None,
+	):
+		validate_priority(
+			priority,
+			255,
+		)
+		if not timestamp: timestamp = int(time())
+		if not id: id = uuid4().hex
+		properties = BasicProperties(
+			type=event if event else '',
+			headers=headers,
+			content_type=self.encode.format,
+			content_encoding=self.encode.charset,
+			priority=priority,
+			timestamp=timestamp,
+			expiration=expiration,
+			message_id=id,
+			delivery_mode=2 if persistent else 1,
+			reply_to=reply_to,
+			correlation_id=correlation_id,
+		)
+		self.logger.debug('Publishing message: queue=%s message_id=%s priority=%s persistent=%s publisher_confirm=False', self.queue, id, priority, persistent)
+		try:
+			self.channel.basic_publish(
+				exchange='',
+				routing_key=self.queue,
+				properties=properties,
+				body=self.encode(body)
+			)
+		except Exception as error:
+			self.logger.error('Message publication failed: queue=%s message_id=%s error_type=%s', self.queue, id, type(error).__name__)
+			raise
+		self.logger.debug('Publish API call completed: queue=%s message_id=%s does not imply Consumer processing completion', self.queue, id)
+		return id
+
+	def call(
+		self,
+		body: Any,
+		timeout: float = 5,
+		*,
+		priority: Optional[int] = None,
+	) -> Any:
+		validate_priority(
+			priority,
+			255,
+		)
+		if self.rpc_parameters is None:
+			raise RuntimeError('RPC requires connection parameters; obtain the queue through Connection.queue')
+		if isinstance(
+			timeout,
+			bool,
+		) or not isinstance(
+			timeout,
+			(
+				int,
+				float,
+			),
+		) or not isfinite(timeout) or timeout <= 0:
+			raise ValueError('RPC timeout must be a finite positive number of seconds')
+		deadline = monotonic() + timeout
+		settings = deepcopy(self.rpc_parameters)
+		settings.connection_attempts = 1
+		settings.socket_timeout = min(
+			settings.socket_timeout,
+			timeout,
+		)
+		settings.stack_timeout = timeout
+		settings.blocked_connection_timeout = timeout
+		timed_out = False
+		self.logger.debug('Opening dedicated RPC connection: queue=%s timeout=%s', self.queue, timeout)
+		try:
+			connection = BlockingConnection(settings)
+		except Exception as error:
+			self.logger.error('Dedicated RPC connection failed: queue=%s error_type=%s', self.queue, type(error).__name__)
+			raise
+		try:
+			channel = connection.channel()
+			reply_queue = channel.queue_declare(
+				queue='',
+				exclusive=True,
+				auto_delete=True,
+			).method.queue
+			channel.confirm_delivery()
+			correlation_id = uuid4().hex
+			self.logger.debug('RPC reply queue created: queue=%s reply_to=%s correlation_id=%s publisher_confirm=True', self.queue, reply_queue, correlation_id)
+			responses = []
+
+			def receive(
+				channel,
+				method,
+				properties,
+				payload,
+			):
+				if properties.correlation_id == correlation_id:
+					responses.append((
+						properties,
+						payload,
+					))
+					self.logger.debug('RPC response received: queue=%s correlation_id=%s', self.queue, correlation_id)
+				else:
+					self.logger.debug('Ignoring RPC response with mismatched correlation_id: queue=%s correlation_id=%s', self.queue, properties.correlation_id)
+
+			channel.basic_consume(
+				reply_queue,
+				on_message_callback=receive,
+				auto_ack=True,
+			)
+			remaining = deadline - monotonic()
+			if remaining <= 0:
+				timed_out = True
+				self.logger.warning('RPC timed out before publication: queue=%s correlation_id=%s', self.queue, correlation_id)
+				raise TimeoutError('RPC timed out before request publication')
+			self.logger.debug('Publishing RPC request: queue=%s correlation_id=%s priority=%s', self.queue, correlation_id, priority)
+			channel.basic_publish(
+				exchange='',
+				routing_key=self.queue,
+				body=self.encode(body),
+				properties=BasicProperties(
+					priority=priority,
+					reply_to=reply_queue,
+					correlation_id=correlation_id,
+					content_type=self.encode.format,
+					content_encoding=self.encode.charset,
+					expiration=str(max(
+						1,
+						ceil(remaining * 1000),
+					)),
+				),
+				mandatory=True,
+			)
+			self.logger.debug('RPC request published: queue=%s correlation_id=%s', self.queue, correlation_id)
+			while not responses:
+				remaining = deadline - monotonic()
+				if remaining <= 0:
+					timed_out = True
+					self.logger.warning('RPC response timed out after publication; delivered work is not cancelled: queue=%s correlation_id=%s', self.queue, correlation_id)
+					raise TimeoutError(f'RPC response timed out for queue {self.queue!r}')
+				connection.process_data_events(time_limit=min(
+					remaining,
+					0.1,
+				))
+			properties, payload = responses[0]
+			response = self.decode(
+				payload,
+				properties.content_type,
+				properties.content_encoding,
+			)
+			if not isinstance(
+				response,
+				dict,
+			) or type(response.get('ok')) is not bool:
+				raise ValueError('Invalid RPC response envelope')
+			if response['ok']:
+				if 'result' not in response:
+					raise ValueError('RPC response has no result')
+				self.logger.debug('RPC response validated: queue=%s correlation_id=%s elapsed=%s', self.queue, correlation_id, timeout - (deadline - monotonic()))
+				return response['result']
+			error = response.get('error')
+			if not isinstance(
+				error,
+				dict,
+			) or not isinstance(
+				error.get('code'),
+				str,
+			) or not isinstance(
+				error.get('message'),
+				str,
+			):
+				raise ValueError('Invalid RPC error response')
+			self.logger.warning('RPC remote task error: queue=%s correlation_id=%s', self.queue, correlation_id)
+			raise RemoteError(
+				error['code'],
+				error['message'],
+			)
+		except TimeoutError:
+			if not timed_out:
+				self.logger.error('RPC internal timeout error: queue=%s', self.queue)
+			raise
+		except RemoteError:
+			raise
+		except Exception as error:
+			self.logger.error('RPC transmission or response handling failed: queue=%s error_type=%s', self.queue, type(error).__name__)
+			raise
+		finally:
+			try:
+				if connection.is_open:
+					connection.close()
+					self.logger.debug('Dedicated RPC connection and reply queue cleanup completed: queue=%s', self.queue)
+			except Exception as error:
+				self.logger.error('Dedicated RPC connection cleanup failed: queue=%s error_type=%s', self.queue, type(error).__name__)
+				raise
+
+	def pop(self, timeout: int = None) -> Optional[Event]:
+		self.logger.debug('Waiting for a single message: queue=%s timeout_ms=%s', self.queue, timeout)
+		channel = self.channel
+		try:
+			channel.basic_qos(0, 1, False)
+			deadline = time() + (timeout / 1000) if timeout else None
+			while True:
+				method, properties, body = channel.basic_get(queue=self.queue, auto_ack=False)
+				if method:
+					event = Event(
+						channel,
+						self.queue,
+						None,
+						method.delivery_tag,
+						properties,
+						self.decode(body, properties.content_type, properties.content_encoding),
+					)
+					self.logger.debug('Single message received and decoded: queue=%s delivery_tag=%s message_id=%s', self.queue, method.delivery_tag, properties.message_id)
+					return event
+				if deadline and time() >= deadline:
+					break
+				sleep(0.1)
+		except Exception as error:
+			self.logger.error('Single-message reception failed: queue=%s error_type=%s', self.queue, type(error).__name__)
+			raise
+		self.logger.debug('Single-message wait ended: queue=%s', self.queue)
+		return

DefStream/Broker/Implements/RabbitMQ/Serializer.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+from abc import (
+	ABCMeta,
+	abstractmethod,
+)
+
+from json import (
+	dumps,
+	loads,
+)
+from typing import Any
+
+__all__ = (
+	'Encoder',
+	'Decoder',
+	'JavaScriptObjectNotationEncoder',
+	'JavaScriptObjectNotationDecoder',
+)
+
+class Encoder(metaclass=ABCMeta):
+	"""Encoder Interface"""
+	@abstractmethod
+	def __call__(self, obj: Any) -> bytes:
+		raise NotImplementedError('{} must be implemented __call__'.format(self.__class__.__name__))
+
+	@property
+	@abstractmethod
+	def format(self) -> str:
+		raise NotImplementedError('{} must be implemented format'.format(self.__class__.__name__))	
+
+	@property
+	@abstractmethod
+	def charset(self) -> str:
+		raise NotImplementedError('{} must be implemented charset'.format(self.__class__.__name__))
+	
+
+class Decoder(metaclass=ABCMeta):
+	"""Decoder Interface"""
+	@abstractmethod
+	def __call__(self, obj: bytes, format: str, charset: str) -> Any:
+		raise NotImplementedError('{} must be implemented __call__'.format(self.__class__.__name__))
+
+
+class JavaScriptObjectNotationEncoder(Encoder):
+	"""JavaScriptObjectNotation Encoder"""
+	def __call__(self, obj: Any) -> bytes:
+		return dumps(obj, ensure_ascii=False).encode(self.charset)
+
+	@property
+	def format(self) -> str: return 'application/json'
+
+	@property
+	def charset(self) -> str: return 'utf-8'
+
+
+class JavaScriptObjectNotationDecoder(Decoder):
+	"""JavaScriptObjectNotation Decoder"""
+	def __call__(self, obj: bytes, format: str, charset: str) -> Any:
+		if format != 'application/json': raise ValueError('format is not support')
+		return loads(obj.decode(charset))