DefStream-RabbitMQ 0.1.3__7-py3-none-any.whl

DefStream/Broker/Implements/RabbitMQ/Consumer.py

@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+
+from DefStream.Broker import (
+	Consumer as BaseConsumer,
+	EventHandler,
+	Event as BaseEvent,
+	EventError,
+)
+
+from .Serializer import (
+	Decoder,
+	Encoder,
+	JavaScriptObjectNotationEncoder,
+)
+from logging import getLogger
+
+from .Event import Event
+from .Policies import Requeue
+from .Subscription import Subscription
+from typing import (
+	Any,
+	Callable,
+	Optional,
+)
+
+from pika import (
+	BasicProperties,
+	BlockingConnection,
+)
+from pika.exceptions import UnroutableError
+
+__all__ = (
+	'Consumer'
+)
+
+logger = getLogger(__name__)
+
+
+class Consumer(BaseConsumer):
+	"""Consumer of Event Broker for RabbitMQ"""
+	def __init__(
+		self,
+		connection: BlockingConnection,
+		decode: Decoder,
+		handler: Optional[EventHandler] = None,
+		qos: int = 1,
+		offset: int = None,
+		encode: Optional[Encoder] = None,
+	):
+		self.connection = connection
+		self.decode = decode
+		self.channel = self.connection.channel()
+		self.qos = qos
+		self.channel.basic_qos(
+			prefetch_count=qos,
+			global_qos=False,
+		)
+		self.channel.auto_decode = False
+		self.offset = offset
+		self.handler = handler
+		self.encode = encode if encode is not None else JavaScriptObjectNotationEncoder()
+		self.response_channel = None
+		self.queues = set()
+		self.stopped = False
+		return
+
+	def __del__(self):
+		self.close()
+		return
+
+	def subs(
+		self,
+		queue: Any,
+		handler: Optional[EventHandler] = None,
+	) -> None:
+		if handler is None:
+			handler = self.handler
+		if handler is None:
+			raise ValueError('subs requires a handler or a default handler')
+		if isinstance(
+			queue,
+			str,
+		):
+			subscription = Subscription(
+				queue=queue,
+				offset=self.offset,
+			)
+		else:
+			subscription = Subscription.parse(queue)
+		if subscription.queue in self.queues:
+			raise ValueError('A Run is already subscribed to queue {}'.format(subscription.queue))
+		self.channel.basic_qos(
+			prefetch_count=self.qos if subscription.qos is None else subscription.qos,
+			global_qos=False,
+		)
+		self.channel.basic_consume(
+			subscription.queue,
+			Callback(
+				self.decode,
+				handler,
+				subscription.queue,
+			),
+			auto_ack=False,
+			exclusive=False,
+			arguments={'x-stream-offset': subscription.offset} if subscription.offset is not None else None,
+		)
+		self.queues.add(subscription.queue)
+
+	def call(self, callback: Callable[[], None]) -> None:
+		self.connection.add_callback_threadsafe(callback)
+
+	def fail(
+		self,
+		event: BaseEvent,
+		policy: Optional[EventError] = None,
+	) -> None:
+		if isinstance(
+			event,
+			Event,
+		) and event.properties.reply_to:
+			event = FailedRequest(
+				self,
+				event,
+			)
+		(policy if policy is not None else Requeue())(event)
+
+	def complete(
+		self,
+		event: BaseEvent,
+		result: Any,
+	) -> None:
+		if isinstance(
+			event,
+			Event,
+		) and event.properties.reply_to:
+			self.respond(
+				event,
+				{
+					'ok': True,
+					'result': result,
+				},
+			)
+		event.ack()
+
+	def respond(
+		self,
+		event: Event,
+		response: Any,
+	) -> None:
+		if not event.properties.correlation_id:
+			raise ValueError('RPC request requires correlation_id')
+		channel = self.response_channel
+		if channel is None:
+			channel = self.connection.channel()
+			channel.confirm_delivery()
+			self.response_channel = channel
+		try:
+			channel.basic_publish(
+				exchange='',
+				routing_key=event.properties.reply_to,
+				body=self.encode(response),
+				properties=BasicProperties(
+					correlation_id=event.properties.correlation_id,
+					content_type=self.encode.format,
+					content_encoding=self.encode.charset,
+				),
+				mandatory=True,
+			)
+		except UnroutableError:
+			logger.warning(
+				'RPC reply is unroutable; settling request without a response: correlation_id=%s reply_to=%s',
+				event.properties.correlation_id,
+				event.properties.reply_to,
+			)
+
+	def close(self) -> None:
+		response_channel = getattr(self, 'response_channel', None)
+		if response_channel is not None and response_channel.is_open:
+			response_channel.close()
+		channel = getattr(self, 'channel', None)
+		if channel is not None and channel.is_open:
+			channel.close()
+
+	def run(self):
+		self.channel.start_consuming()
+		return
+
+	def stop(self):
+		if not self.stopped:
+			self.stopped = True
+			self.channel.stop_consuming()
+		return
+
+
+class FailedRequest(BaseEvent):
+	def __init__(
+		self,
+		consumer: Consumer,
+		event: Event,
+	):
+		self.consumer = consumer
+		self.event = event
+
+	@property
+	def body(self) -> Any:
+		return self.event.body
+
+	def ack(self) -> None:
+		self.event.ack()
+
+	def nack(self) -> None:
+		self.event.nack()
+
+	def reject(self) -> None:
+		self.consumer.respond(
+			self.event,
+			{
+				'ok': False,
+				'error': {
+					'code': 'ProcessingError',
+					'message': 'Request processing failed',
+				},
+			},
+		)
+		self.event.reject()
+
+
+class Callback(object):
+	def __init__(self, decode: Decoder, handler: EventHandler, queue: str):
+		self.decode = decode
+		self.handler = handler
+		self.queue = queue
+		return
+	def __call__(self, channel, method, properties, body):
+		return self.handler(Event(
+			channel,
+			self.queue,
+			method.consumer_tag,
+			method.delivery_tag,
+			properties,
+			self.decode(body, properties.content_type, properties.content_encoding),
+		))

DefStream/Broker/Implements/RabbitMQ/Event.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+
+from DefStream.Broker import (
+	Event as BaseEvent,
+	EventHandler as BaseEventHandler,
+)
+
+from abc import abstractmethod
+
+__all__ = (
+	'Event',
+	'EventHandler',
+)
+
+
+class Event(BaseEvent):
+	"""Event of Event Broker for RabbitMQ"""
+	def __init__(
+		self,
+		channel,
+		queue,
+		consumer,
+		transaction,
+		properties,
+		body,
+	):
+		self.channel = channel
+		self.queue = queue
+		self.consumer = consumer
+		self.transaction = transaction
+		self.properties = properties
+		self.props = properties.headers
+		self.payload = body
+		return
+
+	def __repr__(self):
+		return 'Event(id={}, body={}, type={}, headers={})'.format(
+			self.properties.message_id,
+			self.payload,
+			self.properties.type,
+			self.props,
+		)
+
+	def ack(self):
+		self.channel.basic_ack(self.transaction)
+		return
+
+	def nack(self):
+		self.channel.basic_nack(
+			self.transaction,
+			requeue=True,
+		)
+		return
+
+	def reject(self):
+		self.channel.basic_reject(
+			self.transaction,
+			requeue=False,
+		)
+		return
+
+	def headers(self):
+		return self.props
+
+	def header(
+		self,
+		key,
+	):
+		if key not in self.props:
+			return None
+		return self.props[key]
+
+	@property
+	def src(self):
+		return self.queue
+
+	@property
+	def id(self):
+		return self.properties.message_id
+
+	@property
+	def type(self):
+		return self.properties.type
+
+	@property
+	def body(self):
+		return self.payload
+	
+	@property
+	def priority(self):
+		if not hasattr(
+			self.properties,
+			'priority',
+		):
+			return None
+		return self.properties.priority
+
+
+class EventHandler(BaseEventHandler):
+	"""Event Handler Interface of Event Broker for RabbitMQ"""
+	@abstractmethod
+	def __call__(
+		self,
+		event: Event,
+	):
+		raise NotImplementedError('{} must be implemented __call__'.format(self.__class__.__name__))

DefStream/Broker/Implements/RabbitMQ/Exchange.py

@@ -0,0 +1,227 @@
+# -*- coding: utf-8 -*-
+
+from DefStream.Broker import (
+	Sendable,
+	Callable,
+	RemoteError,
+	validate_priority,
+)
+
+from .Serializer import (
+	Encoder,
+	Decoder,
+)
+from copy import deepcopy
+from math import (
+	ceil,
+	isfinite,
+)
+
+from pika import (
+	BlockingConnection,
+	BasicProperties,
+	ConnectionParameters,
+)
+
+from time import (
+	time,
+	monotonic,
+)
+from enum import Enum
+from uuid import uuid4
+
+from typing import (
+	Any,
+	Dict,
+	Optional,
+)
+
+__all__ = (
+	'Exchange',
+)
+
+
+class Exchange(
+	Sendable,
+	Callable,
+):
+	"""Exchange of Event Broker for RabbitMQ"""
+	def __init__(
+		self,
+		connection: BlockingConnection,
+		encode: Encoder,
+		name: str = None,
+		decode: Optional[Decoder] = None,
+		rpc_parameters: Optional[ConnectionParameters] = None,
+	):
+		self.connection = connection
+		self.encode = encode
+		self.channel = self.connection.channel()
+		self.channel.auto_decode = False
+		self.exchange = name
+		self.decode = decode
+		self.rpc_parameters = rpc_parameters
+		return
+
+	def __del__(self):
+		if self.channel and self.channel.is_open:
+			self.channel.close()
+		return
+	
+	def __str__(self): return self.exchange
+
+	def send(
+		self,
+		body,
+		event: str = None,
+		headers: Dict = {},
+		priority: Optional[int] = None,
+		expiration: int = None,
+		timestamp: int = None,
+		persistent: bool = True,
+		id: str = None,
+	):
+		validate_priority(
+			priority,
+			255,
+		)
+		if not timestamp: timestamp = int(time())
+		if not id: id = uuid4().hex
+		properties = BasicProperties(
+			type=event if event else '',
+			headers=headers,
+			content_type=self.encode.format,
+			content_encoding=self.encode.charset,
+			priority=priority,
+			timestamp=timestamp,
+			expiration=expiration,
+			message_id=id,
+			delivery_mode=2 if persistent else 1,
+		)
+		self.channel.basic_publish(
+			exchange=self.exchange,
+			routing_key=event if event else '',
+			properties=properties,
+			body=self.encode(body),
+		)
+		return id
+
+	def call(
+		self,
+		body: Any,
+		timeout: float = 5,
+		*,
+		event: Optional[str] = None,
+	) -> Any:
+		if self.rpc_parameters is None or self.decode is None:
+			raise RuntimeError('RPC requires decoder and connection parameters; obtain the exchange through Connection.exchange')
+		if isinstance(
+			timeout,
+			bool,
+		) or not isinstance(
+			timeout,
+			(
+				int,
+				float,
+			),
+		) or not isfinite(timeout) or timeout <= 0:
+			raise ValueError('RPC timeout must be a finite positive number of seconds')
+		deadline = monotonic() + timeout
+		settings = deepcopy(self.rpc_parameters)
+		settings.connection_attempts = 1
+		settings.socket_timeout = min(
+			settings.socket_timeout,
+			timeout,
+		)
+		settings.stack_timeout = timeout
+		settings.blocked_connection_timeout = timeout
+		connection = BlockingConnection(settings)
+		try:
+			channel = connection.channel()
+			reply_queue = channel.queue_declare(
+				queue='',
+				exclusive=True,
+				auto_delete=True,
+			).method.queue
+			channel.confirm_delivery()
+			correlation_id = uuid4().hex
+			responses = []
+
+			def receive(
+				channel,
+				method,
+				properties,
+				payload,
+			):
+				if properties.correlation_id == correlation_id:
+					responses.append((
+						properties,
+						payload,
+					))
+
+			channel.basic_consume(
+				reply_queue,
+				on_message_callback=receive,
+				auto_ack=True,
+			)
+			remaining = deadline - monotonic()
+			if remaining <= 0:
+				raise TimeoutError('RPC timed out before request publication')
+			channel.basic_publish(
+				exchange=self.exchange,
+				routing_key=event if event is not None else '',
+				body=self.encode(body),
+				properties=BasicProperties(
+					reply_to=reply_queue,
+					correlation_id=correlation_id,
+					content_type=self.encode.format,
+					content_encoding=self.encode.charset,
+					expiration=str(max(
+						1,
+						ceil(remaining * 1000),
+					)),
+				),
+				mandatory=True,
+			)
+			while not responses:
+				remaining = deadline - monotonic()
+				if remaining <= 0:
+					raise TimeoutError(f'RPC response timed out for exchange {self.exchange!r} routing_key={event!r}')
+				connection.process_data_events(time_limit=min(
+					remaining,
+					0.1,
+				))
+			properties, payload = responses[0]
+			response = self.decode(
+				payload,
+				properties.content_type,
+				properties.content_encoding,
+			)
+			if not isinstance(
+				response,
+				dict,
+			) or type(response.get('ok')) is not bool:
+				raise ValueError('Invalid RPC response envelope')
+			if response['ok']:
+				if 'result' not in response:
+					raise ValueError('RPC response has no result')
+				return response['result']
+			error = response.get('error')
+			if not isinstance(
+				error,
+				dict,
+			) or not isinstance(
+				error.get('code'),
+				str,
+			) or not isinstance(
+				error.get('message'),
+				str,
+			):
+				raise ValueError('Invalid RPC error response')
+			raise RemoteError(
+				error['code'],
+				error['message'],
+			)
+		finally:
+			if connection.is_open:
+				connection.close()

DefStream/Broker/Implements/RabbitMQ/Policies.py

@@ -0,0 +1,9 @@
+from DefStream.Broker import (
+	Requeue,
+	Reject,
+)
+
+__all__ = (
+	'Requeue',
+	'Reject',
+)