DefStream-RabbitMQ 0.1.1__py3-none-any.whl

DefStream/Broker/Implements/RabbitMQ/Configuration.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+from ... import Configuration as BaseConfiguration
+
+from .Serializer import (
+	Encoder,
+	Decoder,
+	JavaScriptObjectNotationEncoder,
+	JavaScriptObjectNotationDecoder,
+)
+
+
+from pika import SSLOptions
+from ssl import create_default_context
+
+__all__ = (
+	'Configuration'
+)
+
+
+class Configuration(BaseConfiguration):
+	"""
+	Configuration of Event Broker for RabbitMQ
+	"""
+	def __init__(
+		self,
+		host: str,
+		port: int,
+		username: str = None,
+		password: str = None,
+		ssl: bool = False,
+		vhost: str = '/',
+		timeout: int = None,
+		heartbeat: int = None,
+		encode: Encoder = JavaScriptObjectNotationEncoder(),
+		decode: Decoder = JavaScriptObjectNotationDecoder(),
+	):
+		self.host = host
+		self.port = port
+		self.username = username
+		self.password = password
+		self.ssl = SSLOptions(create_default_context(), host) if ssl else None
+		self.vhost = vhost
+		self.timeout = timeout/1000 if timeout else None
+		self.heartbeat = int(heartbeat*2/1000) if heartbeat else None
+		self.encode = encode
+		self.decode = decode
+		return

DefStream/Broker/Implements/RabbitMQ/Connection.py

@@ -0,0 +1,449 @@
+# -*- coding: utf-8 -*-
+
+from ...Connection import (
+	Connection as BaseConnection,
+	GetExchange,
+	GetQueue,
+	GetStream,
+	GetConsumer,
+	Declarable,
+	TaskSender,
+	Requestable,
+)
+
+from .Configuration import Configuration
+
+from .Exchange import Exchange
+from .Queue import Queue
+from .Stream import Stream
+from .Consumer import Consumer
+from .Subscription import Subscription
+from ...Event import EventHandler
+
+from pika import (
+	BlockingConnection,
+	ConnectionParameters,
+	PlainCredentials,
+	BasicProperties,
+)
+from pika.exceptions import *
+
+from collections.abc import Mapping
+from operator import (
+	eq,
+	ne,
+)
+from enum import Enum
+from uuid import uuid4
+
+from typing import (
+	Any,
+	Iterator,
+	Union,
+	Optional,
+)
+
+__all__ = (
+	'Connection',
+	'ExchangeType',
+	'Parameters',
+	'All',
+	'Any',
+)
+
+
+class ExchangeType(str, Enum):
+	Direct = 'direct'
+	FanOut = 'fanout'
+	Topic  = 'topic'
+	Header = 'headers'
+	def __str__(self): return str(self.value)
+
+
+class Parameters(Mapping):
+	"""Bind Parameters Event Broker for RabbitMQ"""
+	def __init__(self, **kwargs):
+		self.args = kwargs
+		return
+	
+	def __iter__(self) -> Iterator:
+		return self.args.__iter__()
+	
+	def __reversed__(self) -> Iterator:
+		return self.args.__reversed__()
+	
+	def __len__(self) -> int:
+		return self.args.__len__()
+	
+	def __getitem__(self, key: Any) -> Any:
+		return self.args.__getitem__(key)
+	
+	def __contains__(self, key: object) -> bool:
+		return self.args.__contains__(key)
+	
+	def __eq__(self, other) -> bool:
+		return eq(self.args, other)
+	
+	def __ne__(self, other) -> bool:
+		return ne(self.args, other)
+	
+	def keys(self) -> Iterator:
+		return self.args.keys()
+	
+	def items(self):
+		return self.args.items()
+	
+	def values(self):
+		return self.args.values()
+
+
+class Connection(
+	BaseConnection,
+	GetExchange,
+	GetQueue,
+	GetStream,
+	GetConsumer,
+	Declarable,
+	TaskSender,
+	Requestable,
+):
+	"""Connection of Event Broker for RabbitMQ"""
+	def __init__(self, conf: Configuration):
+		self.conf = conf
+		self.parameters = ConnectionParameters(
+			host=self.conf.host,
+			port=self.conf.port,
+			credentials=PlainCredentials(self.conf.username, self.conf.password),
+			virtual_host=self.conf.vhost,
+			ssl_options=self.conf.ssl,
+			blocked_connection_timeout=self.conf.timeout,
+			heartbeat=self.conf.heartbeat
+		)
+		self.connection = None
+		self.publisher = None
+		return
+
+	def __del__(self):
+		self.close()
+		return
+
+	def connect(self):
+		if self.connection and self.connection.is_open:
+			return
+		self.connection = BlockingConnection(parameters=self.parameters)
+		return
+
+	def connected(self):
+		if self.connection is None or not self.connection.is_open:
+			raise RuntimeError('Connect the RabbitMQ connection before use')
+		return self.connection
+
+	def declare(
+		self,
+		parameters: Any,
+	) -> None:
+		subscription = Subscription.parse(parameters)
+		channel = self.connected().channel()
+		try:
+			arguments = {}
+			if subscription.error is not None:
+				channel.queue_declare(
+					subscription.error,
+					durable=True,
+				)
+				arguments = {
+					'x-dead-letter-exchange': '',
+					'x-dead-letter-routing-key': subscription.error,
+				}
+			channel.queue_declare(
+				subscription.queue,
+				durable=True,
+				arguments=arguments,
+			)
+		finally:
+			if channel.is_open:
+				channel.close()
+
+	def send(
+		self,
+		body: Any,
+		parameters: Any,
+	) -> str:
+		subscription = Subscription.parse(parameters)
+		connection = self.connected()
+		channel = self.publisher
+		if channel is None or not channel.is_open:
+			channel = connection.channel()
+			try:
+				channel.confirm_delivery()
+			except Exception:
+				if channel.is_open:
+					channel.close()
+				raise
+			self.publisher = channel
+		id = uuid4().hex
+		channel.basic_publish(
+			exchange='',
+			routing_key=subscription.queue,
+			body=self.conf.encode(body),
+			properties=BasicProperties(
+				message_id=id,
+				headers={},
+				content_type=self.conf.encode.format,
+				content_encoding=self.conf.encode.charset,
+				delivery_mode=2,
+			),
+			mandatory=True,
+		)
+		return id
+
+	def request(
+		self,
+		body: Any,
+		parameters: Any,
+		timeout: float = 5,
+	) -> Any:
+		subscription = Subscription.parse(parameters)
+		self.connected()
+		queue = self.queue(subscription.queue)
+		try:
+			return queue.call(
+				body,
+				timeout=timeout,
+			)
+		finally:
+			if queue.channel.is_open:
+				queue.channel.close()
+
+	def exchange(self, exchange: str) -> Exchange:
+		return Exchange(
+			self.connection,
+			self.conf.encode,
+			exchange,
+			decode=self.conf.decode,
+			rpc_parameters=self.parameters,
+		)
+
+	def queue(self, queue: str) -> Queue:
+		return Queue(
+			self.connection,
+			self.conf.encode,
+			self.conf.decode,
+			queue,
+			rpc_parameters=self.parameters,
+		)
+
+	def stream(self, queue: str) -> Stream:
+		return Stream(self.connection, self.conf.encode, self.conf.decode, queue)
+
+	def consumer(
+		self,
+		handler: Optional[EventHandler] = None,
+		qos: int = 1,
+		offset: int = None,
+	) -> Consumer:
+		return Consumer(
+			self.connection,
+			self.conf.decode,
+			handler,
+			qos=qos,
+			offset=offset,
+			encode=self.conf.encode,
+		)
+
+	def close(self):
+		if self.connection and self.connection.is_open:
+			self.connection.close()
+			self.connection = None
+		self.publisher = None
+		return
+
+	def create_exchange(
+		self,
+		name: str,
+		type: ExchangeType,
+		durable: bool = True,
+		autodelete: bool = False,
+		alter: Union[str, Exchange] = None,
+	):
+		args = {}
+		if alter:
+			args['alternate-exchange'] = str(alter)
+		channel = self.connection.channel()
+		channel.exchange_declare(
+			name,
+			str(type),
+			durable=durable,
+			auto_delete=autodelete,
+			arguments=args
+		)
+		return
+
+	def delete_exchange(
+		self,
+		name: str,
+	):
+		channel = self.connection.channel()
+		channel.exchange_delete(name)
+		return
+
+	def bind_exchange_to_exchange(
+		self,
+		src: Union[str, Exchange],
+		to: Union[str, Exchange],
+		event: str = None,
+		parameters: Parameters = None,
+	):
+		channel = self.connection.channel()
+		channel.exchange_bind(
+			str(src),
+			str(to),
+			routing_key=event if event else '',
+			arguments=parameters
+		)
+		return
+
+	def unbind_exchange_to_exchange(
+		self,
+		src: Union[str, Exchange],
+		to: Union[str, Exchange],
+		event: str = None,
+		parameters: Parameters = None,
+	):
+		channel = self.connection.channel()
+		channel.exchange_unbind(
+			str(src),
+			str(to),
+			routing_key=event if event else '',
+			arguments=parameters,
+		)
+		return
+
+	def create_queue(
+		self,
+		name: str,
+		expires: int = None,
+		ttl: int = None,
+		limit: int = None,
+		size: int = None,
+		maxPriority: int = None,
+		durable: bool = True,
+		autodelete: bool = False,
+		errorQueue: Union[str, Queue] = None,
+		errorExchange: Union[str, Exchange] = None,
+	):
+		args = {}
+		if errorQueue:
+			args['x-dead-letter-exchange'] = ''
+			args['x-dead-letter-routing-key'] = str(errorQueue)
+		if errorExchange:
+			args['x-dead-letter-exchange'] = str(errorExchange)
+		if expires:
+			args['x-expires'] = expires
+		if ttl:
+			args['x-message-ttl'] = ttl
+		if limit:
+			args['x-max-length'] = limit
+		if size:
+			args['x-max-length-bytes'] = size
+		if maxPriority:
+			args['x-max-priority'] = maxPriority
+		channel = self.connection.channel()
+		channel.queue_declare(
+			name,
+			durable=durable,
+			auto_delete=autodelete,
+			arguments=args
+		)
+		return
+
+	def delete_queue(
+		self,
+		queue: str,
+	):
+		channel = self.connection.channel()
+		channel.queue_delete(queue)
+		return
+
+	def bind_exchange_to_queue(
+		self,
+		src: Union[str, Exchange],
+		to: Union[str, Queue],
+		event: str = None,
+		parameters: Parameters = None,
+	):
+		channel = self.connection.channel()
+		channel.queue_bind(
+			str(to),
+			str(src),
+			routing_key=event if event else '',
+			arguments=parameters
+		)
+		return
+
+	def unbind_exchange_to_queue(
+		self,
+		src: Union[str, Exchange],
+		to: Union[str, Queue],
+		event: str = None,
+		parameters: Parameters = None,
+	):
+		channel = self.connection.channel()
+		channel.queue_unbind(
+			str(to),
+			str(src),
+			routing_key=event if event else '',
+			arguments=parameters,
+		)
+		return
+	
+	def create_stream(
+		self,
+		name: str,
+		size: int = None,
+		durable: bool = True,
+		autodelete: bool = False,
+	):
+		channel = self.connection.channel()
+		args = {
+			'x-queue-type': 'stream',
+		}
+		if size:
+			args['x-max-length-bytes'] = size
+		channel.queue_declare(
+			name,
+			durable=durable,
+			auto_delete=autodelete,
+			arguments=args
+		)
+		return
+	
+	def delete_stream(
+		self,
+		queue: str,
+	):
+		channel = self.connection.channel()
+		channel.queue_delete(queue)
+		return
+
+
+class All(Parameters):
+	"""All Bind Parameters Event Broker for RabbitMQ"""
+	def __init__(self, **kwargs):
+		headers = {
+			'x-match': 'all-with-x'
+		}
+		headers.update(kwargs)
+		super().__init__(**headers)
+		return
+
+
+class Any(Parameters):
+	"""Any Bind Parameters Event Broker for RabbitMQ"""
+	def __init__(self, **kwargs):
+		headers = {
+			'x-match': 'any-with-x'
+		}
+		headers.update(kwargs)
+		super().__init__(**headers)
+		return

DefStream/Broker/Implements/RabbitMQ/Consumer.py

@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*-
+
+from ... import (
+	Consumer as BaseConsumer,
+	EventHandler,
+)
+
+from .Serializer import (
+	Decoder,
+	Encoder,
+	JavaScriptObjectNotationEncoder,
+)
+from logging import getLogger
+
+from .Event import Event
+from .Policies import Requeue
+from .Subscription import Subscription
+from ...Event import (
+	Event as BaseEvent,
+	EventError,
+)
+from typing import (
+	Any,
+	Callable,
+	Optional,
+)
+
+from pika import (
+	BasicProperties,
+	BlockingConnection,
+)
+from pika.exceptions import UnroutableError
+
+__all__ = (
+	'Consumer'
+)
+
+logger = getLogger(__name__)
+
+
+class Consumer(BaseConsumer):
+	"""Consumer of Event Broker for RabbitMQ"""
+	def __init__(
+		self,
+		connection: BlockingConnection,
+		decode: Decoder,
+		handler: Optional[EventHandler] = None,
+		qos: int = 1,
+		offset: int = None,
+		encode: Optional[Encoder] = None,
+	):
+		self.connection = connection
+		self.decode = decode
+		self.channel = self.connection.channel()
+		self.qos = qos
+		self.channel.basic_qos(
+			prefetch_count=qos,
+			global_qos=False,
+		)
+		self.channel.auto_decode = False
+		self.offset = offset
+		self.handler = handler
+		self.encode = encode if encode is not None else JavaScriptObjectNotationEncoder()
+		self.response_channel = None
+		self.queues = set()
+		self.stopped = False
+		return
+
+	def __del__(self):
+		self.close()
+		return
+
+	def subs(
+		self,
+		queue: Any,
+		handler: Optional[EventHandler] = None,
+	) -> None:
+		if handler is None:
+			handler = self.handler
+		if handler is None:
+			raise ValueError('subs requires a handler or a default handler')
+		if isinstance(
+			queue,
+			str,
+		):
+			subscription = Subscription(
+				queue=queue,
+				offset=self.offset,
+			)
+		else:
+			subscription = Subscription.parse(queue)
+		if subscription.queue in self.queues:
+			raise ValueError('A Run is already subscribed to queue {}'.format(subscription.queue))
+		self.channel.basic_qos(
+			prefetch_count=self.qos if subscription.qos is None else subscription.qos,
+			global_qos=False,
+		)
+		self.channel.basic_consume(
+			subscription.queue,
+			Callback(
+				self.decode,
+				handler,
+				subscription.queue,
+			),
+			auto_ack=False,
+			exclusive=False,
+			arguments={'x-stream-offset': subscription.offset} if subscription.offset is not None else None,
+		)
+		self.queues.add(subscription.queue)
+
+	def call(self, callback: Callable[[], None]) -> None:
+		self.connection.add_callback_threadsafe(callback)
+
+	def fail(
+		self,
+		event: BaseEvent,
+		policy: Optional[EventError] = None,
+	) -> None:
+		if isinstance(
+			event,
+			Event,
+		) and event.properties.reply_to:
+			event = FailedRequest(
+				self,
+				event,
+			)
+		(policy if policy is not None else Requeue())(event)
+
+	def complete(
+		self,
+		event: BaseEvent,
+		result: Any,
+	) -> None:
+		if isinstance(
+			event,
+			Event,
+		) and event.properties.reply_to:
+			self.respond(
+				event,
+				{
+					'ok': True,
+					'result': result,
+				},
+			)
+		event.ack()
+
+	def respond(
+		self,
+		event: Event,
+		response: Any,
+	) -> None:
+		if not event.properties.correlation_id:
+			raise ValueError('RPC request requires correlation_id')
+		channel = self.response_channel
+		if channel is None:
+			channel = self.connection.channel()
+			channel.confirm_delivery()
+			self.response_channel = channel
+		try:
+			channel.basic_publish(
+				exchange='',
+				routing_key=event.properties.reply_to,
+				body=self.encode(response),
+				properties=BasicProperties(
+					correlation_id=event.properties.correlation_id,
+					content_type=self.encode.format,
+					content_encoding=self.encode.charset,
+				),
+				mandatory=True,
+			)
+		except UnroutableError:
+			logger.warning(
+				'RPC reply is unroutable; settling request without a response: correlation_id=%s reply_to=%s',
+				event.properties.correlation_id,
+				event.properties.reply_to,
+			)
+
+	def close(self) -> None:
+		response_channel = getattr(self, 'response_channel', None)
+		if response_channel is not None and response_channel.is_open:
+			response_channel.close()
+		channel = getattr(self, 'channel', None)
+		if channel is not None and channel.is_open:
+			channel.close()
+
+	def run(self):
+		self.channel.start_consuming()
+		return
+
+	def stop(self):
+		if not self.stopped:
+			self.stopped = True
+			self.channel.stop_consuming()
+		return
+
+
+class FailedRequest(BaseEvent):
+	def __init__(
+		self,
+		consumer: Consumer,
+		event: Event,
+	):
+		self.consumer = consumer
+		self.event = event
+
+	@property
+	def body(self) -> Any:
+		return self.event.body
+
+	def ack(self) -> None:
+		self.event.ack()
+
+	def nack(self) -> None:
+		self.event.nack()
+
+	def reject(self) -> None:
+		self.consumer.respond(
+			self.event,
+			{
+				'ok': False,
+				'error': {
+					'code': 'ProcessingError',
+					'message': 'Request processing failed',
+				},
+			},
+		)
+		self.event.reject()
+
+
+class Callback(object):
+	def __init__(self, decode: Decoder, handler: EventHandler, queue: str):
+		self.decode = decode
+		self.handler = handler
+		self.queue = queue
+		return
+	def __call__(self, channel, method, properties, body):
+		return self.handler(Event(
+			channel,
+			self.queue,
+			method.consumer_tag,
+			method.delivery_tag,
+			properties,
+			self.decode(body, properties.content_type, properties.content_encoding),
+		))