DefStream 0.1.17__64-py3-none-any.whl

DefStream/Broker/Implements/RabbitMQ/Stream.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+
+from DefStream.Broker import (
+	Sendable,
+	Readable,
+)
+from .Serializer import (
+	Encoder,
+	Decoder,
+)
+from .Event import Event
+
+from pika import (
+	BlockingConnection,
+	BasicProperties,
+)
+
+from time import time
+from logging import getLogger
+from uuid import uuid4
+
+from typing import (
+	Dict,
+	Iterable,
+)
+
+__all__ = (
+	'Queue'
+)
+
+
+class Stream(
+	Sendable,
+	Readable,
+):
+	"""Stream of Event Broker for RabbitMQ"""
+	logger = getLogger('DefStream.Broker.Implements.RabbitMQ.Stream')
+
+	def __init__(
+		self,
+		connection: BlockingConnection,
+		encode: Encoder,
+		decode: Decoder,
+		name: str = None,
+	):
+		self.connection = connection
+		self.encode = encode
+		self.decode = decode
+		try:
+			self.channel = self.connection.channel()
+		except Exception as error:
+			self.logger.error('Stream channel creation failed: queue=%s error_type=%s', name, type(error).__name__)
+			raise
+		self.channel.auto_decode = False
+		self.queue = name
+		self.event = None
+		return
+
+	def __del__(self):
+		channel = getattr(self, 'channel', None)
+		try:
+			if channel and channel.is_open:
+				channel.close()
+				self.logger.debug('Stream channel cleanup completed: queue=%s', self.queue)
+		except Exception as error:
+			self.logger.error('Stream channel cleanup failed: error_type=%s', type(error).__name__)
+			raise
+		return
+	
+	def __str__(self): return self.queue
+
+	def send(
+		self,
+		body,
+		event: str = None,
+		headers: Dict = {},
+		timestamp: int = None,
+		id: str = None,
+	):
+		if not timestamp: timestamp = int(time())
+		if not id: id = uuid4().hex
+		properties = BasicProperties(
+			type=event if event else '',
+			headers=headers,
+			content_type=self.encode.format,
+			content_encoding=self.encode.charset,
+			timestamp=timestamp,
+			message_id=id,
+		)
+		self.logger.debug('Publishing Stream message: queue=%s message_id=%s publisher_confirm=False', self.queue, id)
+		try:
+			self.channel.basic_publish(
+				exchange='',
+				routing_key=self.queue,
+				properties=properties,
+				body=self.encode(body)
+			)
+		except Exception as error:
+			self.logger.error('Stream publication failed: queue=%s message_id=%s error_type=%s', self.queue, id, type(error).__name__)
+			raise
+		self.logger.debug('Stream publish API call completed: queue=%s message_id=%s', self.queue, id)
+		return id
+
+	def read(self, offset: int = 0, timeout: int = None) -> Iterable[Event]:
+		channel = self.channel
+		self.logger.debug('Starting Stream read: queue=%s offset=%s timeout_ms=%s qos=1', self.queue, offset, timeout)
+		try:
+			channel.basic_qos(0, 1, False)
+		except Exception as error:
+			self.logger.error('Failed to apply Stream QoS: queue=%s error_type=%s', self.queue, type(error).__name__)
+			raise
+
+		deadline = time() + (timeout / 1000) if timeout else None
+
+		failure_observed = False
+		try:
+			for method, properties, body in channel.consume(
+				queue=self.queue,
+				auto_ack=False,
+				inactivity_timeout=1,
+				arguments={'x-stream-offset': offset},
+			):
+				if method is None:
+					if deadline and time() >= deadline:
+						break
+					continue
+
+				try:
+					decoded = self.decode(body, properties.content_type, properties.content_encoding)
+				except Exception as error:
+					failure_observed = True
+					self.logger.error('Stream decoding failed: queue=%s delivery_tag=%s error_type=%s', self.queue, method.delivery_tag, type(error).__name__)
+					raise
+				self.logger.debug('Stream message received and decoded: queue=%s consumer_tag=%s delivery_tag=%s message_id=%s', self.queue, method.consumer_tag, method.delivery_tag, properties.message_id)
+				event = Event(
+					channel,
+					self.queue,
+					method.consumer_tag,
+					method.delivery_tag,
+					properties,
+					decoded,
+				)
+
+				try:
+					failure_observed = True
+					yield event
+					failure_observed = False
+					self.logger.debug('Stream ack requested: queue=%s delivery_tag=%s', self.queue, method.delivery_tag)
+					try:
+						channel.basic_ack(delivery_tag=method.delivery_tag)
+					except Exception as error:
+						failure_observed = True
+						self.logger.error('Stream ack failed: queue=%s delivery_tag=%s error_type=%s', self.queue, method.delivery_tag, type(error).__name__)
+						raise
+					self.logger.debug('Stream ack API call completed: queue=%s delivery_tag=%s', self.queue, method.delivery_tag)
+				except Exception:
+					failure_observed = True
+					self.logger.warning('Attempting requeue after Stream processing exception: queue=%s delivery_tag=%s', self.queue, method.delivery_tag)
+					self.logger.debug('Stream nack requested: queue=%s delivery_tag=%s requeue=True', self.queue, method.delivery_tag)
+					try:
+						channel.basic_nack(delivery_tag=method.delivery_tag, requeue=True)
+					except Exception as error:
+						self.logger.error('Stream nack failed: queue=%s delivery_tag=%s error_type=%s', self.queue, method.delivery_tag, type(error).__name__)
+						raise
+					self.logger.debug('Stream nack API call completed: queue=%s delivery_tag=%s', self.queue, method.delivery_tag)
+					raise
+		except Exception as error:
+			if not failure_observed:
+				self.logger.error('Stream consumption loop failed: queue=%s error_type=%s', self.queue, type(error).__name__)
+			raise
+		finally:
+			self.logger.debug('Ending Stream read loop; requesting Consumer cancellation: queue=%s', self.queue)
+			try:
+				channel.cancel()
+			except Exception as error:
+				self.logger.error('Stream Consumer cancellation failed: queue=%s error_type=%s', self.queue, type(error).__name__)
+				raise
+			self.logger.debug('Stream Consumer cancelled: queue=%s', self.queue)
+		return

DefStream/Broker/Implements/RabbitMQ/Subscription.py

@@ -0,0 +1,74 @@
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import (
+	Any,
+	Optional,
+)
+from DefStream.Broker import validate_priority
+
+__all__ = (
+	'Subscription',
+)
+
+
+@dataclass(frozen=True)
+class Subscription:
+	queue: str
+	offset: Optional[int] = None
+	qos: Optional[int] = None
+	error: Optional[str] = None
+	max_priority: Optional[int] = None
+	priority: Optional[int] = None
+
+	def __post_init__(self):
+		if self.priority is not None and self.max_priority is None:
+			object.__setattr__(
+				self,
+				'max_priority',
+				255,
+			)
+		validate_priority(
+			self.max_priority,
+			255,
+			minimum=1,
+			name='max_priority',
+		)
+		self.validate_priority(self.priority)
+		if not isinstance(self.queue, str) or not self.queue:
+			raise ValueError('Subscription queue must be a non-empty string')
+		if self.error is not None:
+			if not isinstance(
+				self.error,
+				str,
+			) or not self.error:
+				raise ValueError('Subscription error must be a non-empty string or None')
+		if self.offset is not None and (isinstance(self.offset, bool) or not isinstance(self.offset, int)):
+			raise ValueError('Subscription offset must be an integer')
+		if self.qos is not None:
+			if isinstance(
+				self.qos,
+				bool,
+			) or not isinstance(
+				self.qos,
+				int,
+			) or not 0 <= self.qos <= 65535:
+				raise ValueError('Subscription qos must be an integer between 0 and 65535')
+		return
+
+	def validate_priority(
+		self,
+		priority: Optional[int],
+	) -> None:
+		validate_priority(
+			priority,
+			255 if self.max_priority is None else self.max_priority,
+		)
+		return
+
+	@classmethod
+	def parse(cls, parameters: Any) -> 'Subscription':
+		if isinstance(parameters, cls):
+			return parameters
+		if isinstance(parameters, Mapping):
+			return cls(**dict(parameters))
+		raise TypeError('RabbitMQ subscriptions require a Subscription or mapping')

DefStream/Broker/Implements/RabbitMQ/__init__.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+from .Configuration import Configuration
+from .Connection import (
+	Connection,
+	ExchangeType,
+	Parameters,
+	All,
+	Any,
+)
+from .Exchange import Exchange
+from .Queue import Queue
+from .Stream import Stream
+from .Consumer import Consumer
+from .Subscription import Subscription
+from .Policies import (
+	Requeue,
+	Reject,
+)
+from .Event import (
+	Event,
+	EventHandler,
+)
+from .Serializer import (
+	Encoder,
+	Decoder,
+)
+
+__all__ = (
+	'Configuration',
+	'Connection',
+	'ExchangeType',
+	'Parameters',
+	'All',
+	'Any',
+	'Exchange',
+	'Queue',
+	'Stream',
+	'Consumer',
+	'Subscription',
+	'Requeue',
+	'Reject',
+	'Event',
+	'EventHandler',
+	'Encoder',
+	'Decoder',
+)

DefStream/Broker/Implements/Redis/Configuration.py

@@ -0,0 +1,59 @@
+from enum import Enum
+from typing import Optional
+
+from DefStream.Broker import Configuration as BaseConfiguration
+from .Serializer import (
+	Decoder,
+	Encoder,
+	JavaScriptObjectNotationDecoder,
+	JavaScriptObjectNotationEncoder,
+)
+
+__all__ = (
+	'Configuration',
+	'ConnectionType',
+)
+
+
+class ConnectionType(Enum):
+	Pool = 'pool'
+	Cluster = 'cluster'
+
+
+class Configuration(BaseConfiguration):
+	def __init__(
+		self,
+		host: str = '127.0.0.1',
+		port: int = 6379,
+		username: Optional[str] = None,
+		password: Optional[str] = None,
+		secure: bool = False,
+		type: ConnectionType = ConnectionType.Pool,
+		max: int = 10,
+		database: int = 0,
+		timeout: float = 5,
+		encode: Optional[Encoder] = None,
+		decode: Optional[Decoder] = None,
+	):
+		if not isinstance(
+			type,
+			ConnectionType,
+		):
+			raise ValueError('Redis connection type must be a ConnectionType')
+		if max < 1:
+			raise ValueError('Redis max connections must be positive')
+		if database < 0 or (type == ConnectionType.Cluster and database != 0):
+			raise ValueError('Redis database must be non-negative; Cluster requires database 0')
+		if timeout <= 0:
+			raise ValueError('Redis connection timeout must be positive')
+		self.host = host
+		self.port = port
+		self.username = username
+		self.password = password
+		self.secure = secure
+		self.type = type
+		self.max = max
+		self.database = database
+		self.timeout = timeout
+		self.encode = encode if encode is not None else JavaScriptObjectNotationEncoder()
+		self.decode = decode if decode is not None else JavaScriptObjectNotationDecoder()

DefStream/Broker/Implements/Redis/Connection.py

@@ -0,0 +1,265 @@
+from logging import getLogger
+from typing import (
+	Any,
+	Optional,
+	Union,
+)
+
+from redis import Redis
+from redis.cluster import RedisCluster
+
+from DefStream.Broker import (
+	Connection as BaseConnection,
+	GetQueue,
+	GetStream,
+	GetConsumer,
+	Declarable,
+	Emittable,
+	EventHandler,
+)
+from .Configuration import (
+	Configuration,
+	ConnectionType,
+)
+from .Consumer import Consumer
+from .Queue import Queue
+from .Subscription import Subscription
+
+__all__ = (
+	'Connection',
+)
+
+
+class Connection(
+	BaseConnection,
+	GetQueue,
+	GetStream,
+	GetConsumer,
+	Declarable,
+	Emittable,
+):
+	logger = getLogger('DefStream.Broker.Implements.Redis.Connection')
+
+	def __init__(
+		self,
+		conf: Configuration,
+	):
+		self.conf = conf
+		self.connection: Optional[Union[Redis, RedisCluster]] = None
+
+	def connect(self) -> None:
+		if self.connection is not None:
+			self.logger.debug('Reusing existing Redis connection')
+			return
+		connection_type = self.conf.type.value
+		self.logger.info(
+			'Connecting to Redis: type=%s',
+			connection_type,
+		)
+		self.logger.debug(
+			'Redis connection settings: host=%s port=%s database=%s TLS=%s max=%s timeout=%ss',
+			self.conf.host,
+			self.conf.port,
+			self.conf.database,
+			self.conf.secure,
+			self.conf.max,
+			self.conf.timeout,
+		)
+		parameters = {
+			'host': self.conf.host,
+			'port': self.conf.port,
+			'username': self.conf.username,
+			'password': self.conf.password,
+			'ssl': self.conf.secure,
+			'max_connections': self.conf.max,
+			'socket_connect_timeout': self.conf.timeout,
+			'socket_timeout': self.conf.timeout,
+			'decode_responses': False,
+		}
+		connection = None
+		try:
+			if self.conf.type == ConnectionType.Cluster:
+				connection = RedisCluster(**parameters)
+			else:
+				connection = Redis(
+					db=self.conf.database,
+					**parameters,
+				)
+			connection.ping()
+		except Exception as error:
+			self.logger.error(
+				'Redis connection or PING failed: type=%s error_type=%s',
+				connection_type,
+				type(error).__name__,
+			)
+			if connection is not None:
+				connection.close()
+			raise
+		self.connection = connection
+		self.logger.info(
+			'Redis connection verified with PING: type=%s',
+			connection_type,
+		)
+
+	def connected(self) -> Union[Redis, RedisCluster]:
+		if self.connection is None:
+			raise RuntimeError('Connect the Redis broker before use')
+		return self.connection
+
+	def declare(
+		self,
+		parameters: Any,
+	) -> None:
+		subscription = Subscription.parse(parameters)
+		self.logger.debug(
+			'Delegating stream declaration: queue=%s group=%s',
+			subscription.queue,
+			subscription.group,
+		)
+		self.create_queue(
+			subscription.queue,
+			group=subscription.group,
+		)
+
+	def emit(
+		self,
+		body: Any,
+		parameters: Any,
+	) -> str:
+		subscription = Subscription.parse(parameters)
+		self.logger.debug(
+			'Delegating message publication: queue=%s group=%s',
+			subscription.queue,
+			subscription.group,
+		)
+		return self.queue(
+			subscription.queue,
+			group=subscription.group,
+		).send(body)
+
+	def queue(
+		self,
+		name: str,
+		group: str = 'DefStream',
+	) -> Queue:
+		self.logger.debug(
+			'Creating Queue: queue=%s group=%s',
+			name,
+			group,
+		)
+		return Queue(
+			self.connected(),
+			self.conf.encode,
+			self.conf.decode,
+			name,
+			group,
+		)
+
+	def stream(
+		self,
+		name: str,
+		group: str = 'DefStream',
+	) -> Queue:
+		return self.queue(
+			name,
+			group,
+		)
+
+	def consumer(
+		self,
+		handler: Optional[EventHandler] = None,
+		qos: int = 1,
+		reclaim_idle: Optional[int] = 60000,
+	) -> Consumer:
+		self.logger.debug(
+			'Creating Consumer: qos=%s reclaim_idle=%sms',
+			qos,
+			reclaim_idle,
+		)
+		return Consumer(
+			self.connected(),
+			self.conf.encode,
+			self.conf.decode,
+			handler,
+			qos,
+			reclaim_idle,
+		)
+
+	def create_queue(
+		self,
+		name: str,
+		group: str = 'DefStream',
+	) -> None:
+		self.logger.debug(
+			'Delegating Consumer Group creation: queue=%s group=%s',
+			name,
+			group,
+		)
+		self.queue(
+			name,
+			group,
+		).create_group()
+
+	def delete_queue(
+		self,
+		name: Union[str, Queue],
+	) -> int:
+		queue = str(name)
+		self.logger.debug(
+			'Deleting Redis stream: queue=%s',
+			queue,
+		)
+		try:
+			deleted = self.connected().delete(queue)
+		except Exception as error:
+			self.logger.error(
+				'Redis stream deletion failed: queue=%s error_type=%s',
+				queue,
+				type(error).__name__,
+			)
+			raise
+		if isinstance(deleted, int) and deleted > 0:
+			self.logger.info(
+				'Redis stream deletion completed: queue=%s deleted=%s',
+				queue,
+				deleted,
+			)
+		else:
+			self.logger.info(
+				'No Redis stream to delete: queue=%s deleted=%s',
+				queue,
+				deleted,
+			)
+		return deleted
+
+	def create_stream(
+		self,
+		name: str,
+		group: str = 'DefStream',
+	) -> None:
+		self.create_queue(
+			name,
+			group,
+		)
+
+	def delete_stream(
+		self,
+		name: Union[str, Queue],
+	) -> int:
+		return self.delete_queue(name)
+
+	def close(self) -> None:
+		if self.connection is not None:
+			self.logger.info('Closing Redis connection')
+			try:
+				self.connection.close()
+			except Exception as error:
+				self.logger.error(
+					'Redis connection close failed: error_type=%s',
+					type(error).__name__,
+				)
+				raise
+			self.connection = None
+			self.logger.info('Redis connection closed')
+		else:
+			self.logger.debug('No Redis connection to close')