DefStream 0.1.9__25-py3-none-any.whl

DefStream/Worker/Pool.py

@@ -0,0 +1,245 @@
+from abc import (
+	ABC,
+	abstractmethod,
+)
+from concurrent.futures import (
+	Executor as PyExecutor,
+	Future,
+	wait,
+)
+from os import cpu_count
+from logging import getLogger
+from threading import (
+	BoundedSemaphore,
+	Lock,
+)
+from typing import (
+	Any,
+	Callable,
+	Dict,
+	Optional,
+	Sequence,
+	Set,
+	Tuple,
+	Type,
+	Union,
+)
+
+from .Run import Run
+from .Executor import Executor
+
+__all__ = (
+	'Pool',
+	'PoolFull',
+	'PoolSetup',
+	'PoolCleanup',
+	'ThreadPool',
+	'ProcessPool',
+)
+
+
+class PoolFull(RuntimeError):
+	pass
+
+
+class Pool:
+	logger = getLogger('DefStream.Pool')
+
+	def __init__(
+		self,
+		executor: PyExecutor,
+		max: int,
+	):
+		self.pool = executor
+		self.max = max
+		self.slots = BoundedSemaphore(max)
+		self.lock = Lock()
+		self.futures: Set[Future] = set()
+		self.closed = False
+		self.logger.info(
+			'Pool configured: concurrency=%s',
+			max,
+		)
+
+	def submit(
+		self,
+		run: Type[Run],
+		body: Dict[str, Any],
+		callback: Optional[Callable[[Any], None]] = None,
+		error_callback: Optional[Callable[[BaseException], None]] = None,
+	) -> Future:
+		with self.lock:
+			if self.closed:
+				raise RuntimeError('Pool is closed')
+			if not self.slots.acquire(blocking=False):
+				self.logger.debug(
+					'Pool capacity exhausted: active=%s capacity=%s',
+					len(self.futures),
+					self.max,
+				)
+				raise PoolFull('Pool capacity is exhausted')
+			try:
+				future = self.pool.submit(
+					run(),
+					**body,
+				)
+			except BaseException:
+				self.slots.release()
+				raise
+			self.futures.add(future)
+			self.logger.debug(
+				'Task submission accepted: task=%s.%s active=%s',
+				run.__module__,
+				run.__qualname__,
+				len(self.futures),
+			)
+		future.add_done_callback(lambda completed: self.complete(
+			completed,
+			callback,
+			error_callback,
+		))
+		return future
+
+	def complete(
+		self,
+		future: Future,
+		callback: Optional[Callable[[Any], None]] = None,
+		error_callback: Optional[Callable[[BaseException], None]] = None,
+	) -> None:
+		with self.lock:
+			self.futures.remove(future)
+			self.slots.release()
+			self.logger.debug(
+				'Task slot released: active=%s',
+				len(self.futures),
+			)
+		try:
+			result = future.result()
+		except BaseException as error:
+			self.logger.debug(
+				'Delivering failed result: error_type=%s',
+				type(error).__qualname__,
+			)
+			if error_callback is not None:
+				error_callback(error)
+		else:
+			self.logger.debug('Delivering successful result')
+			if callback is not None:
+				callback(result)
+
+	def waits(
+		self,
+		timeout: Optional[float] = None,
+	) -> None:
+		with self.lock:
+			tasks = tuple(self.futures)
+		self.logger.debug(
+			'Waiting for Pool tasks: active=%s timeout=%s',
+			len(tasks),
+			timeout,
+		)
+		if tasks and wait(
+			tasks,
+			timeout=timeout,
+		).not_done:
+			self.logger.warning(
+				'Pool task wait timed out: timeout=%s',
+				timeout,
+			)
+			raise TimeoutError('Pool tasks did not complete in time')
+
+	def stop(self) -> None:
+		self.logger.info('Stopping Pool')
+		with self.lock:
+			self.closed = True
+		self.pool.shutdown(wait=True)
+		self.logger.info('Pool shutdown completed')
+
+	def shutdown(self) -> None:
+		self.stop()
+
+	def size(self) -> int:
+		return self.max
+
+	def count(self) -> int:
+		with self.lock:
+			return len(self.futures)
+
+
+class PoolSetup(ABC):
+	@abstractmethod
+	def __call__(self) -> None:
+		raise NotImplementedError
+
+
+class PoolCleanup(ABC):
+	@abstractmethod
+	def __call__(self) -> None:
+		raise NotImplementedError
+
+
+def pool_hooks(
+	hooks: Optional[Union[Callable[[], None], Sequence[Callable[[], None]]]],
+) -> Tuple[Callable[[], None], ...]:
+	result = () if hooks is None else tuple(hooks) if isinstance(
+		hooks,
+		Sequence,
+	) else (hooks,)
+	if not all(callable(hook) for hook in result):
+		raise TypeError('Pool hooks must be callable')
+	return result
+
+
+def pool_size(max: Optional[int]) -> int:
+	if max is None:
+		return cpu_count() or 1
+	if isinstance(
+		max,
+		bool,
+	) or not isinstance(
+		max,
+		int,
+	) or max < 1:
+		raise ValueError('Pool size must be a positive integer')
+	return max
+
+
+class ThreadPool(Pool):
+	logger = getLogger('DefStream.ThreadPool')
+
+	def __init__(
+		self,
+		max: Optional[int] = None,
+		setups: Optional[Union[PoolSetup, Sequence[PoolSetup]]] = None,
+		cleanups: Optional[Union[PoolCleanup, Sequence[PoolCleanup]]] = None,
+	):
+		size = pool_size(max)
+		super().__init__(
+			Executor(
+				size,
+				pool_hooks(setups),
+				pool_hooks(cleanups),
+			),
+			size,
+		)
+
+
+class ProcessPool(Pool):
+	logger = getLogger('DefStream.ProcessPool')
+
+	def __init__(
+		self,
+		max: Optional[int] = None,
+		setups: Optional[Union[PoolSetup, Sequence[PoolSetup]]] = None,
+		cleanups: Optional[Union[PoolCleanup, Sequence[PoolCleanup]]] = None,
+	):
+		size = pool_size(max)
+		super().__init__(
+			Executor(
+				size,
+				pool_hooks(setups),
+				pool_hooks(cleanups),
+				process=True,
+			),
+			size,
+		)

DefStream/Worker/Run.py

@@ -0,0 +1,311 @@
+from abc import (
+	ABC,
+	abstractmethod,
+)
+from collections.abc import Mapping
+from logging import getLogger
+from time import monotonic
+from typing import (
+	Any,
+	Optional,
+	Sequence,
+	Tuple,
+	Union,
+)
+
+from ..Broker.Event import EventError
+from ..Broker.Priority import validate_priority
+from .Specification import Specification
+
+__all__ = (
+	'Context',
+	'RunSetup',
+	'RunComplete',
+	'RunError',
+	'Run',
+)
+
+
+def priority_setting(
+	parameters: Any,
+	name: str,
+) -> Optional[int]:
+	if isinstance(
+		parameters,
+		Mapping,
+	):
+		return parameters.get(name)
+	return getattr(
+		parameters,
+		name,
+		None,
+	)
+
+
+class Context:
+	def __init__(
+		self,
+		*args,
+		**kwargs,
+	):
+		self.args = args
+		self.kwargs = kwargs
+
+
+class RunSetup(ABC):
+	@abstractmethod
+	def __call__(
+		self,
+		context: Context,
+	) -> None:
+		raise NotImplementedError
+
+
+class RunComplete(ABC):
+	@abstractmethod
+	def __call__(
+		self,
+		context: Context,
+		completion: Any,
+	) -> None:
+		raise NotImplementedError
+
+
+class RunError(ABC):
+	@abstractmethod
+	def __call__(
+		self,
+		context: Context,
+		error: Exception,
+	) -> None:
+		raise NotImplementedError
+
+
+class Run(ABC):
+	logger = getLogger('DefStream.Run')
+	parameters: Any = None
+	on_error: Optional[EventError] = None
+	setups: Tuple[RunSetup, ...] = ()
+	completes: Tuple[RunComplete, ...] = ()
+	errors: Tuple[RunError, ...] = ()
+	request: Optional[Specification] = None
+	response: Optional[Specification] = None
+
+	def __init_subclass__(
+		cls,
+		*,
+		parameters: Any = None,
+		on_error: Optional[EventError] = None,
+		setups: Optional[Union[RunSetup, Sequence[RunSetup]]] = None,
+		completes: Optional[Union[RunComplete, Sequence[RunComplete]]] = None,
+		errors: Optional[Union[RunError, Sequence[RunError]]] = None,
+		request: Optional[Specification] = None,
+		response: Optional[Specification] = None,
+		**kwargs,
+	):
+		super().__init_subclass__(**kwargs)
+		parents = tuple(base for base in cls.__mro__[1:] if issubclass(
+			base,
+			Run,
+		))
+		if parameters is not None:
+			cls.parameters = parameters
+		for name in (
+			'priority',
+			'max_priority',
+		):
+			if any(priority_setting(
+				base.parameters,
+				name,
+			) is not None for base in parents) and priority_setting(
+				cls.parameters,
+				name,
+			) is None:
+				raise ValueError('Inherited task ' + name + ' cannot be cleared')
+		maximum = priority_setting(
+			cls.parameters,
+			'max_priority',
+		)
+		validate_priority(
+			maximum,
+			None,
+			minimum=1,
+			name='max_priority',
+		)
+		validate_priority(
+			priority_setting(
+				cls.parameters,
+				'priority',
+			),
+			maximum,
+		)
+		for name, specification in (
+			(
+				'request',
+				request,
+			),
+			(
+				'response',
+				response,
+			),
+		):
+			if specification is not None:
+				if not isinstance(
+					specification,
+					Specification,
+				):
+					raise TypeError(name + ' must be a Specification')
+				setattr(
+					cls,
+					name,
+					specification,
+				)
+		if on_error is not None:
+			if not callable(on_error):
+				raise TypeError('on_error must be a callable event error policy')
+			cls.on_error = on_error
+		if setups is not None:
+			cls.setups = tuple(setups) if isinstance(
+				setups,
+				Sequence,
+			) else (setups,)
+		if completes is not None:
+			cls.completes = tuple(completes) if isinstance(
+				completes,
+				Sequence,
+			) else (completes,)
+		if errors is not None:
+			cls.errors = tuple(errors) if isinstance(
+				errors,
+				Sequence,
+			) else (errors,)
+		for hooks in (
+			cls.setups,
+			cls.completes,
+			cls.errors,
+		):
+			if not all(callable(hook) for hook in hooks):
+				raise TypeError('Run hooks must be callable')
+		return
+
+	def __call__(
+		self,
+		*args,
+		**kwargs,
+	) -> Any:
+		task = type(self).__module__ + '.' + type(self).__qualname__
+		started = monotonic()
+		self.logger.debug(
+			'Starting task execution: task=%s',
+			task,
+		)
+		context = Context(
+			*args,
+			**kwargs,
+		)
+		for setup in self.setups:
+			self.logger.debug(
+				'Starting task setup: task=%s hook=%s',
+				task,
+				type(setup).__qualname__,
+			)
+			setup(context)
+			self.logger.debug(
+				'Task setup completed: task=%s hook=%s',
+				task,
+				type(setup).__qualname__,
+			)
+		try:
+			if self.request is not None:
+				self.logger.debug(
+					'Validating task input: task=%s',
+					task,
+				)
+				if context.args:
+					raise TypeError('A specified task request requires keyword arguments')
+				body = self.request(context.kwargs)
+				if not isinstance(
+					body,
+					Mapping,
+				) or not all(isinstance(
+					key,
+					str,
+				) for key in body):
+					raise TypeError('A task request must remain a mapping with string keys')
+				context.kwargs = dict(body)
+				self.logger.debug(
+					'Task input validated: task=%s',
+					task,
+				)
+			self.logger.debug(
+				'Calling user task: task=%s',
+				task,
+			)
+			result = self.run(
+				*context.args,
+				**context.kwargs,
+			)
+			self.logger.debug(
+				'User task returned: task=%s',
+				task,
+			)
+			if self.response is not None:
+				self.logger.debug(
+					'Validating task output: task=%s',
+					task,
+				)
+				result = self.response(result)
+				self.logger.debug(
+					'Task output validated: task=%s',
+					task,
+				)
+		except Exception as error:
+			self.logger.debug(
+				'Task execution or validation failed: task=%s error_type=%s',
+				task,
+				type(error).__qualname__,
+			)
+			for handler in self.errors:
+				self.logger.debug(
+					'Starting task error hook: task=%s hook=%s',
+					task,
+					type(handler).__qualname__,
+				)
+				handler(
+					context,
+					error,
+				)
+				self.logger.debug(
+					'Task error hook completed: task=%s hook=%s',
+					task,
+					type(handler).__qualname__,
+				)
+			raise
+		for complete in self.completes:
+			self.logger.debug(
+				'Starting task completion hook: task=%s hook=%s',
+				task,
+				type(complete).__qualname__,
+			)
+			complete(
+				context,
+				result,
+			)
+			self.logger.debug(
+				'Task completion hook completed: task=%s hook=%s',
+				task,
+				type(complete).__qualname__,
+			)
+		self.logger.debug(
+			'Task execution completed: task=%s duration=%s',
+			task,
+			monotonic() - started,
+		)
+		return result
+
+	@abstractmethod
+	def run(
+		self,
+		**kwargs,
+	) -> Any:
+		raise NotImplementedError

DefStream/Worker/Specification.py

@@ -0,0 +1,91 @@
+from collections.abc import Mapping
+from copy import deepcopy
+from dataclasses import is_dataclass
+from re import fullmatch
+from typing import (
+	Any,
+	Callable,
+	Optional,
+)
+
+from ..Descriptor import (
+	DumpSchema,
+	Schema,
+	ToSchema,
+)
+from ..Descriptor.Value import (
+	Condition,
+	Descriptor,
+)
+
+__all__ = (
+	'Specification',
+)
+
+
+class Specification:
+	def __init__(
+		self,
+		schema: Any = None,
+		validator: Optional[Callable[[Any], Any]] = None,
+		example: Any = ...,
+		description: Optional[str] = None,
+		format: str = 'application/json',
+	):
+		if not isinstance(
+			format,
+			str,
+		) or fullmatch(
+			r"[!#$%&'+.^_`|~0-9A-Za-z-]+/[!#$%&'+.^_`|~0-9A-Za-z-]+(?:[ \t]*;[^\r\n]+)?",
+			format,
+		) is None:
+			raise ValueError('Specification format must be a concrete media type')
+		if validator is not None and not callable(validator):
+			raise TypeError('A Specification validator must be callable')
+		if isinstance(
+			schema,
+			type,
+		) and is_dataclass(schema):
+			schema = ToSchema(schema)
+		if schema is not None and not isinstance(
+			schema,
+			(
+				Schema,
+				Descriptor,
+				Condition,
+				Mapping,
+			),
+		):
+			raise TypeError('A Descriptor schema, schema mapping or dataclass type is required')
+		self.schema = schema
+		self.validator = validator
+		self.example = example
+		self.description = description
+		self.format = format
+		self.document()
+
+	def __call__(
+		self,
+		value: Any,
+	) -> Any:
+		return self.validator(value) if self.validator is not None else value
+
+	def document(self) -> dict:
+		payload = {} if self.schema is None else DumpSchema(
+			self.schema if isinstance(
+				self.schema,
+				Schema,
+			) else Schema(
+				'Payload',
+				self.schema,
+			),
+		)
+		message = {
+			'contentType': self.format,
+			'payload': payload,
+		}
+		if self.description is not None:
+			message['description'] = self.description
+		if self.example is not ...:
+			message['examples'] = [{'payload': deepcopy(self.example)}]
+		return message