DefStream 0.1.2__py3-none-any.whl

DefStream/Worker/Run.py

@@ -0,0 +1,187 @@
+from abc import (
+	ABC,
+	abstractmethod,
+)
+from collections.abc import Mapping
+from typing import (
+	Any,
+	Optional,
+	Sequence,
+	Tuple,
+	Union,
+)
+
+from ..Broker.Event import EventError
+from .Specification import Specification
+
+__all__ = (
+	'Context',
+	'RunSetup',
+	'RunComplete',
+	'RunError',
+	'Run',
+)
+
+
+class Context:
+	def __init__(
+		self,
+		*args,
+		**kwargs,
+	):
+		self.args = args
+		self.kwargs = kwargs
+
+
+class RunSetup(ABC):
+	@abstractmethod
+	def __call__(
+		self,
+		context: Context,
+	) -> None:
+		raise NotImplementedError
+
+
+class RunComplete(ABC):
+	@abstractmethod
+	def __call__(
+		self,
+		context: Context,
+		completion: Any,
+	) -> None:
+		raise NotImplementedError
+
+
+class RunError(ABC):
+	@abstractmethod
+	def __call__(
+		self,
+		context: Context,
+		error: Exception,
+	) -> None:
+		raise NotImplementedError
+
+
+class Run(ABC):
+	parameters: Any = None
+	on_error: Optional[EventError] = None
+	setups: Tuple[RunSetup, ...] = ()
+	completes: Tuple[RunComplete, ...] = ()
+	errors: Tuple[RunError, ...] = ()
+	request: Optional[Specification] = None
+	response: Optional[Specification] = None
+
+	def __init_subclass__(
+		cls,
+		*,
+		parameters: Any = None,
+		on_error: Optional[EventError] = None,
+		setups: Optional[Union[RunSetup, Sequence[RunSetup]]] = None,
+		completes: Optional[Union[RunComplete, Sequence[RunComplete]]] = None,
+		errors: Optional[Union[RunError, Sequence[RunError]]] = None,
+		request: Optional[Specification] = None,
+		response: Optional[Specification] = None,
+		**kwargs,
+	):
+		super().__init_subclass__(**kwargs)
+		if parameters is not None:
+			cls.parameters = parameters
+		for name, specification in (
+			(
+				'request',
+				request,
+			),
+			(
+				'response',
+				response,
+			),
+		):
+			if specification is not None:
+				if not isinstance(
+					specification,
+					Specification,
+				):
+					raise TypeError(name + ' must be a Specification')
+				setattr(
+					cls,
+					name,
+					specification,
+				)
+		if on_error is not None:
+			if not callable(on_error):
+				raise TypeError('on_error must be a callable event error policy')
+			cls.on_error = on_error
+		if setups is not None:
+			cls.setups = tuple(setups) if isinstance(
+				setups,
+				Sequence,
+			) else (setups,)
+		if completes is not None:
+			cls.completes = tuple(completes) if isinstance(
+				completes,
+				Sequence,
+			) else (completes,)
+		if errors is not None:
+			cls.errors = tuple(errors) if isinstance(
+				errors,
+				Sequence,
+			) else (errors,)
+		for hooks in (
+			cls.setups,
+			cls.completes,
+			cls.errors,
+		):
+			if not all(callable(hook) for hook in hooks):
+				raise TypeError('Run hooks must be callable')
+
+	def __call__(
+		self,
+		*args,
+		**kwargs,
+	) -> Any:
+		context = Context(
+			*args,
+			**kwargs,
+		)
+		for setup in self.setups:
+			setup(context)
+		try:
+			if self.request is not None:
+				if context.args:
+					raise TypeError('A specified task request requires keyword arguments')
+				body = self.request(context.kwargs)
+				if not isinstance(
+					body,
+					Mapping,
+				) or not all(isinstance(
+					key,
+					str,
+				) for key in body):
+					raise TypeError('A task request must remain a mapping with string keys')
+				context.kwargs = dict(body)
+			result = self.run(
+				*context.args,
+				**context.kwargs,
+			)
+			if self.response is not None:
+				result = self.response(result)
+		except Exception as error:
+			for handler in self.errors:
+				handler(
+					context,
+					error,
+				)
+			raise
+		for complete in self.completes:
+			complete(
+				context,
+				result,
+			)
+		return result
+
+	@abstractmethod
+	def run(
+		self,
+		**kwargs,
+	) -> Any:
+		raise NotImplementedError

DefStream/Worker/Specification.py

@@ -0,0 +1,91 @@
+from collections.abc import Mapping
+from copy import deepcopy
+from dataclasses import is_dataclass
+from re import fullmatch
+from typing import (
+	Any,
+	Callable,
+	Optional,
+)
+
+from ..Descriptor import (
+	DumpSchema,
+	Schema,
+	ToSchema,
+)
+from ..Descriptor.Value import (
+	Condition,
+	Descriptor,
+)
+
+__all__ = (
+	'Specification',
+)
+
+
+class Specification:
+	def __init__(
+		self,
+		schema: Any = None,
+		validator: Optional[Callable[[Any], Any]] = None,
+		example: Any = ...,
+		description: Optional[str] = None,
+		format: str = 'application/json',
+	):
+		if not isinstance(
+			format,
+			str,
+		) or fullmatch(
+			r"[!#$%&'+.^_`|~0-9A-Za-z-]+/[!#$%&'+.^_`|~0-9A-Za-z-]+(?:[ \t]*;[^\r\n]+)?",
+			format,
+		) is None:
+			raise ValueError('Specification format must be a concrete media type')
+		if validator is not None and not callable(validator):
+			raise TypeError('A Specification validator must be callable')
+		if isinstance(
+			schema,
+			type,
+		) and is_dataclass(schema):
+			schema = ToSchema(schema)
+		if schema is not None and not isinstance(
+			schema,
+			(
+				Schema,
+				Descriptor,
+				Condition,
+				Mapping,
+			),
+		):
+			raise TypeError('A Descriptor schema, schema mapping or dataclass type is required')
+		self.schema = schema
+		self.validator = validator
+		self.example = example
+		self.description = description
+		self.format = format
+		self.document()
+
+	def __call__(
+		self,
+		value: Any,
+	) -> Any:
+		return self.validator(value) if self.validator is not None else value
+
+	def document(self) -> dict:
+		payload = {} if self.schema is None else DumpSchema(
+			self.schema if isinstance(
+				self.schema,
+				Schema,
+			) else Schema(
+				'Payload',
+				self.schema,
+			),
+		)
+		message = {
+			'contentType': self.format,
+			'payload': payload,
+		}
+		if self.description is not None:
+			message['description'] = self.description
+		if self.example is not ...:
+			message['examples'] = [{'payload': deepcopy(self.example)}]
+		return message

DefStream/Worker/Worker.py

@@ -0,0 +1,321 @@
+from abc import (
+	ABC,
+	abstractmethod,
+)
+from collections.abc import Mapping
+from logging import getLogger
+from threading import Event as PyEvent
+from typing import (
+	Any,
+	List,
+	Callable,
+	Optional,
+	Sequence,
+	Type,
+	Tuple,
+	Union,
+	Protocol,
+)
+
+from ..Broker import (
+	Connection,
+	GetConsumer,
+	Consumer,
+	EventHandler,
+)
+from ..Broker.Event import Event
+from .Message import task_parameters
+from ..Inspector import Inspector
+from .Pool import (
+	Pool,
+	PoolFull,
+)
+from .Run import Run
+from .Executor import run_cleanups
+
+__all__ = (
+	'Worker',
+	'WorkerConnection',
+	'WorkerSetup',
+	'WorkerCleanup',
+)
+
+logger = getLogger(__name__)
+
+
+class WorkerConnection(
+	Connection,
+	GetConsumer,
+	Protocol,
+):
+	pass
+
+
+class WorkerSetup(ABC):
+	@abstractmethod
+	def __call__(
+		self,
+		worker: 'Worker',
+	) -> None:
+		raise NotImplementedError
+
+
+class WorkerCleanup(ABC):
+	@abstractmethod
+	def __call__(
+		self,
+		worker: 'Worker',
+	) -> None:
+		raise NotImplementedError
+
+
+def worker_hooks(
+	hooks: Optional[Union[Callable[['Worker'], None], Sequence[Callable[['Worker'], None]]]],
+) -> Tuple[Callable[['Worker'], None], ...]:
+	result = () if hooks is None else tuple(hooks) if isinstance(
+		hooks,
+		Sequence,
+	) else (hooks,)
+	if not all(callable(hook) for hook in result):
+		raise TypeError('Worker hooks must be callable')
+	return result
+
+
+class Worker:
+	def __init__(
+		self,
+		broker: WorkerConnection,
+		pool: Pool,
+		setups: Optional[Union[WorkerSetup, Sequence[WorkerSetup]]] = None,
+		cleanups: Optional[Union[WorkerCleanup, Sequence[WorkerCleanup]]] = None,
+		inspector: Optional[Inspector] = None,
+	):
+		self.broker = broker
+		self.pool = pool
+		self.tasks: List[Type[Run]] = []
+		self.consumer: Optional[Consumer] = None
+		self.started = False
+		self.running = PyEvent()
+		self.stopping = PyEvent()
+		self.finished = PyEvent()
+		self.pending = 0
+		self.setups = worker_hooks(setups)
+		self.cleanups = worker_hooks(cleanups)
+		self.inspector = inspector
+
+	def add(
+		self,
+		run: Type[Run],
+	) -> None:
+		if self.started:
+			raise RuntimeError('Cannot register runs after starting the worker')
+		task_parameters(run)
+		if run in self.tasks:
+			raise ValueError('Run is already registered')
+		self.tasks.append(run)
+
+	def run(self) -> None:
+		if self.started:
+			raise RuntimeError('Worker can only be started once')
+		if not self.tasks:
+			raise RuntimeError('Register at least one Run before starting')
+		self.started = True
+		try:
+			if self.inspector is not None:
+				self.inspector.start(self)
+			self.broker.connect()
+			for setup in self.setups:
+				setup(self)
+			self.consumer = self.broker.consumer()
+			for run in self.tasks:
+				self.consumer.subs(
+					run.parameters,
+					Handler(
+						self,
+						run,
+					),
+				)
+			if not self.stopping.is_set():
+				self.running.set()
+				self.consumer.run()
+		finally:
+			self.running.clear()
+			self.finished.set()
+			if self.pending:
+				logger.error(
+					'Consumer exited with %s unconfirmed deliveries',
+					self.pending,
+				)
+			try:
+				self.pool.stop()
+			finally:
+				try:
+					self.cleanup()
+				finally:
+					try:
+						if self.consumer is not None:
+							self.consumer.close()
+					finally:
+						try:
+							self.broker.close()
+						finally:
+							if self.inspector is not None and self.inspector.worker is self:
+								self.inspector.stop()
+
+	def cleanup(self) -> None:
+		run_cleanups(
+			self.cleanups,
+			self,
+		)
+
+	def stop(self) -> None:
+		self.stopping.set()
+		if self.consumer is not None and not self.finished.is_set():
+			self.consumer.call(self.try_stop)
+
+	def try_stop(self) -> None:
+		if self.stopping.is_set() and self.pending == 0 and self.consumer is not None:
+			self.consumer.stop()
+
+	def receive(
+		self,
+		run: Type[Run],
+		event: Event,
+	) -> None:
+		if self.stopping.is_set():
+			event.nack()
+			return
+		if not isinstance(
+			event.body,
+			Mapping,
+		):
+			self.failed(
+				run,
+				event,
+				TypeError('Run message body must be a mapping'),
+			)
+			return
+		self.pending += 1
+		consumer = self.consumer
+		if consumer is None:
+			self.pending -= 1
+			raise RuntimeError('Worker consumer is not initialized')
+		inspector = self.inspector
+		task = run.__module__ + '.' + run.__qualname__
+		contexts = inspector.begin(task) if inspector is not None else ()
+
+		def schedule(
+			error: Optional[BaseException] = None,
+			result: Any = None,
+		) -> None:
+			if inspector is not None:
+				inspector.end(
+					task,
+					contexts,
+					error is None,
+				)
+			if self.finished.is_set():
+				logger.error(
+					'Consumer exited before Run %s delivery could be confirmed',
+					run.__name__,
+				)
+				return
+			consumer.call(lambda: self.complete(
+				run,
+				event,
+				error,
+				result,
+			))
+
+		try:
+			self.pool.submit(
+				run,
+				dict(event.body),
+				callback=lambda result: schedule(result=result),
+				error_callback=schedule,
+			)
+		except PoolFull:
+			if inspector is not None:
+				inspector.end(
+					task,
+					contexts,
+					None,
+				)
+			self.pending -= 1
+			event.nack()
+			return
+		except Exception:
+			if inspector is not None:
+				inspector.end(
+					task,
+					contexts,
+					None,
+				)
+			self.pending -= 1
+			raise
+
+	def complete(
+		self,
+		run: Type[Run],
+		event: Event,
+		error: Optional[BaseException] = None,
+		result: Any = None,
+	) -> None:
+		try:
+			if error is not None:
+				self.failed(
+					run,
+					event,
+					error,
+				)
+			else:
+				if self.consumer is None:
+					raise RuntimeError('Worker consumer is not initialized')
+				self.consumer.complete(
+					event,
+					result,
+				)
+		finally:
+			self.pending -= 1
+			self.try_stop()
+
+	def failed(
+		self,
+		run: Type[Run],
+		event: Event,
+		error: BaseException,
+	) -> None:
+		logger.error(
+			'Run %s failed',
+			run.__name__,
+			exc_info=(
+				type(error),
+				error,
+				error.__traceback__,
+			),
+		)
+		if self.consumer is None:
+			raise RuntimeError('Worker consumer is not initialized')
+		self.consumer.fail(
+			event,
+			run.on_error,
+		)
+
+
+class Handler(EventHandler):
+	def __init__(
+		self,
+		worker: Worker,
+		run: Type[Run],
+	):
+		self.worker = worker
+		self.run = run
+
+	def __call__(
+		self,
+		event: Event,
+	) -> None:
+		self.worker.receive(
+			self.run,
+			event,
+		)

DefStream/Worker/__init__.py

@@ -0,0 +1,49 @@
+from .Worker import (
+	Worker,
+	WorkerConnection,
+	WorkerSetup,
+	WorkerCleanup,
+)
+from .Message import (
+	declare,
+	send,
+	request,
+)
+from .Run import (
+	Run,
+	Context,
+	RunSetup,
+	RunComplete,
+	RunError,
+)
+from .Specification import Specification
+from .Pool import (
+	Pool,
+	PoolFull,
+	PoolSetup,
+	PoolCleanup,
+	ThreadPool,
+	ProcessPool,
+)
+
+__all__ = (
+	'Worker',
+	'declare',
+	'send',
+	'request',
+	'WorkerConnection',
+	'WorkerSetup',
+	'WorkerCleanup',
+	'Run',
+	'Specification',
+	'Pool',
+	'PoolFull',
+	'PoolSetup',
+	'PoolCleanup',
+	'ThreadPool',
+	'ProcessPool',
+	'Context',
+	'RunSetup',
+	'RunComplete',
+	'RunError',
+)