DefStream 0.1.22__93-py3-none-any.whl

DefStream/Worker/Run.py

@@ -0,0 +1,357 @@
+from abc import (
+	ABC,
+	ABCMeta,
+	abstractmethod,
+)
+from collections.abc import Mapping
+from logging import getLogger
+from re import fullmatch
+from time import monotonic
+from typing import (
+	Any,
+	Dict,
+	Optional,
+	Sequence,
+	Tuple,
+	Union,
+)
+
+from ..Broker.Event import EventErrorPolicy
+from ..Broker.Priority import validate_priority
+from .Specification import Specification
+
+__all__ = (
+	'Context',
+	'RunSetup',
+	'RunComplete',
+	'RunError',
+	'Run',
+)
+
+
+def priority_setting(
+	parameters: Any,
+	name: str,
+) -> Optional[int]:
+	if isinstance(
+		parameters,
+		Mapping,
+	):
+		return parameters.get(name)
+	return getattr(
+		parameters,
+		name,
+		None,
+	)
+
+
+class Context:
+	def __init__(
+		self,
+		*args,
+		**kwargs,
+	):
+		self.args = args
+		self.kwargs = kwargs
+
+
+class RunSetup(ABC):
+	@abstractmethod
+	def __call__(
+		self,
+		context: Context,
+	) -> None:
+		raise NotImplementedError
+
+
+class RunComplete(ABC):
+	@abstractmethod
+	def __call__(
+		self,
+		context: Context,
+		completion: Any,
+	) -> None:
+		raise NotImplementedError
+
+
+class RunError(ABC):
+	@abstractmethod
+	def __call__(
+		self,
+		context: Context,
+		error: Exception,
+	) -> None:
+		raise NotImplementedError
+
+
+class RunMeta(ABCMeta):
+	"""구버전 ABCMeta의 클래스 이름 인자와 Run의 name 키워드 충돌을 방지한다."""
+
+	def __new__(
+		mcls,
+		class_name: str,
+		bases: Tuple[type, ...],
+		namespace: Dict[str, Any],
+		/,
+		**kwargs: Any,
+	) -> 'RunMeta':
+		if 'name' in kwargs:
+			kwargs['run_name'] = kwargs.pop('name')
+		return super().__new__(
+			mcls,
+			class_name,
+			bases,
+			namespace,
+			**kwargs,
+		)
+
+
+class Run(ABC, metaclass=RunMeta):
+	logger = getLogger('Task')
+	parameters: Any = None
+	name: str = __module__ + '.' + __qualname__
+	description: Optional[str] = None
+	on_error: Optional[EventErrorPolicy] = None
+	setups: Tuple[RunSetup, ...] = ()
+	completes: Tuple[RunComplete, ...] = ()
+	errors: Tuple[RunError, ...] = ()
+	request: Optional[Specification] = None
+	response: Optional[Specification] = None
+
+	def __init_subclass__(
+		cls,
+		*,
+		run_name: Optional[str] = None,
+		description: Optional[str] = None,
+		parameters: Any = None,
+		on_error: Optional[EventErrorPolicy] = None,
+		setups: Optional[Union[RunSetup, Sequence[RunSetup]]] = None,
+		completes: Optional[Union[RunComplete, Sequence[RunComplete]]] = None,
+		errors: Optional[Union[RunError, Sequence[RunError]]] = None,
+		request: Optional[Specification] = None,
+		response: Optional[Specification] = None,
+		**kwargs,
+	):
+		super().__init_subclass__(**kwargs)
+		if run_name is not None and not isinstance(
+			run_name,
+			str,
+		):
+			raise TypeError('Run name must be a string or None')
+		if run_name is not None and fullmatch(
+			r'[A-Za-z0-9_.-]+',
+			run_name,
+		) is None:
+			raise ValueError('Run name must contain only ASCII letters, digits, underscores, dots or hyphens')
+		if description is not None and not isinstance(
+			description,
+			str,
+		):
+			raise TypeError('Run description must be a string or None')
+		cls.name = run_name if run_name is not None else cls.__module__ + '.' + cls.__qualname__
+		cls.description = description
+		parents = tuple(base for base in cls.__mro__[1:] if issubclass(
+			base,
+			Run,
+		))
+		if parameters is not None:
+			cls.parameters = parameters
+		for name in (
+			'priority',
+			'max_priority',
+		):
+			if any(priority_setting(
+				base.parameters,
+				name,
+			) is not None for base in parents) and priority_setting(
+				cls.parameters,
+				name,
+			) is None:
+				raise ValueError('Inherited task ' + name + ' cannot be cleared')
+		maximum = priority_setting(
+			cls.parameters,
+			'max_priority',
+		)
+		validate_priority(
+			maximum,
+			None,
+			minimum=1,
+			name='max_priority',
+		)
+		validate_priority(
+			priority_setting(
+				cls.parameters,
+				'priority',
+			),
+			maximum,
+		)
+		for name, specification in (
+			(
+				'request',
+				request,
+			),
+			(
+				'response',
+				response,
+			),
+		):
+			if specification is not None:
+				if not isinstance(
+					specification,
+					Specification,
+				):
+					raise TypeError(name + ' must be a Specification')
+				setattr(
+					cls,
+					name,
+					specification,
+				)
+		if on_error is not None:
+			if not callable(on_error):
+				raise TypeError('on_error must be a callable failure policy')
+			cls.on_error = on_error
+		if setups is not None:
+			cls.setups = tuple(setups) if isinstance(
+				setups,
+				Sequence,
+			) else (setups,)
+		if completes is not None:
+			cls.completes = tuple(completes) if isinstance(
+				completes,
+				Sequence,
+			) else (completes,)
+		if errors is not None:
+			cls.errors = tuple(errors) if isinstance(
+				errors,
+				Sequence,
+			) else (errors,)
+		for hooks in (
+			cls.setups,
+			cls.completes,
+			cls.errors,
+		):
+			if not all(callable(hook) for hook in hooks):
+				raise TypeError('Run hooks must be callable')
+		return
+
+	def __call__(
+		self,
+		*args,
+		**kwargs,
+	) -> Any:
+		task = type(self).__module__ + '.' + type(self).__qualname__
+		started = monotonic()
+		self.logger.debug(
+			'Starting task execution: task=%s',
+			task,
+		)
+		context = Context(
+			*args,
+			**kwargs,
+		)
+		for setup in self.setups:
+			self.logger.debug(
+				'Starting task setup: task=%s hook=%s',
+				task,
+				type(setup).__qualname__,
+			)
+			setup(context)
+			self.logger.debug(
+				'Task setup completed: task=%s hook=%s',
+				task,
+				type(setup).__qualname__,
+			)
+		try:
+			if self.request is not None:
+				self.logger.debug(
+					'Validating task input: task=%s',
+					task,
+				)
+				if context.args:
+					raise TypeError('A specified task request requires keyword arguments')
+				body = self.request(context.kwargs)
+				if not isinstance(
+					body,
+					Mapping,
+				) or not all(isinstance(
+					key,
+					str,
+				) for key in body):
+					raise TypeError('A task request must remain a mapping with string keys')
+				context.kwargs = dict(body)
+				self.logger.debug(
+					'Task input validated: task=%s',
+					task,
+				)
+			self.logger.debug(
+				'Calling user task: task=%s',
+				task,
+			)
+			result = self.run(
+				*context.args,
+				**context.kwargs,
+			)
+			self.logger.debug(
+				'User task returned: task=%s',
+				task,
+			)
+			if self.response is not None:
+				self.logger.debug(
+					'Validating task output: task=%s',
+					task,
+				)
+				result = self.response(result)
+				self.logger.debug(
+					'Task output validated: task=%s',
+					task,
+				)
+		except Exception as error:
+			self.logger.debug(
+				'Task execution or validation failed: task=%s error_type=%s',
+				task,
+				type(error).__qualname__,
+			)
+			for handler in self.errors:
+				self.logger.debug(
+					'Starting task error hook: task=%s hook=%s',
+					task,
+					type(handler).__qualname__,
+				)
+				handler(
+					context,
+					error,
+				)
+				self.logger.debug(
+					'Task error hook completed: task=%s hook=%s',
+					task,
+					type(handler).__qualname__,
+				)
+			raise
+		for complete in self.completes:
+			self.logger.debug(
+				'Starting task completion hook: task=%s hook=%s',
+				task,
+				type(complete).__qualname__,
+			)
+			complete(
+				context,
+				result,
+			)
+			self.logger.debug(
+				'Task completion hook completed: task=%s hook=%s',
+				task,
+				type(complete).__qualname__,
+			)
+		self.logger.debug(
+			'Task execution completed: task=%s duration=%s',
+			task,
+			monotonic() - started,
+		)
+		return result
+
+	@abstractmethod
+	def run(
+		self,
+		**kwargs,
+	) -> Any:
+		raise NotImplementedError

DefStream/Worker/Specification.py

@@ -0,0 +1,87 @@
+from collections.abc import Mapping
+from copy import deepcopy
+from dataclasses import is_dataclass
+from re import fullmatch
+from typing import (
+	Any,
+	Callable,
+	Optional,
+)
+
+from ..Descriptor import (
+	DumpSchema,
+	Schema,
+	ToSchema,
+)
+from ..Descriptor.Value import (
+	Condition,
+	Descriptor,
+)
+
+__all__ = (
+	'Specification',
+)
+
+
+class Specification:
+	def __init__(
+		self,
+		schema: Any = None,
+		validator: Optional[Callable[[Any], Any]] = None,
+		example: Any = ...,
+		format: str = 'application/json',
+	):
+		if not isinstance(
+			format,
+			str,
+		) or fullmatch(
+			r"[!#$%&'+.^_`|~0-9A-Za-z-]+/[!#$%&'+.^_`|~0-9A-Za-z-]+(?:[ \t]*;[^\r\n]+)?",
+			format,
+		) is None:
+			raise ValueError('Specification format must be a concrete media type')
+		if validator is not None and not callable(validator):
+			raise TypeError('A Specification validator must be callable')
+		if isinstance(
+			schema,
+			type,
+		) and is_dataclass(schema):
+			schema = ToSchema(schema)
+		if schema is not None and not isinstance(
+			schema,
+			(
+				Schema,
+				Descriptor,
+				Condition,
+				Mapping,
+			),
+		):
+			raise TypeError('A Descriptor schema, schema mapping or dataclass type is required')
+		self.schema = schema
+		self.validator = validator
+		self.example = example
+		self.format = format
+		self.document()
+
+	def __call__(
+		self,
+		value: Any,
+	) -> Any:
+		return self.validator(value) if self.validator is not None else value
+
+	def document(self) -> dict:
+		payload = {} if self.schema is None else DumpSchema(
+			self.schema if isinstance(
+				self.schema,
+				Schema,
+			) else Schema(
+				'Payload',
+				self.schema,
+			),
+		)
+		message = {
+			'contentType': self.format,
+			'payload': payload,
+		}
+		if self.example is not ...:
+			message['examples'] = [{'payload': deepcopy(self.example)}]
+		return message

DefStream/Worker/Startup.py

@@ -0,0 +1,174 @@
+from collections.abc import Mapping
+from os import environ, getpid
+import sys
+from typing import (
+	Any,
+	TYPE_CHECKING,
+)
+
+if TYPE_CHECKING:
+	from .Worker import Worker
+
+__all__ = (
+	'startup',
+)
+
+
+LOGO = r"""
+▄▄▄▄▄▄           ▄▄  ▄▄▄▄▄▄▄                                  
+███▀▀██▄        ██  █████▀▀▀  ██                              
+███  ███ ▄█▀█▄ ▀██▀  ▀████▄  ▀██▀▀ ████▄ ▄█▀█▄  ▀▀█▄ ███▄███▄ 
+███  ███ ██▄█▀  ██     ▀████  ██   ██ ▀▀ ██▄█▀ ▄█▀██ ██ ██ ██ 
+██████▀  ▀█▄▄▄  ██  ███████▀  ██   ██    ▀█▄▄▄ ▀█▄██ ██ ██ ██ 
+
+Just def. Let workers do the work. You don't need to await this.
+"""
+
+
+def supports_color() -> bool:
+	return (
+		getattr(sys.stdout, 'isatty', lambda: False)()
+		and not environ.get('NO_COLOR')
+	)
+
+
+def colorize_logo(logo: str) -> str:
+	colors = (
+		'\033[38;5;39m',
+		'\033[38;5;45m',
+		'\033[38;5;51m',
+		'\033[38;5;45m',
+		'\033[38;5;39m',
+	)
+	reset = '\033[0m'
+	colorized_lines = []
+	art_line = 0
+	for line in logo.splitlines(keepends=True):
+		if line.strip() and art_line < len(colors):
+			newline = '\n' if line.endswith('\n') else ''
+			colorized_lines.append(
+				f'{colors[art_line]}{line[:-1] if newline else line}'
+				f'{reset}{newline}',
+			)
+			art_line += 1
+		else:
+			colorized_lines.append(line)
+	return ''.join(colorized_lines)
+
+
+def subscription_value(
+	parameters: Any,
+	name: str,
+) -> Any:
+	if isinstance(
+		parameters,
+		str,
+	):
+		return parameters if name == 'queue' else None
+	if isinstance(
+		parameters,
+		Mapping,
+	):
+		return parameters.get(name)
+	return getattr(
+		parameters,
+		name,
+		None,
+	)
+
+
+def startup(worker: 'Worker') -> None:
+	broker = type(worker.broker)
+	broker_name = broker.__qualname__
+	for base in broker.__mro__:
+		if '.Broker.Implements.' in base.__module__:
+			broker_name = base.__module__.split('.Broker.Implements.')[1].split('.')[0]
+			break
+	pool = type(worker.pool)
+	conf = getattr(
+		worker.broker,
+		'conf',
+		None,
+	)
+	host = getattr(
+		conf,
+		'host',
+		None,
+	)
+	port = getattr(
+		conf,
+		'port',
+		None,
+	)
+	if isinstance(
+		host,
+		str,
+	):
+		address = f'[{host}]' if ':' in host and not host.startswith('[') else host
+		broker_name += f'://{address}'
+		if isinstance(
+			port,
+			(
+				str,
+				int,
+			),
+		):
+			broker_name += f':{port}'
+		vhost = getattr(
+			conf,
+			'vhost',
+			None,
+		)
+		if isinstance(
+			vhost,
+			str,
+		) and vhost:
+			broker_name += '/' + vhost.lstrip('/')
+	logo = LOGO.rstrip()
+	starting_worker = 'Starting Worker'
+	if supports_color():
+		logo = colorize_logo(logo)
+		starting_worker = f'\033[1;38;5;51m{starting_worker}\033[0m'
+	lines = [
+		logo,
+		'',
+		starting_worker,
+		f'  PID: {getpid()}',
+		f'  Broker: {broker_name}',
+	]
+	lines.extend((
+		f'  Pool: {pool.__name__} (concurrency={worker.pool.size()})',
+		f'  Tasks: {len(worker.tasks)} tasks',
+	))
+	queues = []
+	for task in worker.tasks:
+		name = task.__qualname__
+		queue = subscription_value(
+			task.parameters,
+			'queue',
+		)
+		lines.append(f'    - {name}')
+		lines.append(f'      queue={queue!r}' if queue is not None else '      queue=<adapter-defined>')
+		if queue is not None and queue not in queues:
+			queues.append(queue)
+		for option in (
+			'group',
+			'qos',
+			'offset',
+			'error',
+		):
+			value = subscription_value(
+				task.parameters,
+				option,
+			)
+			if value is not None:
+				lines.append(f'      {option}={value!r}')
+		policy = type(task.on_error).__name__ if task.on_error is not None else 'broker default'
+		lines.append(f'      on_error={policy}')
+	lines.append(f'  Queues: {len(queues)} queues')
+	lines.extend(f'    - {queue!r}' for queue in queues)
+	lines.append('\n')
+	print(
+		'\n'.join(lines),
+		flush=True,
+	)