DefStream 0.1.18__74-py3-none-any.whl

DefStream/Inspector/Extensions/Metrics.py

@@ -0,0 +1,316 @@
+from dataclasses import (
+	dataclass,
+	field,
+)
+from logging import getLogger
+from math import floor
+from threading import Lock
+from time import monotonic
+from typing import (
+	Dict,
+	Optional,
+	TypedDict,
+	TYPE_CHECKING,
+)
+from uuid import uuid4
+
+from ..Extension import Extension
+from ..Response import Response
+
+if TYPE_CHECKING:
+	from ..Inspector import Inspector
+
+__all__ = (
+	'Metrics',
+)
+
+
+class WindowSnapshot(TypedDict):
+	success: int
+	failure: int
+	duration: float
+	max_duration: float
+
+
+class TaskSnapshot(TypedDict):
+	active: int
+	success: int
+	failure: int
+	duration: float
+	oldest_active: float
+	windows: Dict[str, WindowSnapshot]
+
+
+class MetricsSnapshot(TypedDict):
+	instance: str
+	sampled_at: float
+	uptime: float
+	running: bool
+	tasks: Dict[str, TaskSnapshot]
+	window_end: int
+
+
+@dataclass
+class TaskMetrics:
+	active: int = 0
+	success: int = 0
+	failure: int = 0
+	duration: float = 0
+	starts: Dict[float, int] = field(default_factory=dict)
+	buckets: Dict[int, WindowSnapshot] = field(default_factory=dict)
+
+	def prune(
+		self,
+		now: float,
+	) -> None:
+		for second in tuple(self.buckets):
+			if second < floor(now) - 60:
+				del self.buckets[second]
+
+	def window(
+		self,
+		end: int,
+		seconds: int,
+	) -> WindowSnapshot:
+		buckets = [
+			bucket for second, bucket in self.buckets.items()
+			if end - seconds <= second < end
+		]
+		return {
+			'success': sum(bucket['success'] for bucket in buckets),
+			'failure': sum(bucket['failure'] for bucket in buckets),
+			'duration': sum(bucket['duration'] for bucket in buckets),
+			'max_duration': max(
+				(bucket['max_duration'] for bucket in buckets),
+				default=0,
+			),
+		}
+
+
+def label(
+	value: str,
+) -> str:
+	return value.replace(
+		'\\',
+		'\\\\',
+	).replace(
+		'\n',
+		'\\n',
+	).replace(
+		'"',
+		'\\"',
+	)
+
+
+class Metrics(Extension):
+	logger = getLogger('DefStream.Inspector.Extensions.Metrics')
+
+	def __init__(
+		self,
+		path: str,
+	):
+		super().__init__(path)
+		self.lock = Lock()
+		self.tasks: Dict[str, TaskMetrics] = {}
+		self.started_at: Optional[float] = None
+		self.instance = ''
+
+	def start(
+		self,
+		inspector: 'Inspector',
+	) -> None:
+		worker = inspector.worker
+		if worker is None:
+			raise RuntimeError('Metrics requires an attached Worker')
+		tasks = {}
+		for task in worker.tasks:
+			if task.name in tasks:
+				raise ValueError('Duplicate Metrics task name: ' + task.name)
+			tasks[task.name] = TaskMetrics()
+		with self.lock:
+			self.started_at = monotonic()
+			self.instance = uuid4().hex
+			self.tasks = tasks
+		self.logger.info(
+			'Metrics collection initialized: path=%s tasks=%d',
+			self.path,
+			len(self.tasks),
+		)
+
+	def begin(
+		self,
+		task: str,
+	) -> float:
+		with self.lock:
+			started = monotonic()
+			stats = self.tasks[task]
+			stats.active += 1
+			stats.starts[started] = stats.starts.get(
+				started,
+				0,
+			) + 1
+			self.logger.debug(
+				'Metrics task observation started: task=%s',
+				task,
+			)
+			return started
+
+	def end(
+		self,
+		task: str,
+		context: object,
+		success: Optional[bool],
+	) -> None:
+		if not isinstance(
+			context,
+			float,
+		):
+			raise TypeError('Metrics completion requires its submission timestamp')
+		with self.lock:
+			stats = self.tasks[task]
+			now = monotonic()
+			stats.starts[context] -= 1
+			if stats.starts[context] == 0:
+				del stats.starts[context]
+			stats.active -= 1
+			stats.prune(now)
+			if success is not None:
+				stats.success += int(success)
+				stats.failure += int(not success)
+				duration = now - context
+				stats.duration += duration
+				bucket = stats.buckets.setdefault(
+					floor(now),
+					{
+						'success': 0,
+						'failure': 0,
+						'duration': 0,
+						'max_duration': 0,
+					},
+				)
+				bucket['success'] += int(success)
+				bucket['failure'] += int(not success)
+				bucket['duration'] += duration
+				bucket['max_duration'] = max(
+					bucket['max_duration'],
+					duration,
+				)
+			else:
+				duration = now - context
+			self.logger.debug(
+				'Metrics task observation completed: task=%s success=%s duration=%.6f active=%d',
+				task,
+				success,
+				duration,
+				stats.active,
+			)
+
+	def snapshot(
+		self,
+		inspector: 'Inspector',
+	) -> MetricsSnapshot:
+		worker = inspector.worker
+		with self.lock:
+			if self.started_at is None:
+				raise RuntimeError('Metrics has not been started')
+			sampled_at = monotonic()
+			for stats in self.tasks.values():
+				stats.prune(sampled_at)
+			return {
+				'instance': self.instance,
+				'sampled_at': sampled_at,
+				'uptime': sampled_at - self.started_at,
+				'running': worker is not None and worker.running.is_set() and not worker.stopping.is_set() and not worker.finished.is_set(),
+				'window_end': floor(sampled_at),
+				'tasks': {
+					task: {
+						'active': stats.active,
+						'success': stats.success,
+						'failure': stats.failure,
+						'duration': stats.duration,
+						'oldest_active': sampled_at - min(stats.starts) if stats.starts else 0,
+						'windows': {
+							str(seconds): stats.window(
+								floor(sampled_at),
+								seconds,
+							)
+							for seconds in (
+								1,
+								60,
+							)
+						},
+					}
+					for task, stats in self.tasks.items()
+				},
+			}
+		self.logger.debug(
+			'Metrics snapshot built: tasks=%d running=%s',
+			len(snapshot['tasks']),
+			snapshot['running'],
+		)
+		return snapshot
+
+	def render(
+		self,
+		inspector: 'Inspector',
+	) -> str:
+		self.logger.debug('Rendering Metrics in Prometheus format')
+		snapshot = self.snapshot(inspector)
+		running = snapshot['running']
+		uptime = snapshot['uptime']
+		lines = [
+			'# HELP defstream_worker_running Worker consumer lifecycle is running; not a heartbeat probe.',
+			'# TYPE defstream_worker_running gauge',
+			f'defstream_worker_running {int(running)}',
+			'# HELP defstream_worker_uptime_seconds Seconds since Inspector startup.',
+			'# TYPE defstream_worker_uptime_seconds gauge',
+			f'defstream_worker_uptime_seconds {uptime}',
+			'# HELP defstream_tasks_active Submitted task executions not yet completed.',
+			'# TYPE defstream_tasks_active gauge',
+			'# HELP defstream_tasks_total Completed execution attempts including retries.',
+			'# TYPE defstream_tasks_total counter',
+			'# HELP defstream_task_duration_seconds End-to-end execution time including pool wait and hooks.',
+			'# TYPE defstream_task_duration_seconds summary',
+			'# HELP defstream_task_oldest_active_seconds Age of oldest submitted unfinished attempt including pool wait.',
+			'# TYPE defstream_task_oldest_active_seconds gauge',
+			'# HELP defstream_tasks_window Completed attempts in the last fully elapsed second or minute window.',
+			'# TYPE defstream_tasks_window gauge',
+			'# HELP defstream_task_window_duration_seconds_sum Total duration of attempts completed in the window.',
+			'# TYPE defstream_task_window_duration_seconds_sum gauge',
+			'# HELP defstream_task_window_duration_seconds_max Maximum duration of attempts completed in the window.',
+			'# TYPE defstream_task_window_duration_seconds_max gauge',
+		]
+		for task, stats in snapshot['tasks'].items():
+			labels = 'task="' + label(task) + '"'
+			lines.extend((
+				f'defstream_tasks_active{{{labels}}} {stats["active"]}',
+				f'defstream_tasks_total{{{labels},outcome="success"}} {stats["success"]}',
+				f'defstream_tasks_total{{{labels},outcome="failure"}} {stats["failure"]}',
+				f'defstream_task_duration_seconds_count{{{labels}}} {stats["success"] + stats["failure"]}',
+				f'defstream_task_duration_seconds_sum{{{labels}}} {stats["duration"]}',
+				f'defstream_task_oldest_active_seconds{{{labels}}} {stats["oldest_active"]}',
+			))
+			for seconds, window in stats['windows'].items():
+				window_labels = labels + ',window_seconds="' + seconds + '"'
+				lines.extend((
+					f'defstream_tasks_window{{{window_labels},outcome="success"}} {window["success"]}',
+					f'defstream_tasks_window{{{window_labels},outcome="failure"}} {window["failure"]}',
+					f'defstream_task_window_duration_seconds_sum{{{window_labels}}} {window["duration"]}',
+					f'defstream_task_window_duration_seconds_max{{{window_labels}}} {window["max_duration"]}',
+				))
+		return '\n'.join(lines) + '\n'
+
+	def __call__(
+		self,
+		inspector: 'Inspector',
+	) -> Response:
+		body = self.render(inspector).encode('utf-8')
+		self.logger.debug(
+			'Metrics HTTP response built: bytes=%d',
+			len(body),
+		)
+		return Response(
+			'200 OK',
+			body,
+			'text/plain; version=0.0.4; charset=utf-8',
+		)

DefStream/Inspector/Extensions/Ping.py

@@ -0,0 +1,107 @@
+from json import dumps
+from logging import getLogger
+from math import isfinite
+from threading import (
+	Event,
+	Lock,
+)
+from typing import (
+	Optional,
+	TYPE_CHECKING,
+)
+
+from ..Extension import Extension
+from ..Response import Response
+
+if TYPE_CHECKING:
+	from ..Inspector import Inspector
+
+__all__ = (
+	'Ping',
+)
+
+class Ping(Extension):
+	logger = getLogger('DefStream.Inspector.Extensions.Ping')
+
+	def __init__(
+		self,
+		path: str,
+		timeout: float = 1,
+	):
+		super().__init__(path)
+		if isinstance(
+			timeout,
+			bool,
+		) or not isfinite(timeout) or timeout <= 0:
+			raise ValueError('Ping probe timeout must be finite and positive')
+		self.timeout = timeout
+		self.lock = Lock()
+		self.probe: Optional[Event] = None
+
+	def state(
+		self,
+		inspector: 'Inspector',
+	) -> str:
+		worker = inspector.worker
+		if worker is None:
+			return 'starting'
+		if worker.finished.is_set():
+			return 'stopped'
+		if worker.stopping.is_set():
+			return 'stopping'
+		if not worker.running.is_set():
+			return 'starting'
+		return 'running'
+
+	def healthy(
+		self,
+		inspector: 'Inspector',
+	) -> bool:
+		if self.state(inspector) != 'running':
+			return False
+		worker = inspector.worker
+		if worker is None or worker.consumer is None:
+			return False
+		with self.lock:
+			if self.probe is None or self.probe.is_set():
+				self.probe = Event()
+				try:
+					self.logger.debug('Scheduling Ping Consumer probe')
+					worker.consumer.call(self.probe.set)
+				except Exception as error:
+					self.probe = None
+					self.logger.error(
+						'Ping Consumer probe scheduling failed: error=%s',
+						type(error).__name__,
+					)
+					return False
+			probe = self.probe
+		responded = probe.wait(self.timeout)
+		healthy = responded and self.state(inspector) == 'running'
+		self.logger.debug(
+			'Ping Consumer probe completed: responded=%s healthy=%s',
+			responded,
+			healthy,
+		)
+		return healthy
+
+	def __call__(
+		self,
+		inspector: 'Inspector',
+	) -> Response:
+		healthy = self.healthy(inspector)
+		state = 'running' if healthy else self.state(inspector)
+		if not healthy and state == 'running':
+			state = 'unresponsive'
+		self.logger.debug(
+			'Building Ping response: status=%s state=%s',
+			'200' if healthy else '503',
+			state,
+		)
+		return Response(
+			'200 OK' if healthy else '503 Service Unavailable',
+			dumps({
+				'status': state,
+			}).encode('utf-8'),
+			'application/json; charset=utf-8',
+		)

DefStream/Inspector/Extensions/Specifications.py

@@ -0,0 +1,176 @@
+from collections.abc import Mapping
+from json import dumps
+from logging import getLogger
+from re import fullmatch
+from typing import (
+	Any,
+	TYPE_CHECKING,
+	Optional,
+)
+
+from ..Extension import Extension
+from ..Response import Response
+
+if TYPE_CHECKING:
+	from ..Inspector import Inspector
+
+__all__ = (
+	'Specifications',
+)
+
+
+def reference(
+	*parts: str,
+) -> dict:
+	return {'$ref': '#/' + '/'.join(part.replace(
+		'~',
+		'~0',
+	).replace(
+		'/',
+		'~1',
+	) for part in parts)}
+
+
+class Specifications(Extension):
+	logger = getLogger('DefStream.Inspector.Extensions.Specifications')
+
+	def __init__(
+		self,
+		path: str,
+		version: str = '1.0.0',
+		description: Optional[str] = None,
+		format: str = 'application/json',
+	):
+		super().__init__(path)
+		if not isinstance(
+			version,
+			str,
+		) or not version:
+			raise ValueError('A document version is required')
+		self.version = version
+		self.description = description
+		if not isinstance(
+			format,
+			str,
+		) or fullmatch(
+			r"[!#$%&'+.^_`|~0-9A-Za-z-]+/[!#$%&'+.^_`|~0-9A-Za-z-]+(?:[ \t]*;[^\r\n]+)?",
+			format,
+		) is None:
+			raise ValueError('Specifications format must be a concrete media type')
+		self.format = format
+
+	def document(
+		self,
+		inspector: 'Inspector',
+	) -> dict:
+		self.logger.debug('Building Specifications AsyncAPI document')
+		worker = inspector.worker
+		if worker is None:
+			raise RuntimeError('Specifications requires an attached Worker')
+		info = {
+			'title': 'DefStream',
+			'version': self.version,
+		}
+		if self.description is not None:
+			info['description'] = self.description
+		channels: dict[str, Any] = {}
+		operations: dict[str, Any] = {}
+		for task in worker.tasks:
+			key = task.name
+			if key in operations:
+				raise ValueError('Duplicate task specification identifier: ' + key)
+			parameters = task.parameters
+			if isinstance(
+				parameters,
+				str,
+			):
+				queue = parameters
+			elif isinstance(
+				parameters,
+				Mapping,
+			):
+				queue = parameters.get('queue')
+			else:
+				queue = getattr(
+					parameters,
+					'queue',
+					None,
+				)
+			if not isinstance(
+				queue,
+				str,
+			) or not queue:
+				raise ValueError('Task specification requires a nonempty queue address: ' + key)
+			if queue in channels:
+				raise ValueError('Duplicate channel identifier: ' + queue)
+			message: dict[str, Any] = task.request.document() if task.request is not None else {
+				'payload': {'type': 'object'},
+			}
+			message['name'] = task.__name__ + 'Request'
+			channels[queue] = {
+				'address': queue,
+				'messages': {'request': message},
+			}
+			operation = {
+				'action': 'receive',
+				'channel': reference(
+					'channels',
+					queue,
+				),
+				'messages': [reference(
+					'channels',
+					queue,
+					'messages',
+					'request',
+				)],
+			}
+			if task.description is not None:
+				operation['description'] = task.description
+			if task.response is not None:
+				channels[queue]['messages']['response'] = task.response.document()
+				operation['reply'] = {
+					'channel': reference(
+						'channels',
+						queue,
+					),
+					'messages': [reference(
+						'channels',
+						queue,
+						'messages',
+						'response',
+					)],
+				}
+			operations[key] = operation
+		document = {
+			'asyncapi': '3.1.0',
+			'info': info,
+			'defaultContentType': self.format,
+			'channels': channels,
+			'operations': operations,
+		}
+		self.logger.debug(
+			'Specifications AsyncAPI document built: tasks=%d channels=%d',
+			len(operations),
+			len(channels),
+		)
+		return document
+
+	def __call__(
+		self,
+		inspector: 'Inspector',
+	) -> Response:
+		body = dumps(
+			self.document(inspector),
+			ensure_ascii=False,
+			allow_nan=False,
+			indent=2,
+		).encode('utf-8')
+		self.logger.debug(
+			'Specifications HTTP response built: bytes=%d',
+			len(body),
+		)
+		return Response(
+			'200 OK',
+			body,
+			'application/json; charset=utf-8',
+		)